DCCONSTRUCTIONS
e515ab1b8c
Preserve the completed teach-and-repeat laboratory stage: reference preparation, cascaded acquisition, local tracking and recovery, recording lifecycle, replay qualification, and persistent Rerun scene controls. Document the open grid-picking regression and Rerun upgrade contract. No autonomous driving or loop-closure optimization is claimed.
1998 lines
76 KiB
Python
1998 lines
76 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import shutil
|
|
from collections.abc import AsyncIterator, Iterable
|
|
from contextlib import asynccontextmanager, suppress
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect
|
|
from fastapi.exceptions import RequestValidationError
|
|
from fastapi.responses import JSONResponse
|
|
from pydantic import ValidationError
|
|
from starlette.middleware.gzip import GZipMiddleware
|
|
|
|
from k1link import __version__
|
|
from k1link.artifact_gateway import configured_artifact_gateway
|
|
from k1link.compute import (
|
|
IntegratedPerceptionOverlayStore,
|
|
RecordedCalibratedFusionStore,
|
|
RecordedPerceptionEpochStore,
|
|
RecordedPerceptionOverlayMux,
|
|
RecordedPerceptionOverlayStore,
|
|
)
|
|
from k1link.compute.pipeline_telemetry import JsonlPipelineTelemetrySink
|
|
from k1link.laboratory import (
|
|
LaboratoryEvidenceRegistry,
|
|
LaboratoryEvidenceReportService,
|
|
LaboratoryExecutionRegistry,
|
|
LaboratoryRunner,
|
|
LaboratoryValueReviewRegistry,
|
|
)
|
|
from k1link.laboratory.m48_raw_evidence import (
|
|
M48_EXPECTED_THREAT_RESULT_ID,
|
|
M48RawEvidenceError,
|
|
M48RawEvidenceReader,
|
|
)
|
|
from k1link.observatory import (
|
|
LaboratorySetupRegistry,
|
|
LaboratorySetupRegistryError,
|
|
ObservatoryRunPreparationLedger,
|
|
load_observatory_run_preparation_ledger,
|
|
)
|
|
from k1link.observatory.composition_runs import CompositionRunError, CompositionRunStore
|
|
from k1link.observatory.domain_ontology import (
|
|
ObservatoryDomainOntology,
|
|
ObservatoryOntologyError,
|
|
)
|
|
from k1link.observatory.lab_view_profiles import LabViewProfileError, LabViewProfileStore
|
|
from k1link.observatory.m49_queue_binding import (
|
|
M49QueueBindingConfig,
|
|
M49QueueBindingError,
|
|
M49RecordedQueueBindingService,
|
|
)
|
|
from k1link.observatory.modular_composition import CompositionError, ModuleRegistry
|
|
from k1link.observatory.modular_composition_store import ModularCompositionStore
|
|
from k1link.observatory.portable_publication_reconciler import (
|
|
PortablePublicationReconciler,
|
|
)
|
|
from k1link.observatory.portable_queue_binding import (
|
|
PortableQueueBindingError,
|
|
PortableRecordedQueueBindingService,
|
|
)
|
|
from k1link.observatory.portable_replay import PortableReplayService
|
|
from k1link.observatory.portable_result_cache import PortableResultCache
|
|
from k1link.observatory.portable_result_contract import (
|
|
PortableCalculationProfileRegistry,
|
|
PortableResultContractValidatorRegistry,
|
|
)
|
|
from k1link.observatory.portable_result_publisher import (
|
|
resolve_published_portable_calculation_profile,
|
|
)
|
|
from k1link.observatory.portable_result_view import PortableResultViewService
|
|
from k1link.observatory.portable_run_definitions import (
|
|
PortableRunDefinitionRegistry,
|
|
PortableRunDefinitionRegistryError,
|
|
)
|
|
from k1link.observatory.portable_setup_projection import (
|
|
PortableSetupProjectionError,
|
|
PortableSetupProjector,
|
|
portable_calculation_profile_registry,
|
|
)
|
|
from k1link.observatory.portable_worker_integration import (
|
|
OBSERVATORY_WORKER_LOCAL_ENABLED_ENV,
|
|
PortableObservatoryWorkerIntegration,
|
|
PortableWorkerIntegrationError,
|
|
PortableWorkerStorageRoots,
|
|
build_portable_observatory_worker_integration,
|
|
observatory_worker_local_enabled,
|
|
portable_result_validator_registry,
|
|
)
|
|
from k1link.observatory.recorded_jobs import (
|
|
ObservatoryRecordedJobQueue,
|
|
ObservatoryRecordedQueueError,
|
|
RecordedRunDefinitionRegistry,
|
|
)
|
|
from k1link.sessions import (
|
|
MaterializedRecording,
|
|
RecordedCameraFrameService,
|
|
RecordedCameraPlaybackSource,
|
|
RecordedMediaInspector,
|
|
RecordedMediaManifest,
|
|
RecordingPreparationQueueFull,
|
|
ReplayCommand,
|
|
SessionRecordingMaterializer,
|
|
SessionRecordingPreparationManager,
|
|
SessionStore,
|
|
)
|
|
from k1link.sessions.models import SessionSummary
|
|
from k1link.simulation.projects import SimulationProjectService, SimulationProjectStore
|
|
from k1link.web.advanced_laboratory_api import build_advanced_laboratory_router
|
|
from k1link.web.artifact_health_api import build_artifact_health_router
|
|
from k1link.web.compute_contour_api import build_compute_contour_router
|
|
from k1link.web.fleet_api import router as fleet_router
|
|
from k1link.web.device_plugin_composition import load_installed_device_plugins
|
|
from k1link.web.e30_engineering_api import build_e30_engineering_router
|
|
from k1link.web.e30_human_review_api import build_e30_human_review_router
|
|
from k1link.web.e30_review_api import build_e30_review_router
|
|
from k1link.web.e40_case_review_api import build_e40_case_review_router
|
|
from k1link.web.e46_blind_review_api import build_e46_blind_review_router
|
|
from k1link.web.e46a_ai_engineering_preannotation_api import (
|
|
build_e46a_ai_engineering_preannotation_router,
|
|
)
|
|
from k1link.web.e46b_temporal_motion_api import build_e46b_temporal_motion_router
|
|
from k1link.web.e46c_full_replay_world_tracks_api import (
|
|
build_e46c_full_replay_world_tracks_router,
|
|
)
|
|
from k1link.web.e46d_temporal_failure_audit_api import (
|
|
build_e46d_temporal_failure_audit_router,
|
|
)
|
|
from k1link.web.e46e_ready_stack_api import build_e46e_ready_stack_router
|
|
from k1link.web.e46f_dashcam_bakeoff_api import build_e46f_dashcam_bakeoff_router
|
|
from k1link.web.e46g_rectified_detector_bakeoff_api import (
|
|
build_e46g_rectified_detector_bakeoff_router,
|
|
)
|
|
from k1link.web.e46h_full_rectified_front_replay_api import (
|
|
build_e46h_full_rectified_front_replay_router,
|
|
)
|
|
from k1link.web.e46i_grounding_dino_full_replay_api import (
|
|
build_e46i_grounding_dino_full_replay_router,
|
|
)
|
|
from k1link.web.e46j_raw_fisheye_realtime_api import (
|
|
build_e46j_raw_fisheye_realtime_router,
|
|
)
|
|
from k1link.web.e47_semantic_slam_api import build_e47_semantic_slam_router
|
|
from k1link.web.environment_api import build_environment_router
|
|
from k1link.web.frontend_assets import ControlStationStaticFiles, frontend_build_id
|
|
from k1link.web.l3_pointpillars_visual_api import (
|
|
build_l3_pointpillars_visual_router,
|
|
)
|
|
from k1link.web.l31_pointpillars_ravnoves_api import (
|
|
build_l31_pointpillars_ravnoves_router,
|
|
)
|
|
from k1link.web.l32_pointpillars_camera_review_api import (
|
|
build_l32_pointpillars_camera_review_router,
|
|
)
|
|
from k1link.web.l33_camera_first_detector_review_api import (
|
|
build_l33_camera_first_detector_review_router,
|
|
)
|
|
from k1link.web.l34_annotation_api import (
|
|
build_l34_annotation_router,
|
|
build_l34d_blind_annotation_router,
|
|
)
|
|
from k1link.web.l34_right_yolox_truth_island_api import (
|
|
build_l34_right_yolox_truth_island_freeze_router,
|
|
)
|
|
from k1link.web.l34a_assisted_yolox_error_audit_api import (
|
|
build_l34a_assisted_yolox_error_audit_router,
|
|
)
|
|
from k1link.web.l34b_nested_box_consolidation_api import (
|
|
build_l34b_nested_box_consolidation_router,
|
|
)
|
|
from k1link.web.l34c_tile_seam_stitch_api import (
|
|
build_l34c_tile_seam_stitch_router,
|
|
)
|
|
from k1link.web.l34d_cumulative_postprocessing_api import (
|
|
build_l34d_cumulative_postprocessing_router,
|
|
)
|
|
from k1link.web.l34e_self_review_diagnostic_api import (
|
|
build_l34e_self_review_diagnostic_router,
|
|
)
|
|
from k1link.web.l34f_adjudication_api import build_l34f_adjudication_router
|
|
from k1link.web.laboratory_api import build_laboratory_router
|
|
from k1link.web.laboratory_report_api import build_laboratory_report_router
|
|
from k1link.web.lidar_api import build_lidar_router
|
|
from k1link.web.lidar_local_surface_service import K1LocalSurfaceReadService
|
|
from k1link.web.m4_threat_replay_api import build_m4_threat_replay_router
|
|
from k1link.web.m48_object_quality_api import build_m48_object_quality_router
|
|
from k1link.web.m48r3_static_occupancy_api import (
|
|
build_m48r3_static_occupancy_router,
|
|
)
|
|
from k1link.web.m48s_fixed_class_detector_lab_api import (
|
|
build_m48s_fixed_class_detector_lab_router,
|
|
)
|
|
from k1link.web.m48t_risk_quality_lab_api import build_m48t_risk_quality_lab_router
|
|
from k1link.web.m49_physical_safety_playback_api import (
|
|
build_m49_physical_safety_playback_router,
|
|
)
|
|
from k1link.web.m49_tgs_fail_closed_api import build_m49_tgs_fail_closed_router
|
|
from k1link.web.m49_tgs_full_shadow_api import build_m49_tgs_full_shadow_router
|
|
from k1link.web.map_api import (
|
|
MapGatewayConfiguration,
|
|
MapGatewayProxy,
|
|
build_map_router,
|
|
)
|
|
from k1link.web.map_view_api import build_map_view_router
|
|
from k1link.web.modular_observatory_api import build_modular_observatory_router
|
|
from k1link.web.observatory_api import build_observatory_router
|
|
from k1link.web.observatory_worker_api import (
|
|
ObservatoryWorkerAuthentication,
|
|
build_observatory_worker_router,
|
|
load_observatory_worker_authentication,
|
|
)
|
|
from k1link.web.plugin_catalog import DevicePluginCatalog, PluginCatalogError
|
|
from k1link.web.plugin_runtime import (
|
|
STATE_READ_ACTION_ID,
|
|
DevicePluginActionRequest,
|
|
DevicePluginDispatcher,
|
|
PluginActionNotFoundError,
|
|
PluginExecutionError,
|
|
PluginNotFoundError,
|
|
PluginRuntimeUnavailableError,
|
|
)
|
|
from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root
|
|
from k1link.web.portable_replay_api import build_portable_replay_router
|
|
from k1link.web.runtime_diagnostics import configure_scanner_diagnostics
|
|
from k1link.web.runtime_readiness import (
|
|
BackgroundReconcilerReadiness,
|
|
build_runtime_readiness,
|
|
)
|
|
from k1link.web.session_api import build_session_router
|
|
from k1link.web.session_overview_api import build_session_overview_router
|
|
from k1link.sessions.overview import SessionOverviewService
|
|
from k1link.missions.sources import PlanningSources
|
|
from k1link.missions.drafts import MissionDrafts
|
|
from k1link.missions.registration_runs import RegistrationRuns
|
|
from k1link.web.mission_registration_api import build_mission_registration_router
|
|
from k1link.web.mission_planner_api import build_mission_planner_router
|
|
from k1link.web.simulation_projects_api import build_simulation_projects_router
|
|
from k1link.web.simulation_world_provider_api import build_simulation_world_provider_router
|
|
from k1link.web.system_telemetry_api import build_system_telemetry_router
|
|
from k1link.web.vegetation_shadow_lab_api import (
|
|
build_vegetation_benchmark_lab_router,
|
|
build_vegetation_shadow_lab_router,
|
|
)
|
|
from k1link.web.viewer_diagnostics_api import build_viewer_diagnostics_router
|
|
|
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
|
|
INVALID_REQUEST_DETAIL = "Некорректные параметры запроса."
|
|
LABORATORY_EVIDENCE_REGISTRY = LaboratoryEvidenceRegistry.from_directory(
|
|
REPOSITORY_ROOT / "config" / "laboratories"
|
|
)
|
|
LABORATORY_EXECUTION_REGISTRY = LaboratoryExecutionRegistry.from_file(
|
|
REPOSITORY_ROOT / "config" / "laboratory-execution.json",
|
|
LABORATORY_EVIDENCE_REGISTRY,
|
|
)
|
|
LABORATORY_RUNNER = LaboratoryRunner(
|
|
registry=LABORATORY_EXECUTION_REGISTRY,
|
|
evidence_registry=LABORATORY_EVIDENCE_REGISTRY,
|
|
sink=JsonlPipelineTelemetrySink(
|
|
REPOSITORY_ROOT / ".runtime" / "telemetry" / "laboratory-runs.jsonl"
|
|
),
|
|
)
|
|
LABORATORY_VALUE_REVIEW_REGISTRY = LaboratoryValueReviewRegistry.from_file(
|
|
REPOSITORY_ROOT / "config" / "laboratory-value-review.json"
|
|
)
|
|
OBSERVATORY_LABORATORY_SETUP_REGISTRY: LaboratorySetupRegistry | None
|
|
OBSERVATORY_LABORATORY_SETUP_REGISTRY_ERROR: str | None
|
|
try:
|
|
OBSERVATORY_LABORATORY_SETUP_REGISTRY = LaboratorySetupRegistry.from_file(
|
|
REPOSITORY_ROOT / "config" / "observatory-laboratory-setups.json",
|
|
repository_root=REPOSITORY_ROOT,
|
|
)
|
|
OBSERVATORY_LABORATORY_SETUP_REGISTRY_ERROR = None
|
|
except (LaboratorySetupRegistryError, OSError) as exc:
|
|
# Observatory is an optional observation-only slice. Its configuration must
|
|
# fail closed locally without preventing K1, Simulation or legacy LAB from
|
|
# starting.
|
|
OBSERVATORY_LABORATORY_SETUP_REGISTRY = None
|
|
OBSERVATORY_LABORATORY_SETUP_REGISTRY_ERROR = str(exc)
|
|
LABORATORY_EVIDENCE_REPORTS = LaboratoryEvidenceReportService(
|
|
LABORATORY_EVIDENCE_REGISTRY,
|
|
lambda: REPOSITORY_ROOT / ".runtime" / "compute-experiments",
|
|
)
|
|
|
|
|
|
def _resolve_media_tool(name: str) -> Path | None:
|
|
"""Resolve media tools under interactive shells and minimal launchd PATHs."""
|
|
|
|
discovered = shutil.which(name)
|
|
candidates = (
|
|
Path(discovered) if discovered is not None else None,
|
|
Path("/opt/homebrew/bin") / name,
|
|
Path("/usr/local/bin") / name,
|
|
Path("/usr/bin") / name,
|
|
)
|
|
for candidate in candidates:
|
|
if candidate is not None and candidate.is_file() and os.access(candidate, os.X_OK):
|
|
return candidate
|
|
return None
|
|
|
|
|
|
plugin_environment = load_installed_device_plugins(REPOSITORY_ROOT)
|
|
plugin_catalog: DevicePluginCatalog = plugin_environment.catalog
|
|
plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher
|
|
session_store = SessionStore(REPOSITORY_ROOT)
|
|
OBSERVATORY_AI_COMPOSITIONS: ModularCompositionStore | None
|
|
OBSERVATORY_AI_COMPOSITION_RUNS: CompositionRunStore | None
|
|
OBSERVATORY_DOMAIN_ONTOLOGY: ObservatoryDomainOntology | None
|
|
OBSERVATORY_LAB_VIEW_PROFILES: LabViewProfileStore | None
|
|
try:
|
|
OBSERVATORY_DOMAIN_ONTOLOGY = ObservatoryDomainOntology.from_file(
|
|
REPOSITORY_ROOT / "config" / "observatory-domain-ontology.json"
|
|
)
|
|
OBSERVATORY_AI_COMPOSITIONS = ModularCompositionStore(
|
|
session_store.data_dir / "observatory-ai-compositions",
|
|
ModuleRegistry.from_file(REPOSITORY_ROOT / "config" / "observatory-ai-modules.json"),
|
|
)
|
|
OBSERVATORY_AI_COMPOSITION_RUNS = CompositionRunStore(
|
|
session_store.data_dir / "observatory-ai-composition-runs"
|
|
)
|
|
OBSERVATORY_LAB_VIEW_PROFILES = LabViewProfileStore(
|
|
session_store.data_dir / "observatory-lab-view-profiles"
|
|
)
|
|
except (
|
|
CompositionError,
|
|
CompositionRunError,
|
|
LabViewProfileError,
|
|
ObservatoryOntologyError,
|
|
OSError,
|
|
ValueError,
|
|
):
|
|
# A modular catalog failure cannot disable recordings, existing LABs or Legacy.
|
|
OBSERVATORY_AI_COMPOSITIONS = None
|
|
OBSERVATORY_AI_COMPOSITION_RUNS = None
|
|
OBSERVATORY_DOMAIN_ONTOLOGY = None
|
|
OBSERVATORY_LAB_VIEW_PROFILES = None
|
|
|
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY: PortableRunDefinitionRegistry | None
|
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR: str | None
|
|
OBSERVATORY_PORTABLE_CALCULATION_PROFILES: PortableCalculationProfileRegistry | None
|
|
OBSERVATORY_PORTABLE_RESULT_VALIDATORS: PortableResultContractValidatorRegistry | None
|
|
try:
|
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY = PortableRunDefinitionRegistry.from_file(
|
|
REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
|
)
|
|
OBSERVATORY_PORTABLE_CALCULATION_PROFILES = portable_calculation_profile_registry(
|
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY
|
|
)
|
|
OBSERVATORY_PORTABLE_RESULT_VALIDATORS = portable_result_validator_registry(
|
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY
|
|
)
|
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR = None
|
|
except (
|
|
PortableRunDefinitionRegistryError,
|
|
PortableWorkerIntegrationError,
|
|
OSError,
|
|
ValueError,
|
|
) as exc:
|
|
# Portable definitions are an optional observation-only slice. Registry
|
|
# drift cannot affect K1, Simulation, legacy LAB, or the exact M49 binding.
|
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY = None
|
|
OBSERVATORY_PORTABLE_CALCULATION_PROFILES = None
|
|
OBSERVATORY_PORTABLE_RESULT_VALIDATORS = None
|
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR = str(exc)
|
|
|
|
|
|
def _resolve_observatory_calculation_profile(
|
|
summary: SessionSummary,
|
|
) -> dict[str, object] | None:
|
|
if OBSERVATORY_LABORATORY_SETUP_REGISTRY is not None:
|
|
legacy = OBSERVATORY_LABORATORY_SETUP_REGISTRY.observatory_calculation_profile(summary)
|
|
if legacy is not None:
|
|
return legacy
|
|
if (
|
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None
|
|
or OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None
|
|
):
|
|
return None
|
|
return resolve_published_portable_calculation_profile(
|
|
summary,
|
|
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
|
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
|
|
)
|
|
|
|
|
|
def _load_optional_observatory_worker_authentication(
|
|
recorded_job_queue: ObservatoryRecordedJobQueue | None,
|
|
*,
|
|
token_path: Path,
|
|
) -> tuple[ObservatoryWorkerAuthentication | None, str | None]:
|
|
"""Load the optional Worker credential without widening app startup risk."""
|
|
|
|
if recorded_job_queue is None:
|
|
return None, "Observatory recorded-job queue is unavailable"
|
|
try:
|
|
return load_observatory_worker_authentication(token_path), None
|
|
except ValueError as exc:
|
|
# Worker pull transport is optional. A missing or unsafe credential
|
|
# disables only this router; K1, Simulation and legacy LAB still start.
|
|
return None, str(exc)
|
|
|
|
|
|
OBSERVATORY_RUN_PREPARATION_LEDGER: ObservatoryRunPreparationLedger | None
|
|
OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR: str | None
|
|
(
|
|
OBSERVATORY_RUN_PREPARATION_LEDGER,
|
|
OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR,
|
|
) = load_observatory_run_preparation_ledger(session_store.data_dir)
|
|
OBSERVATORY_RECORDED_BINDING_SERVICE: M49RecordedQueueBindingService | None
|
|
OBSERVATORY_RECORDED_JOB_QUEUE: ObservatoryRecordedJobQueue | None
|
|
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR: str | None
|
|
try:
|
|
if OBSERVATORY_LABORATORY_SETUP_REGISTRY is None:
|
|
raise M49QueueBindingError("observatory setup registry is unavailable")
|
|
OBSERVATORY_RECORDED_BINDING_SERVICE = M49RecordedQueueBindingService(
|
|
data_dir=session_store.data_dir,
|
|
session_store=session_store,
|
|
setup_registry=OBSERVATORY_LABORATORY_SETUP_REGISTRY,
|
|
config=M49QueueBindingConfig.from_file(
|
|
REPOSITORY_ROOT / "config" / "observatory-m49-recorded-queue-binding.json"
|
|
),
|
|
)
|
|
recorded_definitions = list(OBSERVATORY_RECORDED_BINDING_SERVICE.definitions.definitions)
|
|
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is not None:
|
|
recorded_definitions.extend(
|
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY.ready_recorded_definitions()
|
|
)
|
|
OBSERVATORY_RECORDED_JOB_QUEUE = ObservatoryRecordedJobQueue(
|
|
session_store.data_dir,
|
|
definitions=RecordedRunDefinitionRegistry(tuple(recorded_definitions)),
|
|
)
|
|
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = None
|
|
except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError) as exc:
|
|
# Recorded execution remains an optional observation-only slice. A drifted
|
|
# seal or queue must fail closed without preventing K1, Simulation or legacy
|
|
# LAB from starting.
|
|
OBSERVATORY_RECORDED_BINDING_SERVICE = None
|
|
OBSERVATORY_RECORDED_JOB_QUEUE = None
|
|
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = str(exc)
|
|
OBSERVATORY_WORKER_TOKEN_PATH = session_store.data_dir / "worker-auth" / "observatory-worker.token"
|
|
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED = False
|
|
OBSERVATORY_WORKER_LOCAL_ENABLED: bool
|
|
OBSERVATORY_WORKER_LOCAL_ENABLED_ERROR: str | None
|
|
try:
|
|
OBSERVATORY_WORKER_LOCAL_ENABLED = observatory_worker_local_enabled()
|
|
OBSERVATORY_WORKER_LOCAL_ENABLED_ERROR = None
|
|
except PortableWorkerIntegrationError as exc:
|
|
OBSERVATORY_WORKER_LOCAL_ENABLED = False
|
|
OBSERVATORY_WORKER_LOCAL_ENABLED_ERROR = str(exc)
|
|
OBSERVATORY_WORKER_AUTHENTICATION: ObservatoryWorkerAuthentication | None
|
|
OBSERVATORY_WORKER_AUTHENTICATION_ERROR: str | None
|
|
OBSERVATORY_WORKER_API_ERROR: str | None
|
|
(
|
|
OBSERVATORY_WORKER_AUTHENTICATION,
|
|
OBSERVATORY_WORKER_AUTHENTICATION_ERROR,
|
|
) = _load_optional_observatory_worker_authentication(
|
|
OBSERVATORY_RECORDED_JOB_QUEUE,
|
|
token_path=OBSERVATORY_WORKER_TOKEN_PATH,
|
|
)
|
|
simulation_project_store = SimulationProjectStore(session_store.data_dir)
|
|
simulation_project_service = SimulationProjectService(simulation_project_store)
|
|
session_artifact_gateway = configured_artifact_gateway(session_store.data_dir)
|
|
lidar_local_surface_read_service = K1LocalSurfaceReadService(
|
|
session_store.data_dir / "lidar-read-cache"
|
|
)
|
|
session_recording_materializer = SessionRecordingMaterializer(
|
|
session_store.data_dir,
|
|
exporters=plugin_environment.recording_exporters,
|
|
artifact_gateway=session_artifact_gateway,
|
|
)
|
|
session_overview_service = SessionOverviewService(session_store, plugin_environment.overview_exporters)
|
|
mission_drafts = MissionDrafts(session_store.data_dir / 'missions', PlanningSources(
|
|
session_store, plugin_environment.planning_exporters, plugin_environment.submap_extractors,
|
|
plugin_environment.scene_submap_extractors))
|
|
mission_registration_runs = RegistrationRuns(mission_drafts)
|
|
from k1link.missions.live_tests import PlanningLiveTests
|
|
from k1link.web.planning_live_api import build_planning_live_router
|
|
planning_live_tests = PlanningLiveTests(mission_drafts, plugin_environment.live_planning_sources, mission_registration_runs.lock)
|
|
session_recorded_media_inspector = RecordedMediaInspector(
|
|
session_store.data_dir / "recorded-media-preparations"
|
|
)
|
|
OBSERVATORY_PORTABLE_WORKER_INTEGRATION: PortableObservatoryWorkerIntegration | None
|
|
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR: str | None
|
|
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS: PortableWorkerStorageRoots | None = None
|
|
try:
|
|
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None:
|
|
raise PortableWorkerIntegrationError(
|
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR
|
|
or "portable definition registry is unavailable"
|
|
)
|
|
if OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None:
|
|
raise PortableWorkerIntegrationError("portable calculation profile registry is unavailable")
|
|
if OBSERVATORY_PORTABLE_RESULT_VALIDATORS is None:
|
|
raise PortableWorkerIntegrationError("portable result validator registry is unavailable")
|
|
if OBSERVATORY_RECORDED_JOB_QUEUE is None:
|
|
raise PortableWorkerIntegrationError(
|
|
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR or "Observatory recorded-job queue is unavailable"
|
|
)
|
|
if session_artifact_gateway is None:
|
|
raise PortableWorkerIntegrationError("central artifact store is not configured")
|
|
if session_artifact_gateway.status().central_status != "ready":
|
|
raise PortableWorkerIntegrationError("central artifact store is unavailable")
|
|
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS = PortableWorkerStorageRoots.from_environment(
|
|
artifact_store_root=session_artifact_gateway.store.root,
|
|
)
|
|
OBSERVATORY_PORTABLE_WORKER_INTEGRATION = build_portable_observatory_worker_integration(
|
|
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
|
session_store=session_store,
|
|
media_inspector=session_recorded_media_inspector,
|
|
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
|
artifact_store=session_artifact_gateway.store,
|
|
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
|
|
validators=OBSERVATORY_PORTABLE_RESULT_VALIDATORS,
|
|
source_cas_root=(OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.source_cas_root),
|
|
result_staging_root=(OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.result_staging_root),
|
|
)
|
|
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = None
|
|
except (PortableWorkerIntegrationError, OSError, ValueError) as exc:
|
|
# Constructing this dormant foundation does not enable the Worker router.
|
|
# Failure remains isolated from K1, Simulation and legacy LAB.
|
|
OBSERVATORY_PORTABLE_WORKER_INTEGRATION = None
|
|
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = str(exc)
|
|
OBSERVATORY_PUBLICATION_RECONCILER = (
|
|
None
|
|
if (OBSERVATORY_RECORDED_JOB_QUEUE is None or OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None)
|
|
else PortablePublicationReconciler(
|
|
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
|
artifact_transport=OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport,
|
|
result_publisher=OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher,
|
|
)
|
|
)
|
|
OBSERVATORY_WORKER_API_GATE_ENABLED = OBSERVATORY_WORKER_LOCAL_ENABLED
|
|
OBSERVATORY_WORKER_CLAIM_LEASE_READY = (
|
|
OBSERVATORY_WORKER_API_GATE_ENABLED and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
|
)
|
|
OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = (
|
|
OBSERVATORY_WORKER_API_GATE_ENABLED and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
|
|
)
|
|
OBSERVATORY_WORKER_DISPATCH_READY = (
|
|
OBSERVATORY_WORKER_CLAIM_LEASE_READY
|
|
and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY
|
|
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
|
and OBSERVATORY_WORKER_AUTHENTICATION is not None
|
|
and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
|
|
)
|
|
if OBSERVATORY_WORKER_DISPATCH_READY:
|
|
OBSERVATORY_WORKER_API_ERROR = None
|
|
else:
|
|
worker_api_errors: list[str] = []
|
|
if not OBSERVATORY_WORKER_API_GATE_ENABLED:
|
|
worker_api_errors.append(
|
|
OBSERVATORY_WORKER_LOCAL_ENABLED_ERROR
|
|
or (
|
|
"local-only Worker API gate is disabled; set "
|
|
f"{OBSERVATORY_WORKER_LOCAL_ENABLED_ENV}=1 to enable it"
|
|
)
|
|
)
|
|
if OBSERVATORY_WORKER_AUTHENTICATION_ERROR is not None:
|
|
worker_api_errors.append(
|
|
f"authentication unavailable: {OBSERVATORY_WORKER_AUTHENTICATION_ERROR}"
|
|
)
|
|
if OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR is not None:
|
|
worker_api_errors.append(
|
|
f"integration unavailable: {OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR}"
|
|
)
|
|
OBSERVATORY_WORKER_API_ERROR = "Worker pull API is disabled; " + "; ".join(worker_api_errors)
|
|
OBSERVATORY_PORTABLE_BINDING_SERVICE: PortableRecordedQueueBindingService | None
|
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableSetupProjector | None
|
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
|
|
OBSERVATORY_PORTABLE_RESULT_CACHE: PortableResultCache | None = None
|
|
try:
|
|
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None:
|
|
raise PortableSetupProjectionError(
|
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR
|
|
or "portable definition registry is unavailable"
|
|
)
|
|
if (
|
|
session_artifact_gateway is not None
|
|
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
|
and OBSERVATORY_PORTABLE_CALCULATION_PROFILES is not None
|
|
):
|
|
OBSERVATORY_PORTABLE_RESULT_CACHE = PortableResultCache(
|
|
sessions=session_store,
|
|
artifacts=session_artifact_gateway.store,
|
|
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
|
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
|
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
|
|
)
|
|
OBSERVATORY_PORTABLE_BINDING_SERVICE = PortableRecordedQueueBindingService(
|
|
data_dir=session_store.data_dir,
|
|
session_store=session_store,
|
|
media_inspector=session_recorded_media_inspector,
|
|
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
|
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
|
published_result_available=(
|
|
None
|
|
if OBSERVATORY_PORTABLE_RESULT_CACHE is None
|
|
else OBSERVATORY_PORTABLE_RESULT_CACHE.available
|
|
),
|
|
)
|
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableSetupProjector(
|
|
registry=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
|
capability_probe=OBSERVATORY_PORTABLE_BINDING_SERVICE,
|
|
dispatch_available=OBSERVATORY_WORKER_DISPATCH_READY,
|
|
equipment_capture_registry=session_store.equipment_capture_registry,
|
|
result_cache=OBSERVATORY_PORTABLE_RESULT_CACHE,
|
|
)
|
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None
|
|
except (
|
|
PortableQueueBindingError,
|
|
PortableRunDefinitionRegistryError,
|
|
PortableSetupProjectionError,
|
|
OSError,
|
|
ValueError,
|
|
) as exc:
|
|
# Portable setup execution is an optional observation-only slice. A drifted
|
|
# registry cannot affect K1, Simulation, legacy LAB, or the exact M49 queue.
|
|
OBSERVATORY_PORTABLE_BINDING_SERVICE = None
|
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = None
|
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = str(exc)
|
|
_ffmpeg = _resolve_media_tool("ffmpeg")
|
|
_ffprobe = _resolve_media_tool("ffprobe")
|
|
session_recorded_camera_frame_service = (
|
|
RecordedCameraFrameService(
|
|
session_store,
|
|
session_recorded_media_inspector,
|
|
ffmpeg_path=_ffmpeg,
|
|
cache_root=session_store.data_dir / "camera-frame-cache",
|
|
)
|
|
if _ffmpeg is not None
|
|
else None
|
|
)
|
|
try:
|
|
m48_raw_evidence_reader: M48RawEvidenceReader | None = M48RawEvidenceReader.from_repository(
|
|
repository_root=REPOSITORY_ROOT,
|
|
threat_result_root=(
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "m4"
|
|
/ "replay-threat"
|
|
/ M48_EXPECTED_THREAT_RESULT_ID
|
|
),
|
|
)
|
|
except (M48RawEvidenceError, OSError, ValueError):
|
|
m48_raw_evidence_reader = None
|
|
session_legacy_perception_overlay_store = (
|
|
RecordedPerceptionOverlayStore(
|
|
jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs",
|
|
results_root=REPOSITORY_ROOT / ".runtime" / "compute-results",
|
|
cache_root=session_store.data_dir / "perception-overlays",
|
|
ffmpeg_path=_ffmpeg,
|
|
ffprobe_path=_ffprobe,
|
|
)
|
|
if _ffmpeg is not None and _ffprobe is not None
|
|
else None
|
|
)
|
|
session_calibrated_fusion_store = RecordedCalibratedFusionStore(
|
|
jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs",
|
|
perception_results_root=REPOSITORY_ROOT / ".runtime" / "compute-results",
|
|
fusion_results_root=REPOSITORY_ROOT / ".runtime" / "compute-fusions",
|
|
cache_root=session_store.data_dir / "calibrated-fusion-overlays",
|
|
)
|
|
session_previous_perception_overlay_store = RecordedPerceptionOverlayMux(
|
|
session_calibrated_fusion_store, session_legacy_perception_overlay_store
|
|
)
|
|
session_integrated_perception_store = (
|
|
IntegratedPerceptionOverlayStore(
|
|
jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs",
|
|
results_root=(
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e10" / "worker-results"
|
|
),
|
|
lidar_packs_root=(
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e10" / "lidar-packs"
|
|
),
|
|
cache_root=session_store.data_dir / "integrated-perception-overlays",
|
|
ffmpeg_path=_ffmpeg,
|
|
artifact_gateway=session_artifact_gateway,
|
|
)
|
|
if _ffmpeg is not None
|
|
else None
|
|
)
|
|
session_perception_overlay_store = RecordedPerceptionOverlayMux(
|
|
session_integrated_perception_store or session_previous_perception_overlay_store,
|
|
(
|
|
session_previous_perception_overlay_store
|
|
if session_integrated_perception_store is not None
|
|
else None
|
|
),
|
|
)
|
|
session_perception_epoch_store = (
|
|
RecordedPerceptionEpochStore(
|
|
jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs",
|
|
results_root=REPOSITORY_ROOT / ".runtime" / "compute-results",
|
|
ffprobe_path=_ffprobe,
|
|
)
|
|
if _ffprobe is not None
|
|
else None
|
|
)
|
|
map_gateway_proxy = MapGatewayProxy(MapGatewayConfiguration.from_environment())
|
|
recording_reconciler_readiness = BackgroundReconcilerReadiness()
|
|
|
|
|
|
def _prepare_recorded_media_for_launch(
|
|
command: ReplayCommand,
|
|
_: MaterializedRecording,
|
|
) -> tuple[RecordedMediaManifest, ...]:
|
|
"""Prepare all camera descriptors inside the background job."""
|
|
|
|
return tuple(
|
|
session_recorded_media_inspector.inspect(artifact, command)
|
|
for artifact in session_store.list_recorded_media(command.session_id)
|
|
)
|
|
|
|
|
|
def _restore_recorded_media_for_launch(
|
|
command: ReplayCommand,
|
|
_: MaterializedRecording,
|
|
) -> tuple[RecordedMediaManifest, ...] | None:
|
|
"""Restore only previously published media descriptors."""
|
|
|
|
restored: list[RecordedMediaManifest] = []
|
|
for artifact in session_store.list_recorded_media(command.session_id):
|
|
manifest = session_recorded_media_inspector.restore_prepared(artifact, command)
|
|
if manifest is None:
|
|
return None
|
|
restored.append(manifest)
|
|
return tuple(restored)
|
|
|
|
|
|
session_recording_preparation_manager = SessionRecordingPreparationManager(
|
|
session_recording_materializer,
|
|
ready_preparer=_prepare_recorded_media_for_launch,
|
|
ready_restorer=_restore_recorded_media_for_launch,
|
|
)
|
|
|
|
|
|
def _m48_recorded_camera_playback_source(
|
|
session_id: str,
|
|
) -> RecordedCameraPlaybackSource:
|
|
"""Publish the durable replay package before exposing its manifest URL."""
|
|
|
|
if session_recorded_camera_frame_service is None:
|
|
raise RuntimeError("recorded camera playback is unavailable")
|
|
command = session_store.prepare_replay(session_id, speed=1.0, loop=False)
|
|
snapshot = session_recording_preparation_manager.restore_published(command)
|
|
if snapshot is None or snapshot.state != "ready" or snapshot.recorded_media is None:
|
|
raise RuntimeError("recorded camera playback package is not published")
|
|
return session_recorded_camera_frame_service.playback_source(session_id)
|
|
|
|
|
|
def _canonical_lab_recording_source(session_id: str) -> tuple[Path, str] | None:
|
|
"""Resolve one already-published immutable RRD without starting new work."""
|
|
|
|
snapshot = session_recording_preparation_manager.status(session_id)
|
|
if snapshot is None or snapshot.state != "ready" or snapshot.recording is None:
|
|
return None
|
|
return snapshot.recording.path, snapshot.recording.sha256
|
|
|
|
|
|
def refresh_observation_catalog() -> tuple[str, ...]:
|
|
"""Discover completed or recoverable local evidence without copying payloads."""
|
|
|
|
imported = [
|
|
session_id
|
|
for archive in plugin_environment.observation_archives
|
|
for session_id in session_store.reconcile_archive(archive)
|
|
]
|
|
return tuple(dict.fromkeys(imported))
|
|
|
|
|
|
def finalized_replayable_recording_ids() -> tuple[str, ...]:
|
|
"""List finalized replayable catalog identities without scheduling work."""
|
|
|
|
finalized: list[str] = []
|
|
cursor: str | None = None
|
|
while True:
|
|
page = session_store.list_recent(
|
|
limit=100,
|
|
cursor=cursor,
|
|
include_capability_projections=False,
|
|
)
|
|
finalized.extend(
|
|
summary.session_id
|
|
for summary in page.items
|
|
if summary.replayable and summary.status in {"ready", "interrupted", "failed"}
|
|
)
|
|
cursor = page.next_cursor
|
|
if cursor is None:
|
|
return tuple(finalized)
|
|
|
|
|
|
def enqueue_replayable_recordings(session_ids: Iterable[str]) -> tuple[str, ...]:
|
|
"""Schedule only explicitly selected newly finalized sessions."""
|
|
|
|
enqueued: list[str] = []
|
|
for session_id in dict.fromkeys(session_ids):
|
|
try:
|
|
command = session_store.prepare_replay(session_id)
|
|
session_recording_preparation_manager.enqueue(
|
|
command,
|
|
retry_interrupted=True,
|
|
)
|
|
except RecordingPreparationQueueFull:
|
|
return tuple(enqueued)
|
|
except Exception:
|
|
# One stale/corrupt row must not starve a later newly completed
|
|
# session. Historical cold caches remain operator-triggered.
|
|
continue
|
|
enqueued.append(session_id)
|
|
return tuple(enqueued)
|
|
|
|
|
|
def newly_finalized_recording_ids(
|
|
known_finalized: set[str] | None,
|
|
current_finalized: Iterable[str],
|
|
) -> tuple[str, ...]:
|
|
"""Return only post-startup completions; the first scan is a baseline."""
|
|
|
|
if known_finalized is None:
|
|
return ()
|
|
return tuple(sorted(set(current_finalized) - known_finalized))
|
|
|
|
|
|
def observation_archive_revision(roots: Iterable[Path]) -> tuple[tuple[object, ...], ...]:
|
|
"""Return a cheap change fence for direct-child observation archives.
|
|
|
|
The K1 writer creates and retires ``.current_session`` in the archive root,
|
|
and every session directory is a direct child of that same root. Those
|
|
operations advance the directory metadata, while growing capture/media
|
|
files do not. This lets the reconciler notice session start, completion and
|
|
removal without recursively reopening tens of thousands of immutable
|
|
evidence files every two seconds.
|
|
"""
|
|
|
|
revisions: list[tuple[object, ...]] = []
|
|
for configured_root in roots:
|
|
root = configured_root.expanduser().resolve(strict=False)
|
|
try:
|
|
metadata = root.lstat()
|
|
except OSError:
|
|
revisions.append((str(root), None))
|
|
continue
|
|
revisions.append(
|
|
(
|
|
str(root),
|
|
metadata.st_dev,
|
|
metadata.st_ino,
|
|
metadata.st_mtime_ns,
|
|
metadata.st_ctime_ns,
|
|
)
|
|
)
|
|
revisions.sort(key=lambda item: str(item[0]))
|
|
return tuple(revisions)
|
|
|
|
|
|
async def _recording_preparation_reconciler() -> None:
|
|
"""Prepare sessions finalized during this process, never historical rows."""
|
|
|
|
known_finalized: set[str] | None = None
|
|
reconciled_archive_revision: tuple[tuple[object, ...], ...] | None = None
|
|
while True:
|
|
try:
|
|
archive_revision = await asyncio.to_thread(
|
|
observation_archive_revision,
|
|
(archive.root for archive in plugin_environment.observation_archives),
|
|
)
|
|
if archive_revision != reconciled_archive_revision:
|
|
await asyncio.to_thread(refresh_observation_catalog)
|
|
finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids))
|
|
newly_finalized = newly_finalized_recording_ids(
|
|
known_finalized,
|
|
finalized,
|
|
)
|
|
if newly_finalized:
|
|
await asyncio.to_thread(
|
|
enqueue_replayable_recordings,
|
|
newly_finalized,
|
|
)
|
|
known_finalized = finalized
|
|
# Keep the pre-scan revision. If a writer changed the archive
|
|
# during the expensive discovery, the next two-second check
|
|
# observes the newer revision and performs one follow-up pass.
|
|
reconciled_archive_revision = archive_revision
|
|
recording_reconciler_readiness.record_success()
|
|
except Exception as exc:
|
|
# A transient filesystem/catalog failure must not permanently
|
|
# disable preparation of sessions completed later in the run.
|
|
recording_reconciler_readiness.record_failure(exc)
|
|
await asyncio.sleep(2.0)
|
|
|
|
|
|
async def _portable_result_publication_reconciler() -> None:
|
|
service = OBSERVATORY_PUBLICATION_RECONCILER
|
|
if service is None:
|
|
return
|
|
while True:
|
|
# Durable state remains pending/failed and is retried on the next
|
|
# bounded pass or through the explicit operator action.
|
|
with suppress(OSError, ValueError):
|
|
await asyncio.to_thread(service.run_once)
|
|
await asyncio.sleep(15.0)
|
|
|
|
|
|
async def _recorded_blueprint_resource_reaper() -> None:
|
|
from k1link.viewer.recorded import recorded_blueprint_sessions
|
|
|
|
while True:
|
|
await asyncio.sleep(30.0)
|
|
await asyncio.to_thread(recorded_blueprint_sessions.expire)
|
|
|
|
|
|
@asynccontextmanager
|
|
async def app_lifespan(application: FastAPI) -> AsyncIterator[None]:
|
|
import logging
|
|
import sqlite3
|
|
|
|
from k1link.fleet.registry import FleetRegistry
|
|
|
|
fleet = None
|
|
application.state.fleet = None
|
|
try:
|
|
fleet = FleetRegistry(session_store.data_dir / "fleet")
|
|
fleet.start()
|
|
application.state.fleet = fleet
|
|
except (OSError, ValueError, sqlite3.Error):
|
|
# Fail closed for pairing without taking down unrelated operator work.
|
|
if fleet is not None:
|
|
fleet.close()
|
|
fleet = None
|
|
logging.getLogger(__name__).error("Fleet trust storage unavailable; pairing disabled")
|
|
reconciler: asyncio.Task[None] | None = None
|
|
publication_reconciler: asyncio.Task[None] | None = None
|
|
blueprint_reaper: asyncio.Task[None] | None = None
|
|
try:
|
|
configure_scanner_diagnostics(session_store.data_dir / "logs")
|
|
session_recording_preparation_manager.start()
|
|
simulation_project_service.recover_pending()
|
|
# Recovery is intentionally a one-shot startup phase. The archive
|
|
# helper owns a cross-process lease, while ordinary catalog requests
|
|
# only perform discovery and therefore never touch a live writer.
|
|
for archive in plugin_environment.observation_archives:
|
|
await asyncio.to_thread(archive.recover, archive.root)
|
|
# Full evidence discovery, hashing and RRD queue reconciliation can be
|
|
# expensive on field captures. Start it immediately in the background
|
|
# instead of holding the ASGI startup gate.
|
|
reconciler = asyncio.create_task(_recording_preparation_reconciler())
|
|
publication_reconciler = asyncio.create_task(_portable_result_publication_reconciler())
|
|
blueprint_reaper = asyncio.create_task(_recorded_blueprint_resource_reaper())
|
|
yield
|
|
finally:
|
|
from k1link.viewer.recorded import recorded_blueprint_sessions
|
|
|
|
application.state.fleet = None
|
|
if fleet is not None:
|
|
await asyncio.to_thread(fleet.close)
|
|
if blueprint_reaper is not None:
|
|
blueprint_reaper.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await blueprint_reaper
|
|
await asyncio.to_thread(recorded_blueprint_sessions.close)
|
|
await map_gateway_proxy.close()
|
|
if reconciler is not None:
|
|
reconciler.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await reconciler
|
|
if publication_reconciler is not None:
|
|
publication_reconciler.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await publication_reconciler
|
|
await asyncio.to_thread(session_recording_preparation_manager.close)
|
|
await asyncio.to_thread(session_overview_service.close)
|
|
await asyncio.to_thread(planning_live_tests.close)
|
|
await asyncio.to_thread(mission_registration_runs.close)
|
|
await asyncio.to_thread(lidar_local_surface_read_service.close)
|
|
plugin_environment.close()
|
|
|
|
|
|
app = FastAPI(
|
|
title="NODEDC MISSION CORE API",
|
|
version=__version__,
|
|
docs_url="/api/docs",
|
|
redoc_url=None,
|
|
openapi_url="/api/openapi.json",
|
|
lifespan=app_lifespan,
|
|
)
|
|
app.add_middleware(GZipMiddleware, minimum_size=1_024, compresslevel=5)
|
|
|
|
|
|
app.include_router(fleet_router)
|
|
|
|
|
|
@app.exception_handler(RequestValidationError)
|
|
async def request_validation_error_handler(
|
|
_: Request,
|
|
__: RequestValidationError,
|
|
) -> JSONResponse:
|
|
"""Return validation failures without reflecting request values or credentials."""
|
|
|
|
return JSONResponse(status_code=422, content={"detail": INVALID_REQUEST_DETAIL})
|
|
|
|
|
|
@app.get("/api/liveness")
|
|
async def liveness() -> dict[str, object]:
|
|
"""Prove only that the canonical event loop is responsive."""
|
|
|
|
return {
|
|
"ok": True,
|
|
"status": "alive",
|
|
"service": "mission-core-control-plane",
|
|
}
|
|
|
|
|
|
@app.get("/api/health")
|
|
def health() -> dict[str, Any]:
|
|
return build_runtime_readiness(
|
|
version=__version__,
|
|
plugin_runtime_health=plugin_environment.runtime_health,
|
|
recording_materializer=session_recording_materializer,
|
|
artifact_gateway=session_artifact_gateway,
|
|
map_gateway_configured=map_gateway_proxy.configured,
|
|
ffmpeg_available=_ffmpeg is not None,
|
|
ffprobe_available=_ffprobe is not None,
|
|
reconciler=recording_reconciler_readiness,
|
|
)
|
|
|
|
|
|
@app.get("/api/v1/device-plugins")
|
|
def get_device_plugins() -> dict[str, Any]:
|
|
try:
|
|
return {"items": plugin_catalog.plugin_documents()}
|
|
except PluginCatalogError as exc:
|
|
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
|
|
|
|
|
@app.get("/api/v1/device-models")
|
|
def get_device_models() -> dict[str, Any]:
|
|
try:
|
|
return {"items": plugin_catalog.model_documents()}
|
|
except PluginCatalogError as exc:
|
|
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
|
|
|
|
|
@app.get("/api/v1/device-plugin-runtimes")
|
|
def get_device_plugin_runtimes() -> dict[str, Any]:
|
|
return {"items": list(plugin_environment.runtime_health)}
|
|
|
|
|
|
@app.post("/api/v1/device-plugins/{plugin_id}/actions/{action_id}")
|
|
async def invoke_device_plugin_action(
|
|
plugin_id: str,
|
|
action_id: str,
|
|
request: DevicePluginActionRequest,
|
|
) -> dict[str, Any]:
|
|
try:
|
|
state = await plugin_dispatcher.invoke(plugin_id, action_id, request.input)
|
|
return {"state": state}
|
|
except (PluginNotFoundError, PluginActionNotFoundError) as exc:
|
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
|
except ValidationError as exc:
|
|
raise HTTPException(status_code=422, detail=INVALID_REQUEST_DETAIL) from exc
|
|
except ValueError as exc:
|
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
except PluginExecutionError as exc:
|
|
raise HTTPException(status_code=exc.http_status_code, detail=str(exc)) from exc
|
|
except PluginRuntimeUnavailableError as exc:
|
|
raise HTTPException(status_code=503, detail=str(exc)) from exc
|
|
|
|
|
|
_CLOSED_WEBSOCKET_SEND_ERRORS = frozenset(
|
|
{
|
|
"handler is closed",
|
|
'Cannot call "send" once a close message has been sent.',
|
|
(
|
|
"Unexpected ASGI message 'websocket.send', after sending "
|
|
"'websocket.close' or response already completed."
|
|
),
|
|
"Unexpected ASGI message 'websocket.send', after sending 'websocket.close'.",
|
|
}
|
|
)
|
|
|
|
# This stream carries the comparatively heavy control-state snapshot, not
|
|
# camera or point-cloud media. Action requests return their own result, and the
|
|
# frontend also keeps a four-second REST fallback, so a two-second passive
|
|
# cadence preserves status recovery without starving the live media sockets.
|
|
DEVICE_PLUGIN_EVENT_POLL_INTERVAL_SECONDS = 2.0
|
|
|
|
|
|
def _is_closed_websocket_send_error(exc: RuntimeError) -> bool:
|
|
message = " ".join(str(exc).split())
|
|
if message in _CLOSED_WEBSOCKET_SEND_ERRORS:
|
|
return True
|
|
# uvloop raises this after the browser has already torn down the TCP
|
|
# transport, before Starlette can translate the failed send into a
|
|
# WebSocketDisconnect. Match both stable parts so an unrelated RuntimeError
|
|
# containing only "closed" is never swallowed.
|
|
return message.startswith("unable to perform operation on ") and message.endswith(
|
|
"; the handler is closed"
|
|
)
|
|
|
|
|
|
@app.websocket("/api/v1/device-plugins/{plugin_id}/events")
|
|
async def device_plugin_events(websocket: WebSocket, plugin_id: str) -> None:
|
|
await websocket.accept()
|
|
sequence = 0
|
|
try:
|
|
while True:
|
|
state = await plugin_dispatcher.invoke(plugin_id, STATE_READ_ACTION_ID, {})
|
|
sequence += 1
|
|
try:
|
|
await websocket.send_json(
|
|
{"pluginId": plugin_id, "sequence": sequence, "state": state}
|
|
)
|
|
except WebSocketDisconnect:
|
|
return
|
|
except RuntimeError as exc:
|
|
if _is_closed_websocket_send_error(exc):
|
|
return
|
|
raise
|
|
await asyncio.sleep(DEVICE_PLUGIN_EVENT_POLL_INTERVAL_SECONDS)
|
|
except (PluginNotFoundError, PluginActionNotFoundError):
|
|
await websocket.close(code=1008, reason="Device plugin is not available")
|
|
except (PluginExecutionError, PluginRuntimeUnavailableError):
|
|
await websocket.close(code=1011, reason="Device plugin state stream failed")
|
|
|
|
|
|
if session_artifact_gateway is not None and _ffmpeg is not None:
|
|
app.include_router(
|
|
build_portable_replay_router(
|
|
PortableReplayService(
|
|
view=PortableResultViewService(
|
|
sessions=session_store, artifacts=session_artifact_gateway.store
|
|
),
|
|
data_dir=session_store.data_dir,
|
|
media=session_recorded_media_inspector,
|
|
recording_source=_canonical_lab_recording_source,
|
|
ffmpeg_path=_ffmpeg,
|
|
),
|
|
composition_runs=OBSERVATORY_AI_COMPOSITION_RUNS,
|
|
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
|
)
|
|
)
|
|
|
|
for legacy_router in plugin_environment.legacy_routers:
|
|
app.include_router(legacy_router)
|
|
|
|
app.include_router(build_session_overview_router(session_overview_service))
|
|
app.include_router(build_mission_planner_router(mission_drafts))
|
|
app.include_router(build_planning_live_router(planning_live_tests))
|
|
app.include_router(build_mission_registration_router(mission_registration_runs, planning_live_tests))
|
|
|
|
app.include_router(
|
|
build_session_router(
|
|
session_store,
|
|
# Production discovery belongs to the startup/background reconciler.
|
|
# HTTP list/replay paths must never rescan evidence roots inline.
|
|
catalog_refresher=None,
|
|
recording_materializer=session_recording_materializer,
|
|
recording_preparation_manager=session_recording_preparation_manager,
|
|
media_inspector=session_recorded_media_inspector,
|
|
perception_overlay_provider=session_perception_overlay_store,
|
|
perception_media_provider=session_perception_epoch_store,
|
|
point_color_renderers=plugin_environment.point_color_renderers,
|
|
lab_calculation_profile_resolver=(
|
|
None
|
|
if (
|
|
OBSERVATORY_LABORATORY_SETUP_REGISTRY is None
|
|
and OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None
|
|
)
|
|
else _resolve_observatory_calculation_profile
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_observatory_router(
|
|
session_store,
|
|
setup_registry=OBSERVATORY_LABORATORY_SETUP_REGISTRY,
|
|
setup_registry_error=OBSERVATORY_LABORATORY_SETUP_REGISTRY_ERROR,
|
|
run_preparation_ledger=OBSERVATORY_RUN_PREPARATION_LEDGER,
|
|
run_preparation_ledger_error=OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR,
|
|
recorded_binding_service=OBSERVATORY_RECORDED_BINDING_SERVICE,
|
|
recorded_job_queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
|
recorded_job_queue_error=OBSERVATORY_RECORDED_JOB_QUEUE_ERROR,
|
|
portable_setup_projector=OBSERVATORY_PORTABLE_SETUP_PROJECTOR,
|
|
portable_setup_projector_error=OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR,
|
|
portable_binding_service=OBSERVATORY_PORTABLE_BINDING_SERVICE,
|
|
portable_result_view=(
|
|
None
|
|
if session_artifact_gateway is None
|
|
else PortableResultViewService(
|
|
sessions=session_store,
|
|
artifacts=session_artifact_gateway.store,
|
|
)
|
|
),
|
|
portable_artifact_transport=(
|
|
None
|
|
if OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None
|
|
else OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport
|
|
),
|
|
portable_result_publisher=(
|
|
None
|
|
if OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None
|
|
else OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher
|
|
),
|
|
)
|
|
)
|
|
if (
|
|
OBSERVATORY_AI_COMPOSITIONS is not None
|
|
and OBSERVATORY_AI_COMPOSITION_RUNS is not None
|
|
and OBSERVATORY_DOMAIN_ONTOLOGY is not None
|
|
):
|
|
app.include_router(
|
|
build_modular_observatory_router(
|
|
store=session_store,
|
|
compositions=OBSERVATORY_AI_COMPOSITIONS,
|
|
composition_runs=OBSERVATORY_AI_COMPOSITION_RUNS,
|
|
ontology=OBSERVATORY_DOMAIN_ONTOLOGY,
|
|
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
|
binding=OBSERVATORY_PORTABLE_BINDING_SERVICE,
|
|
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
|
view_profiles=OBSERVATORY_LAB_VIEW_PROFILES,
|
|
)
|
|
)
|
|
if OBSERVATORY_WORKER_DISPATCH_READY:
|
|
assert OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
|
assert OBSERVATORY_WORKER_AUTHENTICATION is not None
|
|
assert OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
|
|
app.include_router(
|
|
build_observatory_worker_router(
|
|
OBSERVATORY_RECORDED_JOB_QUEUE,
|
|
authentication=OBSERVATORY_WORKER_AUTHENTICATION,
|
|
artifact_transport=(OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport),
|
|
result_publisher=(OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_environment_router(root_provider=lambda: session_store.data_dir / "ui-environment")
|
|
)
|
|
app.include_router(build_map_router(map_gateway_proxy))
|
|
app.include_router(
|
|
build_map_view_router(
|
|
root_provider=lambda: session_store.data_dir / "map-view",
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_polygon_router(
|
|
root_provider=lambda: (
|
|
configured_polygon_runs_root() or REPOSITORY_ROOT / ".runtime" / "polygon-runs"
|
|
)
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_lidar_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "lidar-replay-v2" / "packs"
|
|
),
|
|
ground_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "lidar-ground-v1" / "benchmarks"
|
|
),
|
|
field_review_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "lidar-field-review-v1"
|
|
/ "reviews"
|
|
),
|
|
local_surface_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "k1-local-surface-v1" / "models"
|
|
),
|
|
e10_source_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e10" / "lidar-packs"
|
|
),
|
|
dataset_admission_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "admission.json"
|
|
),
|
|
dataset_preview_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "preview.json"
|
|
),
|
|
dataset_rellis_preview_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-preview.json"
|
|
),
|
|
dataset_rellis_admission_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-admission.json"
|
|
),
|
|
dataset_ground_preview_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "ground-comparison.json"
|
|
),
|
|
local_surface_read_service=lidar_local_surface_read_service,
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_laboratory_router(
|
|
e29_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e29" / "results"
|
|
),
|
|
local_surface_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "k1-local-surface-v1" / "models"
|
|
),
|
|
source_pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e10" / "lidar-packs"
|
|
),
|
|
source_result_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e10" / "worker-results"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_laboratory_report_router(
|
|
LABORATORY_VALUE_REVIEW_REGISTRY,
|
|
LABORATORY_EVIDENCE_REPORTS,
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_advanced_laboratory_router(
|
|
evidence_registry=LABORATORY_EVIDENCE_REGISTRY,
|
|
evidence_runtime_root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "compute-experiments",
|
|
e31_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e31" / "source-qualifications"
|
|
),
|
|
e32_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e32" / "results"
|
|
),
|
|
e33_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e33" / "results"
|
|
),
|
|
e34_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e34" / "results"
|
|
),
|
|
e35_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e35" / "results"
|
|
),
|
|
e37_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e37" / "results"
|
|
),
|
|
e38_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e38" / "results"
|
|
),
|
|
e39_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e39" / "results"
|
|
),
|
|
e40_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e40" / "results"
|
|
),
|
|
l3_visual_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "visual-audits"
|
|
),
|
|
l31_ravnoves_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "pointpillars-ravnoves"
|
|
),
|
|
l32_camera_review_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "pointpillars-camera-review"
|
|
),
|
|
l33_camera_first_review_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "camera-first-detector-review"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_l3_pointpillars_visual_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "visual-audits"
|
|
)
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_l31_pointpillars_ravnoves_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "pointpillars-ravnoves"
|
|
)
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_l32_pointpillars_camera_review_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "pointpillars-camera-review"
|
|
)
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_l33_camera_first_detector_review_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "camera-first-detector-review"
|
|
)
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_l34_right_yolox_truth_island_freeze_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "right-yolox-truth-island-freeze"
|
|
),
|
|
truth_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results"
|
|
),
|
|
evaluation_pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e46_blind_review_router(
|
|
truth_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results"
|
|
),
|
|
evaluation_pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs"
|
|
),
|
|
annotation_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "laboratory-annotations" / "e46-blind"
|
|
),
|
|
submission_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e48" / "review-submissions"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e46a_ai_engineering_preannotation_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "e46a"
|
|
/ "ai-engineering-preannotations"
|
|
),
|
|
annotation_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "laboratory-annotations" / "e46a-correction"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e46b_temporal_motion_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46b" / "temporal-motion"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e46c_full_replay_world_tracks_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "e46c"
|
|
/ "full-replay-world-tracks"
|
|
),
|
|
e26_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e10" / "worker-results"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e46d_temporal_failure_audit_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "e46d"
|
|
/ "temporal-failure-audits"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_m4_threat_replay_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m4" / "replay-threat"
|
|
),
|
|
repository_root_provider=lambda: REPOSITORY_ROOT,
|
|
camera_frame_provider=(
|
|
session_recorded_camera_frame_service.extract
|
|
if session_recorded_camera_frame_service is not None
|
|
else None
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_m48_object_quality_router(
|
|
pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48" / "object-quality-packs"
|
|
),
|
|
workflow_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "laboratory-annotations" / "m48-object-quality"
|
|
),
|
|
truth_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48" / "object-truth-seals"
|
|
),
|
|
result_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48" / "object-quality-results"
|
|
),
|
|
small_static_result_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "m48"
|
|
/ "small-static-passage-regression-results"
|
|
),
|
|
static_occupancy_result_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "m48"
|
|
/ "static-occupancy-qualification-results"
|
|
),
|
|
camera_frame_provider=(
|
|
session_recorded_camera_frame_service.extract
|
|
if session_recorded_camera_frame_service is not None
|
|
else None
|
|
),
|
|
camera_playback_provider=(
|
|
_m48_recorded_camera_playback_source
|
|
if session_recorded_camera_frame_service is not None
|
|
else None
|
|
),
|
|
spatial_evidence_provider=m48_raw_evidence_reader,
|
|
evaluation_runner=LABORATORY_RUNNER,
|
|
evaluation_receipt_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "laboratory-run-receipts"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_m48r3_static_occupancy_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "m48"
|
|
/ "static-occupancy-shadow-results"
|
|
),
|
|
repository_root_provider=lambda: REPOSITORY_ROOT,
|
|
camera_frame_provider=(
|
|
session_recorded_camera_frame_service.extract
|
|
if session_recorded_camera_frame_service is not None
|
|
else None
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_m49_tgs_fail_closed_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m49" / "tgs-fail-closed-results"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_m49_tgs_full_shadow_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m49" / "tgs-full-shadow-results"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_vegetation_shadow_lab_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "lab-v1-vegetation" / "results"
|
|
),
|
|
canonical_recording_provider=_canonical_lab_recording_source,
|
|
camera_frame_provider=(
|
|
session_recorded_camera_frame_service.extract
|
|
if session_recorded_camera_frame_service is not None
|
|
else None
|
|
),
|
|
jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs",
|
|
rerun_overlay_cache_root=(session_store.data_dir / "laboratory-rerun-overlays"),
|
|
ffmpeg_path=_ffmpeg,
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_vegetation_benchmark_lab_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "lab-v1-vegetation-benchmark"
|
|
/ "results"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_m49_physical_safety_playback_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "m49"
|
|
/ "physical-safety-playback-results"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_m48s_fixed_class_detector_lab_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "m48s-semantic-shadow"
|
|
/ "fixed-class-detector-lab-results"
|
|
),
|
|
repository_root_provider=lambda: REPOSITORY_ROOT,
|
|
camera_frame_provider=(
|
|
session_recorded_camera_frame_service.extract
|
|
if session_recorded_camera_frame_service is not None
|
|
else None
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_m48t_risk_quality_lab_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "m48t-risk-quality"
|
|
/ "lab-results"
|
|
),
|
|
native_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "m48t-risk-quality"
|
|
/ "native-lab-results"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e47_semantic_slam_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e47" / "semantic-slam-results"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e46e_ready_stack_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46e" / "ready-stack-results"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e46f_dashcam_bakeoff_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "e46f"
|
|
/ "dashcam-bakeoff-results"
|
|
),
|
|
e46e_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46e" / "ready-stack-results"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e46g_rectified_detector_bakeoff_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46g" / "results"
|
|
)
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e46h_full_rectified_front_replay_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46h" / "results"
|
|
)
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e46i_grounding_dino_full_replay_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46i" / "results"
|
|
)
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e46j_raw_fisheye_realtime_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46j" / "results"
|
|
)
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_l34_annotation_router(
|
|
root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "right-yolox-truth-island-freeze"
|
|
),
|
|
truth_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results"
|
|
),
|
|
evaluation_pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs"
|
|
),
|
|
annotation_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "laboratory-annotations" / "l34"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_l34a_assisted_yolox_error_audit_router(
|
|
audit_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "assisted-yolox-error-audits"
|
|
),
|
|
l34_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "right-yolox-truth-island-freeze"
|
|
),
|
|
truth_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results"
|
|
),
|
|
evaluation_pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_l34b_nested_box_consolidation_router(
|
|
shadow_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "nested-box-consolidation-shadows"
|
|
),
|
|
l34_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "right-yolox-truth-island-freeze"
|
|
),
|
|
truth_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results"
|
|
),
|
|
evaluation_pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_l34c_tile_seam_stitch_router(
|
|
shadow_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "tile-seam-stitch-shadows"
|
|
),
|
|
l34_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "right-yolox-truth-island-freeze"
|
|
),
|
|
truth_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results"
|
|
),
|
|
evaluation_pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_l34d_cumulative_postprocessing_router(
|
|
candidate_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "cumulative-postprocessing-candidates"
|
|
),
|
|
l34_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "right-yolox-truth-island-freeze"
|
|
),
|
|
truth_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results"
|
|
),
|
|
evaluation_pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_l34d_blind_annotation_router(
|
|
candidate_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "cumulative-postprocessing-candidates"
|
|
),
|
|
l34_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "right-yolox-truth-island-freeze"
|
|
),
|
|
truth_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results"
|
|
),
|
|
evaluation_pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs"
|
|
),
|
|
annotation_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "laboratory-annotations" / "l34d-blind"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_l34e_self_review_diagnostic_router(
|
|
diagnostic_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "self-review-diagnostics"
|
|
),
|
|
candidate_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "cumulative-postprocessing-candidates"
|
|
),
|
|
l34_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "right-yolox-truth-island-freeze"
|
|
),
|
|
truth_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results"
|
|
),
|
|
evaluation_pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_l34f_adjudication_router(
|
|
diagnostic_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "self-review-diagnostics"
|
|
),
|
|
candidate_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "cumulative-postprocessing-candidates"
|
|
),
|
|
l34_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "l3"
|
|
/ "right-yolox-truth-island-freeze"
|
|
),
|
|
truth_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results"
|
|
),
|
|
evaluation_pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs"
|
|
),
|
|
adjudication_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "laboratory-annotations" / "l34f-adjudication"
|
|
),
|
|
result_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "adjudicated-references"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e30_review_router(
|
|
materialization_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "materializations"
|
|
),
|
|
review_pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "review-packs"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e40_case_review_router(
|
|
e40_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e40" / "results"
|
|
),
|
|
operator_review_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e40" / "operator-reviews"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e30_engineering_router(
|
|
generation_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "engineering-generations"
|
|
),
|
|
materialization_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "materializations"
|
|
),
|
|
review_pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "review-packs"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_e30_human_review_router(
|
|
materialization_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "materializations"
|
|
),
|
|
review_pack_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "review-packs"
|
|
),
|
|
engineering_generation_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "engineering-generations"
|
|
),
|
|
draft_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "human-review-drafts"
|
|
),
|
|
generation_root_provider=lambda: (
|
|
REPOSITORY_ROOT
|
|
/ ".runtime"
|
|
/ "compute-experiments"
|
|
/ "e30"
|
|
/ "human-review-generations"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_system_telemetry_router(
|
|
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_artifact_health_router(
|
|
runtime_root_provider=lambda: REPOSITORY_ROOT / ".runtime",
|
|
e44_results_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e44" / "results"
|
|
),
|
|
e50_results_root_provider=lambda: (
|
|
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e50" / "results"
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
build_compute_contour_router(
|
|
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
|
|
)
|
|
)
|
|
app.include_router(build_simulation_world_provider_router())
|
|
app.include_router(
|
|
build_simulation_projects_router(
|
|
store=simulation_project_store,
|
|
service=simulation_project_service,
|
|
)
|
|
)
|
|
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
|
|
app.include_router(
|
|
build_viewer_diagnostics_router(
|
|
expected_ui_build_id=lambda: frontend_build_id(frontend_dist),
|
|
)
|
|
)
|
|
|
|
|
|
if frontend_dist.is_dir():
|
|
app.mount(
|
|
"/",
|
|
ControlStationStaticFiles(directory=frontend_dist, html=True),
|
|
name="frontend",
|
|
)
|