from __future__ import annotations import asyncio import os import shutil from collections.abc import AsyncIterator, Iterable from contextlib import asynccontextmanager, suppress from pathlib import Path from typing import Any from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse from pydantic import ValidationError from k1link import __version__ from k1link.artifact_gateway import configured_artifact_gateway from k1link.compute import ( IntegratedPerceptionOverlayStore, RecordedCalibratedFusionStore, RecordedPerceptionEpochStore, RecordedPerceptionOverlayMux, RecordedPerceptionOverlayStore, ) from k1link.compute.pipeline_telemetry import JsonlPipelineTelemetrySink from k1link.laboratory import ( LaboratoryEvidenceRegistry, LaboratoryEvidenceReportService, LaboratoryExecutionRegistry, LaboratoryRunner, LaboratoryValueReviewRegistry, ) from k1link.laboratory.m48_raw_evidence import ( M48_EXPECTED_THREAT_RESULT_ID, M48RawEvidenceError, M48RawEvidenceReader, ) from k1link.observatory import ( LaboratorySetupRegistry, LaboratorySetupRegistryError, ObservatoryRunPreparationLedger, load_observatory_run_preparation_ledger, ) from k1link.observatory.composition_runs import CompositionRunError, CompositionRunStore from k1link.observatory.domain_ontology import ( ObservatoryDomainOntology, ObservatoryOntologyError, ) from k1link.observatory.lab_view_profiles import LabViewProfileError, LabViewProfileStore from k1link.observatory.m49_queue_binding import ( M49QueueBindingConfig, M49QueueBindingError, M49RecordedQueueBindingService, ) from k1link.observatory.modular_composition import CompositionError, ModuleRegistry from k1link.observatory.modular_composition_store import ModularCompositionStore from k1link.observatory.portable_publication_reconciler import ( PortablePublicationReconciler, ) from k1link.observatory.portable_queue_binding import ( PortableQueueBindingError, PortableRecordedQueueBindingService, ) from k1link.observatory.portable_replay import PortableReplayService from k1link.observatory.portable_result_cache import PortableResultCache from k1link.observatory.portable_result_contract import ( PortableCalculationProfileRegistry, PortableResultContractValidatorRegistry, ) from k1link.observatory.portable_result_publisher import ( resolve_published_portable_calculation_profile, ) from k1link.observatory.portable_result_view import PortableResultViewService from k1link.observatory.portable_run_definitions import ( PortableRunDefinitionRegistry, PortableRunDefinitionRegistryError, ) from k1link.observatory.portable_setup_projection import ( PortableSetupProjectionError, PortableSetupProjector, portable_calculation_profile_registry, ) from k1link.observatory.portable_worker_integration import ( OBSERVATORY_WORKER_LOCAL_ENABLED_ENV, PortableObservatoryWorkerIntegration, PortableWorkerIntegrationError, PortableWorkerStorageRoots, build_portable_observatory_worker_integration, observatory_worker_local_enabled, portable_result_validator_registry, ) from k1link.observatory.recorded_jobs import ( ObservatoryRecordedJobQueue, ObservatoryRecordedQueueError, RecordedRunDefinitionRegistry, ) from k1link.sessions import ( MaterializedRecording, RecordedCameraFrameService, RecordedCameraPlaybackSource, RecordedMediaInspector, RecordedMediaManifest, RecordingPreparationQueueFull, ReplayCommand, SessionRecordingMaterializer, SessionRecordingPreparationManager, SessionStore, ) from k1link.sessions.models import SessionSummary from k1link.simulation.projects import SimulationProjectService, SimulationProjectStore from k1link.web.advanced_laboratory_api import build_advanced_laboratory_router from k1link.web.artifact_health_api import build_artifact_health_router from k1link.web.compute_contour_api import build_compute_contour_router from k1link.web.fleet_api import router as fleet_router from k1link.web.device_plugin_composition import load_installed_device_plugins from k1link.web.e30_engineering_api import build_e30_engineering_router from k1link.web.e30_human_review_api import build_e30_human_review_router from k1link.web.e30_review_api import build_e30_review_router from k1link.web.e40_case_review_api import build_e40_case_review_router from k1link.web.e46_blind_review_api import build_e46_blind_review_router from k1link.web.e46a_ai_engineering_preannotation_api import ( build_e46a_ai_engineering_preannotation_router, ) from k1link.web.e46b_temporal_motion_api import build_e46b_temporal_motion_router from k1link.web.e46c_full_replay_world_tracks_api import ( build_e46c_full_replay_world_tracks_router, ) from k1link.web.e46d_temporal_failure_audit_api import ( build_e46d_temporal_failure_audit_router, ) from k1link.web.e46e_ready_stack_api import build_e46e_ready_stack_router from k1link.web.e46f_dashcam_bakeoff_api import build_e46f_dashcam_bakeoff_router from k1link.web.e46g_rectified_detector_bakeoff_api import ( build_e46g_rectified_detector_bakeoff_router, ) from k1link.web.e46h_full_rectified_front_replay_api import ( build_e46h_full_rectified_front_replay_router, ) from k1link.web.e46i_grounding_dino_full_replay_api import ( build_e46i_grounding_dino_full_replay_router, ) from k1link.web.e46j_raw_fisheye_realtime_api import ( build_e46j_raw_fisheye_realtime_router, ) from k1link.web.e47_semantic_slam_api import build_e47_semantic_slam_router from k1link.web.environment_api import build_environment_router from k1link.web.frontend_assets import ControlStationStaticFiles, frontend_build_id from k1link.web.response_compression import ResponseCompressionMiddleware from k1link.web.l3_pointpillars_visual_api import ( build_l3_pointpillars_visual_router, ) from k1link.web.l31_pointpillars_ravnoves_api import ( build_l31_pointpillars_ravnoves_router, ) from k1link.web.l32_pointpillars_camera_review_api import ( build_l32_pointpillars_camera_review_router, ) from k1link.web.l33_camera_first_detector_review_api import ( build_l33_camera_first_detector_review_router, ) from k1link.web.l34_annotation_api import ( build_l34_annotation_router, build_l34d_blind_annotation_router, ) from k1link.web.l34_right_yolox_truth_island_api import ( build_l34_right_yolox_truth_island_freeze_router, ) from k1link.web.l34a_assisted_yolox_error_audit_api import ( build_l34a_assisted_yolox_error_audit_router, ) from k1link.web.l34b_nested_box_consolidation_api import ( build_l34b_nested_box_consolidation_router, ) from k1link.web.l34c_tile_seam_stitch_api import ( build_l34c_tile_seam_stitch_router, ) from k1link.web.l34d_cumulative_postprocessing_api import ( build_l34d_cumulative_postprocessing_router, ) from k1link.web.l34e_self_review_diagnostic_api import ( build_l34e_self_review_diagnostic_router, ) from k1link.web.l34f_adjudication_api import build_l34f_adjudication_router from k1link.web.laboratory_api import build_laboratory_router from k1link.web.laboratory_report_api import build_laboratory_report_router from k1link.web.lidar_api import build_lidar_router from k1link.web.lidar_local_surface_service import K1LocalSurfaceReadService from k1link.web.m4_threat_replay_api import build_m4_threat_replay_router from k1link.web.m48_object_quality_api import build_m48_object_quality_router from k1link.web.m48r3_static_occupancy_api import ( build_m48r3_static_occupancy_router, ) from k1link.web.m48s_fixed_class_detector_lab_api import ( build_m48s_fixed_class_detector_lab_router, ) from k1link.web.m48t_risk_quality_lab_api import build_m48t_risk_quality_lab_router from k1link.web.m49_physical_safety_playback_api import ( build_m49_physical_safety_playback_router, ) from k1link.web.m49_tgs_fail_closed_api import build_m49_tgs_fail_closed_router from k1link.web.m49_tgs_full_shadow_api import build_m49_tgs_full_shadow_router from k1link.web.map_api import ( MapGatewayConfiguration, MapGatewayProxy, build_map_router, ) from k1link.web.map_view_api import build_map_view_router from k1link.web.modular_observatory_api import build_modular_observatory_router from k1link.web.observatory_api import build_observatory_router from k1link.web.observatory_worker_api import ( ObservatoryWorkerAuthentication, build_observatory_worker_router, load_observatory_worker_authentication, ) from k1link.web.plugin_catalog import DevicePluginCatalog, PluginCatalogError from k1link.web.plugin_runtime import ( STATE_READ_ACTION_ID, DevicePluginActionRequest, DevicePluginDispatcher, PluginActionNotFoundError, PluginExecutionError, PluginNotFoundError, PluginRuntimeUnavailableError, ) from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root from k1link.web.portable_replay_api import build_portable_replay_router from k1link.web.runtime_diagnostics import configure_scanner_diagnostics from k1link.web.runtime_readiness import ( BackgroundReconcilerReadiness, build_runtime_readiness, ) from k1link.web.session_api import build_session_router from k1link.web.session_overview_api import build_session_overview_router from k1link.sessions.overview import SessionOverviewService from k1link.missions.sources import PlanningSources from k1link.missions.drafts import MissionDrafts from k1link.missions.registration_runs import RegistrationRuns from k1link.web.mission_registration_api import build_mission_registration_router from k1link.web.mission_planner_api import build_mission_planner_router from k1link.web.simulation_projects_api import build_simulation_projects_router from k1link.web.simulation_world_provider_api import build_simulation_world_provider_router from k1link.web.system_telemetry_api import build_system_telemetry_router from k1link.web.vegetation_shadow_lab_api import ( build_vegetation_benchmark_lab_router, build_vegetation_shadow_lab_router, ) from k1link.web.viewer_diagnostics_api import build_viewer_diagnostics_router REPOSITORY_ROOT = Path(__file__).resolve().parents[3] INVALID_REQUEST_DETAIL = "Некорректные параметры запроса." LABORATORY_EVIDENCE_REGISTRY = LaboratoryEvidenceRegistry.from_directory( REPOSITORY_ROOT / "config" / "laboratories" ) LABORATORY_EXECUTION_REGISTRY = LaboratoryExecutionRegistry.from_file( REPOSITORY_ROOT / "config" / "laboratory-execution.json", LABORATORY_EVIDENCE_REGISTRY, ) LABORATORY_RUNNER = LaboratoryRunner( registry=LABORATORY_EXECUTION_REGISTRY, evidence_registry=LABORATORY_EVIDENCE_REGISTRY, sink=JsonlPipelineTelemetrySink( REPOSITORY_ROOT / ".runtime" / "telemetry" / "laboratory-runs.jsonl" ), ) LABORATORY_VALUE_REVIEW_REGISTRY = LaboratoryValueReviewRegistry.from_file( REPOSITORY_ROOT / "config" / "laboratory-value-review.json" ) OBSERVATORY_LABORATORY_SETUP_REGISTRY: LaboratorySetupRegistry | None OBSERVATORY_LABORATORY_SETUP_REGISTRY_ERROR: str | None try: OBSERVATORY_LABORATORY_SETUP_REGISTRY = LaboratorySetupRegistry.from_file( REPOSITORY_ROOT / "config" / "observatory-laboratory-setups.json", repository_root=REPOSITORY_ROOT, ) OBSERVATORY_LABORATORY_SETUP_REGISTRY_ERROR = None except (LaboratorySetupRegistryError, OSError) as exc: # Observatory is an optional observation-only slice. Its configuration must # fail closed locally without preventing K1, Simulation or legacy LAB from # starting. OBSERVATORY_LABORATORY_SETUP_REGISTRY = None OBSERVATORY_LABORATORY_SETUP_REGISTRY_ERROR = str(exc) LABORATORY_EVIDENCE_REPORTS = LaboratoryEvidenceReportService( LABORATORY_EVIDENCE_REGISTRY, lambda: REPOSITORY_ROOT / ".runtime" / "compute-experiments", ) def _resolve_media_tool(name: str) -> Path | None: """Resolve media tools under interactive shells and minimal launchd PATHs.""" discovered = shutil.which(name) candidates = ( Path(discovered) if discovered is not None else None, Path("/opt/homebrew/bin") / name, Path("/usr/local/bin") / name, Path("/usr/bin") / name, ) for candidate in candidates: if candidate is not None and candidate.is_file() and os.access(candidate, os.X_OK): return candidate return None plugin_environment = load_installed_device_plugins(REPOSITORY_ROOT) plugin_catalog: DevicePluginCatalog = plugin_environment.catalog plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher session_store = SessionStore(REPOSITORY_ROOT) OBSERVATORY_AI_COMPOSITIONS: ModularCompositionStore | None OBSERVATORY_AI_COMPOSITION_RUNS: CompositionRunStore | None OBSERVATORY_DOMAIN_ONTOLOGY: ObservatoryDomainOntology | None OBSERVATORY_LAB_VIEW_PROFILES: LabViewProfileStore | None try: OBSERVATORY_DOMAIN_ONTOLOGY = ObservatoryDomainOntology.from_file( REPOSITORY_ROOT / "config" / "observatory-domain-ontology.json" ) OBSERVATORY_AI_COMPOSITIONS = ModularCompositionStore( session_store.data_dir / "observatory-ai-compositions", ModuleRegistry.from_file(REPOSITORY_ROOT / "config" / "observatory-ai-modules.json"), ) OBSERVATORY_AI_COMPOSITION_RUNS = CompositionRunStore( session_store.data_dir / "observatory-ai-composition-runs" ) OBSERVATORY_LAB_VIEW_PROFILES = LabViewProfileStore( session_store.data_dir / "observatory-lab-view-profiles" ) except ( CompositionError, CompositionRunError, LabViewProfileError, ObservatoryOntologyError, OSError, ValueError, ): # A modular catalog failure cannot disable recordings, existing LABs or Legacy. OBSERVATORY_AI_COMPOSITIONS = None OBSERVATORY_AI_COMPOSITION_RUNS = None OBSERVATORY_DOMAIN_ONTOLOGY = None OBSERVATORY_LAB_VIEW_PROFILES = None OBSERVATORY_PORTABLE_DEFINITION_REGISTRY: PortableRunDefinitionRegistry | None OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR: str | None OBSERVATORY_PORTABLE_CALCULATION_PROFILES: PortableCalculationProfileRegistry | None OBSERVATORY_PORTABLE_RESULT_VALIDATORS: PortableResultContractValidatorRegistry | None try: OBSERVATORY_PORTABLE_DEFINITION_REGISTRY = PortableRunDefinitionRegistry.from_file( REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" ) OBSERVATORY_PORTABLE_CALCULATION_PROFILES = portable_calculation_profile_registry( OBSERVATORY_PORTABLE_DEFINITION_REGISTRY ) OBSERVATORY_PORTABLE_RESULT_VALIDATORS = portable_result_validator_registry( OBSERVATORY_PORTABLE_DEFINITION_REGISTRY ) OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR = None except ( PortableRunDefinitionRegistryError, PortableWorkerIntegrationError, OSError, ValueError, ) as exc: # Portable definitions are an optional observation-only slice. Registry # drift cannot affect K1, Simulation, legacy LAB, or the exact M49 binding. OBSERVATORY_PORTABLE_DEFINITION_REGISTRY = None OBSERVATORY_PORTABLE_CALCULATION_PROFILES = None OBSERVATORY_PORTABLE_RESULT_VALIDATORS = None OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR = str(exc) def _resolve_observatory_calculation_profile( summary: SessionSummary, ) -> dict[str, object] | None: if OBSERVATORY_LABORATORY_SETUP_REGISTRY is not None: legacy = OBSERVATORY_LABORATORY_SETUP_REGISTRY.observatory_calculation_profile(summary) if legacy is not None: return legacy if ( OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None or OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None ): return None return resolve_published_portable_calculation_profile( summary, definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY, calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES, ) def _load_optional_observatory_worker_authentication( recorded_job_queue: ObservatoryRecordedJobQueue | None, *, token_path: Path, ) -> tuple[ObservatoryWorkerAuthentication | None, str | None]: """Load the optional Worker credential without widening app startup risk.""" if recorded_job_queue is None: return None, "Observatory recorded-job queue is unavailable" try: return load_observatory_worker_authentication(token_path), None except ValueError as exc: # Worker pull transport is optional. A missing or unsafe credential # disables only this router; K1, Simulation and legacy LAB still start. return None, str(exc) OBSERVATORY_RUN_PREPARATION_LEDGER: ObservatoryRunPreparationLedger | None OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR: str | None ( OBSERVATORY_RUN_PREPARATION_LEDGER, OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR, ) = load_observatory_run_preparation_ledger(session_store.data_dir) OBSERVATORY_RECORDED_BINDING_SERVICE: M49RecordedQueueBindingService | None OBSERVATORY_RECORDED_JOB_QUEUE: ObservatoryRecordedJobQueue | None OBSERVATORY_RECORDED_JOB_QUEUE_ERROR: str | None try: if OBSERVATORY_LABORATORY_SETUP_REGISTRY is None: raise M49QueueBindingError("observatory setup registry is unavailable") OBSERVATORY_RECORDED_BINDING_SERVICE = M49RecordedQueueBindingService( data_dir=session_store.data_dir, session_store=session_store, setup_registry=OBSERVATORY_LABORATORY_SETUP_REGISTRY, config=M49QueueBindingConfig.from_file( REPOSITORY_ROOT / "config" / "observatory-m49-recorded-queue-binding.json" ), ) recorded_definitions = list(OBSERVATORY_RECORDED_BINDING_SERVICE.definitions.definitions) if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is not None: recorded_definitions.extend( OBSERVATORY_PORTABLE_DEFINITION_REGISTRY.ready_recorded_definitions() ) OBSERVATORY_RECORDED_JOB_QUEUE = ObservatoryRecordedJobQueue( session_store.data_dir, definitions=RecordedRunDefinitionRegistry(tuple(recorded_definitions)), ) OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = None except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError) as exc: # Recorded execution remains an optional observation-only slice. A drifted # seal or queue must fail closed without preventing K1, Simulation or legacy # LAB from starting. OBSERVATORY_RECORDED_BINDING_SERVICE = None OBSERVATORY_RECORDED_JOB_QUEUE = None OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = str(exc) OBSERVATORY_WORKER_TOKEN_PATH = session_store.data_dir / "worker-auth" / "observatory-worker.token" OBSERVATORY_WORKER_PRODUCTION_API_ENABLED = False OBSERVATORY_WORKER_LOCAL_ENABLED: bool OBSERVATORY_WORKER_LOCAL_ENABLED_ERROR: str | None try: OBSERVATORY_WORKER_LOCAL_ENABLED = observatory_worker_local_enabled() OBSERVATORY_WORKER_LOCAL_ENABLED_ERROR = None except PortableWorkerIntegrationError as exc: OBSERVATORY_WORKER_LOCAL_ENABLED = False OBSERVATORY_WORKER_LOCAL_ENABLED_ERROR = str(exc) OBSERVATORY_WORKER_AUTHENTICATION: ObservatoryWorkerAuthentication | None OBSERVATORY_WORKER_AUTHENTICATION_ERROR: str | None OBSERVATORY_WORKER_API_ERROR: str | None ( OBSERVATORY_WORKER_AUTHENTICATION, OBSERVATORY_WORKER_AUTHENTICATION_ERROR, ) = _load_optional_observatory_worker_authentication( OBSERVATORY_RECORDED_JOB_QUEUE, token_path=OBSERVATORY_WORKER_TOKEN_PATH, ) simulation_project_store = SimulationProjectStore(session_store.data_dir) simulation_project_service = SimulationProjectService(simulation_project_store) session_artifact_gateway = configured_artifact_gateway(session_store.data_dir) lidar_local_surface_read_service = K1LocalSurfaceReadService( session_store.data_dir / "lidar-read-cache" ) session_recording_materializer = SessionRecordingMaterializer( session_store.data_dir, exporters=plugin_environment.recording_exporters, artifact_gateway=session_artifact_gateway, ) from k1link.reconstruction.session_versions import SessionMapVersions from k1link.sessions.map_recording import MapReplaySessionStore from k1link.missions.default_sources import DefaultPlanningSources session_map_versions = SessionMapVersions(session_store.data_dir) operator_session_store = MapReplaySessionStore(session_store, session_map_versions) session_overview_service = SessionOverviewService(session_store, plugin_environment.overview_exporters, map_versions=session_map_versions) original_planning_sources = PlanningSources( session_store, plugin_environment.planning_exporters, plugin_environment.submap_extractors, plugin_environment.scene_submap_extractors) mission_drafts = MissionDrafts(session_store.data_dir / 'missions', DefaultPlanningSources(original_planning_sources, session_map_versions)) mission_registration_runs = RegistrationRuns(mission_drafts) from k1link.missions.live_tests import PlanningLiveTests from k1link.web.planning_live_api import build_planning_live_router planning_live_tests = PlanningLiveTests(mission_drafts, plugin_environment.live_planning_sources, mission_registration_runs.lock, capture_recorder=session_store.record_planning_capture) session_recorded_media_inspector = RecordedMediaInspector( session_store.data_dir / "recorded-media-preparations" ) OBSERVATORY_PORTABLE_WORKER_INTEGRATION: PortableObservatoryWorkerIntegration | None OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR: str | None OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS: PortableWorkerStorageRoots | None = None try: if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None: raise PortableWorkerIntegrationError( OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR or "portable definition registry is unavailable" ) if OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None: raise PortableWorkerIntegrationError("portable calculation profile registry is unavailable") if OBSERVATORY_PORTABLE_RESULT_VALIDATORS is None: raise PortableWorkerIntegrationError("portable result validator registry is unavailable") if OBSERVATORY_RECORDED_JOB_QUEUE is None: raise PortableWorkerIntegrationError( OBSERVATORY_RECORDED_JOB_QUEUE_ERROR or "Observatory recorded-job queue is unavailable" ) if session_artifact_gateway is None: raise PortableWorkerIntegrationError("central artifact store is not configured") if session_artifact_gateway.status().central_status != "ready": raise PortableWorkerIntegrationError("central artifact store is unavailable") OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS = PortableWorkerStorageRoots.from_environment( artifact_store_root=session_artifact_gateway.store.root, ) OBSERVATORY_PORTABLE_WORKER_INTEGRATION = build_portable_observatory_worker_integration( queue=OBSERVATORY_RECORDED_JOB_QUEUE, session_store=session_store, media_inspector=session_recorded_media_inspector, definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY, artifact_store=session_artifact_gateway.store, calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES, validators=OBSERVATORY_PORTABLE_RESULT_VALIDATORS, source_cas_root=(OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.source_cas_root), result_staging_root=(OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.result_staging_root), ) OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = None except (PortableWorkerIntegrationError, OSError, ValueError) as exc: # Constructing this dormant foundation does not enable the Worker router. # Failure remains isolated from K1, Simulation and legacy LAB. OBSERVATORY_PORTABLE_WORKER_INTEGRATION = None OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = str(exc) OBSERVATORY_PUBLICATION_RECONCILER = ( None if (OBSERVATORY_RECORDED_JOB_QUEUE is None or OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None) else PortablePublicationReconciler( queue=OBSERVATORY_RECORDED_JOB_QUEUE, artifact_transport=OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport, result_publisher=OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher, ) ) OBSERVATORY_WORKER_API_GATE_ENABLED = OBSERVATORY_WORKER_LOCAL_ENABLED OBSERVATORY_WORKER_CLAIM_LEASE_READY = ( OBSERVATORY_WORKER_API_GATE_ENABLED and OBSERVATORY_RECORDED_JOB_QUEUE is not None ) OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = ( OBSERVATORY_WORKER_API_GATE_ENABLED and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None ) OBSERVATORY_WORKER_DISPATCH_READY = ( OBSERVATORY_WORKER_CLAIM_LEASE_READY and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY and OBSERVATORY_RECORDED_JOB_QUEUE is not None and OBSERVATORY_WORKER_AUTHENTICATION is not None and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None ) if OBSERVATORY_WORKER_DISPATCH_READY: OBSERVATORY_WORKER_API_ERROR = None else: worker_api_errors: list[str] = [] if not OBSERVATORY_WORKER_API_GATE_ENABLED: worker_api_errors.append( OBSERVATORY_WORKER_LOCAL_ENABLED_ERROR or ( "local-only Worker API gate is disabled; set " f"{OBSERVATORY_WORKER_LOCAL_ENABLED_ENV}=1 to enable it" ) ) if OBSERVATORY_WORKER_AUTHENTICATION_ERROR is not None: worker_api_errors.append( f"authentication unavailable: {OBSERVATORY_WORKER_AUTHENTICATION_ERROR}" ) if OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR is not None: worker_api_errors.append( f"integration unavailable: {OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR}" ) OBSERVATORY_WORKER_API_ERROR = "Worker pull API is disabled; " + "; ".join(worker_api_errors) OBSERVATORY_PORTABLE_BINDING_SERVICE: PortableRecordedQueueBindingService | None OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableSetupProjector | None OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None OBSERVATORY_PORTABLE_RESULT_CACHE: PortableResultCache | None = None try: if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None: raise PortableSetupProjectionError( OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR or "portable definition registry is unavailable" ) if ( session_artifact_gateway is not None and OBSERVATORY_RECORDED_JOB_QUEUE is not None and OBSERVATORY_PORTABLE_CALCULATION_PROFILES is not None ): OBSERVATORY_PORTABLE_RESULT_CACHE = PortableResultCache( sessions=session_store, artifacts=session_artifact_gateway.store, queue=OBSERVATORY_RECORDED_JOB_QUEUE, definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY, calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES, ) OBSERVATORY_PORTABLE_BINDING_SERVICE = PortableRecordedQueueBindingService( data_dir=session_store.data_dir, session_store=session_store, media_inspector=session_recorded_media_inspector, definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY, queue=OBSERVATORY_RECORDED_JOB_QUEUE, published_result_available=( None if OBSERVATORY_PORTABLE_RESULT_CACHE is None else OBSERVATORY_PORTABLE_RESULT_CACHE.available ), ) OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableSetupProjector( registry=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY, capability_probe=OBSERVATORY_PORTABLE_BINDING_SERVICE, dispatch_available=OBSERVATORY_WORKER_DISPATCH_READY, equipment_capture_registry=session_store.equipment_capture_registry, result_cache=OBSERVATORY_PORTABLE_RESULT_CACHE, ) OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None except ( PortableQueueBindingError, PortableRunDefinitionRegistryError, PortableSetupProjectionError, OSError, ValueError, ) as exc: # Portable setup execution is an optional observation-only slice. A drifted # registry cannot affect K1, Simulation, legacy LAB, or the exact M49 queue. OBSERVATORY_PORTABLE_BINDING_SERVICE = None OBSERVATORY_PORTABLE_SETUP_PROJECTOR = None OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = str(exc) _ffmpeg = _resolve_media_tool("ffmpeg") _ffprobe = _resolve_media_tool("ffprobe") session_recorded_camera_frame_service = ( RecordedCameraFrameService( session_store, session_recorded_media_inspector, ffmpeg_path=_ffmpeg, cache_root=session_store.data_dir / "camera-frame-cache", ) if _ffmpeg is not None else None ) try: m48_raw_evidence_reader: M48RawEvidenceReader | None = M48RawEvidenceReader.from_repository( repository_root=REPOSITORY_ROOT, threat_result_root=( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m4" / "replay-threat" / M48_EXPECTED_THREAT_RESULT_ID ), ) except (M48RawEvidenceError, OSError, ValueError): m48_raw_evidence_reader = None session_legacy_perception_overlay_store = ( RecordedPerceptionOverlayStore( jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs", results_root=REPOSITORY_ROOT / ".runtime" / "compute-results", cache_root=session_store.data_dir / "perception-overlays", ffmpeg_path=_ffmpeg, ffprobe_path=_ffprobe, ) if _ffmpeg is not None and _ffprobe is not None else None ) session_calibrated_fusion_store = RecordedCalibratedFusionStore( jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs", perception_results_root=REPOSITORY_ROOT / ".runtime" / "compute-results", fusion_results_root=REPOSITORY_ROOT / ".runtime" / "compute-fusions", cache_root=session_store.data_dir / "calibrated-fusion-overlays", ) session_previous_perception_overlay_store = RecordedPerceptionOverlayMux( session_calibrated_fusion_store, session_legacy_perception_overlay_store ) session_integrated_perception_store = ( IntegratedPerceptionOverlayStore( jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs", results_root=( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e10" / "worker-results" ), lidar_packs_root=( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e10" / "lidar-packs" ), cache_root=session_store.data_dir / "integrated-perception-overlays", ffmpeg_path=_ffmpeg, artifact_gateway=session_artifact_gateway, ) if _ffmpeg is not None else None ) session_perception_overlay_store = RecordedPerceptionOverlayMux( session_integrated_perception_store or session_previous_perception_overlay_store, ( session_previous_perception_overlay_store if session_integrated_perception_store is not None else None ), ) session_perception_epoch_store = ( RecordedPerceptionEpochStore( jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs", results_root=REPOSITORY_ROOT / ".runtime" / "compute-results", ffprobe_path=_ffprobe, ) if _ffprobe is not None else None ) map_gateway_proxy = MapGatewayProxy(MapGatewayConfiguration.from_environment()) recording_reconciler_readiness = BackgroundReconcilerReadiness() def _prepare_recorded_media_for_launch( command: ReplayCommand, _: MaterializedRecording, ) -> tuple[RecordedMediaManifest, ...]: """Prepare all camera descriptors inside the background job.""" return tuple( session_recorded_media_inspector.inspect(artifact, command) for artifact in session_store.list_recorded_media(command.session_id) ) def _restore_recorded_media_for_launch( command: ReplayCommand, _: MaterializedRecording, ) -> tuple[RecordedMediaManifest, ...] | None: """Restore only previously published media descriptors.""" restored: list[RecordedMediaManifest] = [] for artifact in session_store.list_recorded_media(command.session_id): manifest = session_recorded_media_inspector.restore_prepared(artifact, command) if manifest is None: return None restored.append(manifest) return tuple(restored) session_recording_preparation_manager = SessionRecordingPreparationManager( session_recording_materializer, ready_preparer=_prepare_recorded_media_for_launch, ready_restorer=_restore_recorded_media_for_launch, ) def _m48_recorded_camera_playback_source( session_id: str, ) -> RecordedCameraPlaybackSource: """Publish the durable replay package before exposing its manifest URL.""" if session_recorded_camera_frame_service is None: raise RuntimeError("recorded camera playback is unavailable") command = operator_session_store.prepare_replay(session_id, speed=1.0, loop=False) snapshot = session_recording_preparation_manager.restore_published(command) if snapshot is None or snapshot.state != "ready" or snapshot.recorded_media is None: raise RuntimeError("recorded camera playback package is not published") return session_recorded_camera_frame_service.playback_source(session_id) def _canonical_lab_recording_source(session_id: str) -> tuple[Path, str] | None: """Resolve one already-published immutable RRD without starting new work.""" snapshot = session_recording_preparation_manager.status(session_id) if snapshot is None or snapshot.state != "ready" or snapshot.recording is None: return None if snapshot.command.map_version is not None: return None # Original-frame LAB overlays cannot consume corrected operator geometry. return snapshot.recording.path, snapshot.recording.sha256 def refresh_observation_catalog() -> tuple[str, ...]: """Discover completed or recoverable local evidence without copying payloads.""" imported = [ session_id for archive in plugin_environment.observation_archives for session_id in session_store.reconcile_archive(archive) ] return tuple(dict.fromkeys(imported)) def finalized_replayable_recording_ids() -> tuple[str, ...]: """List finalized replayable catalog identities without scheduling work.""" finalized: list[str] = [] cursor: str | None = None while True: page = session_store.list_recent( limit=100, cursor=cursor, include_capability_projections=False, ) finalized.extend( summary.session_id for summary in page.items if summary.replayable and summary.status in {"ready", "interrupted", "failed"} ) cursor = page.next_cursor if cursor is None: return tuple(finalized) def enqueue_replayable_recordings(session_ids: Iterable[str]) -> tuple[str, ...]: """Schedule only explicitly selected newly finalized sessions.""" enqueued: list[str] = [] for session_id in dict.fromkeys(session_ids): try: command = operator_session_store.prepare_replay(session_id) session_recording_preparation_manager.enqueue( command, retry_interrupted=True, ) except RecordingPreparationQueueFull: return tuple(enqueued) except Exception: # One stale/corrupt row must not starve a later newly completed # session. Historical cold caches remain operator-triggered. continue enqueued.append(session_id) return tuple(enqueued) def newly_finalized_recording_ids( known_finalized: set[str] | None, current_finalized: Iterable[str], ) -> tuple[str, ...]: """Return only post-startup completions; the first scan is a baseline.""" if known_finalized is None: return () return tuple(sorted(set(current_finalized) - known_finalized)) def observation_archive_revision(roots: Iterable[Path]) -> tuple[tuple[object, ...], ...]: """Return a cheap change fence for direct-child observation archives. The K1 writer creates and retires ``.current_session`` in the archive root, and every session directory is a direct child of that same root. Those operations advance the directory metadata, while growing capture/media files do not. This lets the reconciler notice session start, completion and removal without recursively reopening tens of thousands of immutable evidence files every two seconds. """ revisions: list[tuple[object, ...]] = [] for configured_root in roots: root = configured_root.expanduser().resolve(strict=False) try: metadata = root.lstat() except OSError: revisions.append((str(root), None)) continue revisions.append( ( str(root), metadata.st_dev, metadata.st_ino, metadata.st_mtime_ns, metadata.st_ctime_ns, ) ) revisions.sort(key=lambda item: str(item[0])) return tuple(revisions) async def _recording_preparation_reconciler() -> None: """Prepare sessions finalized during this process, never historical rows.""" known_finalized: set[str] | None = None reconciled_archive_revision: tuple[tuple[object, ...], ...] | None = None while True: try: archive_revision = await asyncio.to_thread( observation_archive_revision, (archive.root for archive in plugin_environment.observation_archives), ) if archive_revision != reconciled_archive_revision: await asyncio.to_thread(refresh_observation_catalog) finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids)) newly_finalized = newly_finalized_recording_ids( known_finalized, finalized, ) if newly_finalized: await asyncio.to_thread( enqueue_replayable_recordings, newly_finalized, ) known_finalized = finalized # Keep the pre-scan revision. If a writer changed the archive # during the expensive discovery, the next two-second check # observes the newer revision and performs one follow-up pass. reconciled_archive_revision = archive_revision recording_reconciler_readiness.record_success() except Exception as exc: # A transient filesystem/catalog failure must not permanently # disable preparation of sessions completed later in the run. recording_reconciler_readiness.record_failure(exc) await asyncio.sleep(2.0) async def _portable_result_publication_reconciler() -> None: service = OBSERVATORY_PUBLICATION_RECONCILER if service is None: return while True: # Durable state remains pending/failed and is retried on the next # bounded pass or through the explicit operator action. with suppress(OSError, ValueError): await asyncio.to_thread(service.run_once) await asyncio.sleep(15.0) async def _recorded_blueprint_resource_reaper() -> None: from k1link.viewer.recorded import recorded_blueprint_sessions while True: await asyncio.sleep(30.0) await asyncio.to_thread(recorded_blueprint_sessions.expire) @asynccontextmanager async def app_lifespan(application: FastAPI) -> AsyncIterator[None]: import logging import sqlite3 from k1link.fleet.registry import FleetRegistry fleet = None application.state.fleet = None try: fleet = FleetRegistry(session_store.data_dir / "fleet") fleet.start() application.state.fleet = fleet except (OSError, ValueError, sqlite3.Error): # Fail closed for pairing without taking down unrelated operator work. if fleet is not None: fleet.close() fleet = None logging.getLogger(__name__).error("Fleet trust storage unavailable; pairing disabled") reconciler: asyncio.Task[None] | None = None publication_reconciler: asyncio.Task[None] | None = None blueprint_reaper: asyncio.Task[None] | None = None try: configure_scanner_diagnostics(session_store.data_dir / "logs") session_recording_preparation_manager.start() simulation_project_service.recover_pending() # Recovery is intentionally a one-shot startup phase. The archive # helper owns a cross-process lease, while ordinary catalog requests # only perform discovery and therefore never touch a live writer. for archive in plugin_environment.observation_archives: await asyncio.to_thread(archive.recover, archive.root) # Full evidence discovery, hashing and RRD queue reconciliation can be # expensive on field captures. Start it immediately in the background # instead of holding the ASGI startup gate. reconciler = asyncio.create_task(_recording_preparation_reconciler()) publication_reconciler = asyncio.create_task(_portable_result_publication_reconciler()) blueprint_reaper = asyncio.create_task(_recorded_blueprint_resource_reaper()) yield finally: from k1link.viewer.recorded import recorded_blueprint_sessions application.state.fleet = None if fleet is not None: await asyncio.to_thread(fleet.close) if blueprint_reaper is not None: blueprint_reaper.cancel() with suppress(asyncio.CancelledError): await blueprint_reaper await asyncio.to_thread(recorded_blueprint_sessions.close) await map_gateway_proxy.close() if reconciler is not None: reconciler.cancel() with suppress(asyncio.CancelledError): await reconciler if publication_reconciler is not None: publication_reconciler.cancel() with suppress(asyncio.CancelledError): await publication_reconciler await asyncio.to_thread(session_recording_preparation_manager.close) await asyncio.to_thread(session_overview_service.close) await asyncio.to_thread(planning_live_tests.close) await asyncio.to_thread(mission_registration_runs.close) await asyncio.to_thread(lidar_local_surface_read_service.close) plugin_environment.close() app = FastAPI( title="NODEDC MISSION CORE API", version=__version__, docs_url="/api/docs", redoc_url=None, openapi_url="/api/openapi.json", lifespan=app_lifespan, ) app.add_middleware(ResponseCompressionMiddleware, minimum_size=1_024, compresslevel=5) app.include_router(fleet_router) @app.exception_handler(RequestValidationError) async def request_validation_error_handler( _: Request, __: RequestValidationError, ) -> JSONResponse: """Return validation failures without reflecting request values or credentials.""" return JSONResponse(status_code=422, content={"detail": INVALID_REQUEST_DETAIL}) @app.get("/api/liveness") async def liveness() -> dict[str, object]: """Prove only that the canonical event loop is responsive.""" return { "ok": True, "status": "alive", "service": "mission-core-control-plane", } @app.get("/api/health") def health() -> dict[str, Any]: return build_runtime_readiness( version=__version__, plugin_runtime_health=plugin_environment.runtime_health, recording_materializer=session_recording_materializer, artifact_gateway=session_artifact_gateway, map_gateway_configured=map_gateway_proxy.configured, ffmpeg_available=_ffmpeg is not None, ffprobe_available=_ffprobe is not None, reconciler=recording_reconciler_readiness, ) @app.get("/api/v1/device-plugins") def get_device_plugins() -> dict[str, Any]: try: return {"items": plugin_catalog.plugin_documents()} except PluginCatalogError as exc: raise HTTPException(status_code=500, detail=str(exc)) from exc @app.get("/api/v1/device-models") def get_device_models() -> dict[str, Any]: try: return {"items": plugin_catalog.model_documents()} except PluginCatalogError as exc: raise HTTPException(status_code=500, detail=str(exc)) from exc @app.get("/api/v1/device-plugin-runtimes") def get_device_plugin_runtimes() -> dict[str, Any]: return {"items": list(plugin_environment.runtime_health)} @app.post("/api/v1/device-plugins/{plugin_id}/actions/{action_id}") async def invoke_device_plugin_action( plugin_id: str, action_id: str, request: DevicePluginActionRequest, ) -> dict[str, Any]: try: state = await plugin_dispatcher.invoke(plugin_id, action_id, request.input) return {"state": state} except (PluginNotFoundError, PluginActionNotFoundError) as exc: raise HTTPException(status_code=404, detail=str(exc)) from exc except ValidationError as exc: raise HTTPException(status_code=422, detail=INVALID_REQUEST_DETAIL) from exc except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc except PluginExecutionError as exc: raise HTTPException(status_code=exc.http_status_code, detail=str(exc)) from exc except PluginRuntimeUnavailableError as exc: raise HTTPException(status_code=503, detail=str(exc)) from exc _CLOSED_WEBSOCKET_SEND_ERRORS = frozenset( { "handler is closed", 'Cannot call "send" once a close message has been sent.', ( "Unexpected ASGI message 'websocket.send', after sending " "'websocket.close' or response already completed." ), "Unexpected ASGI message 'websocket.send', after sending 'websocket.close'.", } ) # This stream carries the comparatively heavy control-state snapshot, not # camera or point-cloud media. Action requests return their own result, and the # frontend also keeps a four-second REST fallback, so a two-second passive # cadence preserves status recovery without starving the live media sockets. DEVICE_PLUGIN_EVENT_POLL_INTERVAL_SECONDS = 2.0 def _is_closed_websocket_send_error(exc: RuntimeError) -> bool: message = " ".join(str(exc).split()) if message in _CLOSED_WEBSOCKET_SEND_ERRORS: return True # uvloop raises this after the browser has already torn down the TCP # transport, before Starlette can translate the failed send into a # WebSocketDisconnect. Match both stable parts so an unrelated RuntimeError # containing only "closed" is never swallowed. return message.startswith("unable to perform operation on ") and message.endswith( "; the handler is closed" ) @app.websocket("/api/v1/device-plugins/{plugin_id}/events") async def device_plugin_events(websocket: WebSocket, plugin_id: str) -> None: await websocket.accept() sequence = 0 try: while True: state = await plugin_dispatcher.invoke(plugin_id, STATE_READ_ACTION_ID, {}) sequence += 1 try: await websocket.send_json( {"pluginId": plugin_id, "sequence": sequence, "state": state} ) except WebSocketDisconnect: return except RuntimeError as exc: if _is_closed_websocket_send_error(exc): return raise await asyncio.sleep(DEVICE_PLUGIN_EVENT_POLL_INTERVAL_SECONDS) except (PluginNotFoundError, PluginActionNotFoundError): await websocket.close(code=1008, reason="Device plugin is not available") except (PluginExecutionError, PluginRuntimeUnavailableError): await websocket.close(code=1011, reason="Device plugin state stream failed") if session_artifact_gateway is not None and _ffmpeg is not None: app.include_router( build_portable_replay_router( PortableReplayService( view=PortableResultViewService( sessions=session_store, artifacts=session_artifact_gateway.store ), data_dir=session_store.data_dir, media=session_recorded_media_inspector, recording_source=_canonical_lab_recording_source, ffmpeg_path=_ffmpeg, ), composition_runs=OBSERVATORY_AI_COMPOSITION_RUNS, queue=OBSERVATORY_RECORDED_JOB_QUEUE, ) ) for legacy_router in plugin_environment.legacy_routers: app.include_router(legacy_router) app.include_router(build_session_overview_router(session_overview_service)) app.include_router(build_mission_planner_router(mission_drafts)) app.include_router(build_planning_live_router(planning_live_tests)) app.include_router(build_mission_registration_router(mission_registration_runs, planning_live_tests)) app.include_router( build_session_router( operator_session_store, # Production discovery belongs to the startup/background reconciler. # HTTP list/replay paths must never rescan evidence roots inline. catalog_refresher=None, recording_materializer=session_recording_materializer, recording_preparation_manager=session_recording_preparation_manager, media_inspector=session_recorded_media_inspector, perception_overlay_provider=session_perception_overlay_store, perception_media_provider=session_perception_epoch_store, point_color_renderers=plugin_environment.point_color_renderers, point_display_renderers=plugin_environment.point_display_renderers, lab_calculation_profile_resolver=( None if ( OBSERVATORY_LABORATORY_SETUP_REGISTRY is None and OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None ) else _resolve_observatory_calculation_profile ), ) ) app.include_router( build_observatory_router( session_store, setup_registry=OBSERVATORY_LABORATORY_SETUP_REGISTRY, setup_registry_error=OBSERVATORY_LABORATORY_SETUP_REGISTRY_ERROR, run_preparation_ledger=OBSERVATORY_RUN_PREPARATION_LEDGER, run_preparation_ledger_error=OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR, recorded_binding_service=OBSERVATORY_RECORDED_BINDING_SERVICE, recorded_job_queue=OBSERVATORY_RECORDED_JOB_QUEUE, recorded_job_queue_error=OBSERVATORY_RECORDED_JOB_QUEUE_ERROR, portable_setup_projector=OBSERVATORY_PORTABLE_SETUP_PROJECTOR, portable_setup_projector_error=OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR, portable_binding_service=OBSERVATORY_PORTABLE_BINDING_SERVICE, portable_result_view=( None if session_artifact_gateway is None else PortableResultViewService( sessions=session_store, artifacts=session_artifact_gateway.store, ) ), portable_artifact_transport=( None if OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None else OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport ), portable_result_publisher=( None if OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None else OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher ), ) ) if ( OBSERVATORY_AI_COMPOSITIONS is not None and OBSERVATORY_AI_COMPOSITION_RUNS is not None and OBSERVATORY_DOMAIN_ONTOLOGY is not None ): app.include_router( build_modular_observatory_router( store=session_store, compositions=OBSERVATORY_AI_COMPOSITIONS, composition_runs=OBSERVATORY_AI_COMPOSITION_RUNS, ontology=OBSERVATORY_DOMAIN_ONTOLOGY, definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY, binding=OBSERVATORY_PORTABLE_BINDING_SERVICE, queue=OBSERVATORY_RECORDED_JOB_QUEUE, view_profiles=OBSERVATORY_LAB_VIEW_PROFILES, ) ) if OBSERVATORY_WORKER_DISPATCH_READY: assert OBSERVATORY_RECORDED_JOB_QUEUE is not None assert OBSERVATORY_WORKER_AUTHENTICATION is not None assert OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None app.include_router( build_observatory_worker_router( OBSERVATORY_RECORDED_JOB_QUEUE, authentication=OBSERVATORY_WORKER_AUTHENTICATION, artifact_transport=(OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport), result_publisher=(OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher), ) ) app.include_router( build_environment_router(root_provider=lambda: session_store.data_dir / "ui-environment") ) app.include_router(build_map_router(map_gateway_proxy)) app.include_router( build_map_view_router( root_provider=lambda: session_store.data_dir / "map-view", ) ) app.include_router( build_polygon_router( root_provider=lambda: ( configured_polygon_runs_root() or REPOSITORY_ROOT / ".runtime" / "polygon-runs" ) ) ) app.include_router( build_lidar_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "lidar-replay-v2" / "packs" ), ground_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "lidar-ground-v1" / "benchmarks" ), field_review_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "lidar-field-review-v1" / "reviews" ), local_surface_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "k1-local-surface-v1" / "models" ), e10_source_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e10" / "lidar-packs" ), dataset_admission_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "admission.json" ), dataset_preview_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "preview.json" ), dataset_rellis_preview_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-preview.json" ), dataset_rellis_admission_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-admission.json" ), dataset_ground_preview_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "ground-comparison.json" ), local_surface_read_service=lidar_local_surface_read_service, ) ) app.include_router( build_laboratory_router( e29_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e29" / "results" ), local_surface_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "k1-local-surface-v1" / "models" ), source_pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e10" / "lidar-packs" ), source_result_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e10" / "worker-results" ), ) ) app.include_router( build_laboratory_report_router( LABORATORY_VALUE_REVIEW_REGISTRY, LABORATORY_EVIDENCE_REPORTS, ) ) app.include_router( build_advanced_laboratory_router( evidence_registry=LABORATORY_EVIDENCE_REGISTRY, evidence_runtime_root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "compute-experiments", e31_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e31" / "source-qualifications" ), e32_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e32" / "results" ), e33_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e33" / "results" ), e34_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e34" / "results" ), e35_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e35" / "results" ), e37_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e37" / "results" ), e38_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e38" / "results" ), e39_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e39" / "results" ), e40_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e40" / "results" ), l3_visual_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "visual-audits" ), l31_ravnoves_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "pointpillars-ravnoves" ), l32_camera_review_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "pointpillars-camera-review" ), l33_camera_first_review_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "camera-first-detector-review" ), ) ) app.include_router( build_l3_pointpillars_visual_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "visual-audits" ) ) ) app.include_router( build_l31_pointpillars_ravnoves_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "pointpillars-ravnoves" ) ) ) app.include_router( build_l32_pointpillars_camera_review_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "pointpillars-camera-review" ) ) ) app.include_router( build_l33_camera_first_detector_review_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "camera-first-detector-review" ) ) ) app.include_router( build_l34_right_yolox_truth_island_freeze_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "right-yolox-truth-island-freeze" ), truth_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results" ), evaluation_pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs" ), ) ) app.include_router( build_e46_blind_review_router( truth_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results" ), evaluation_pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs" ), annotation_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "laboratory-annotations" / "e46-blind" ), submission_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e48" / "review-submissions" ), ) ) app.include_router( build_e46a_ai_engineering_preannotation_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46a" / "ai-engineering-preannotations" ), annotation_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "laboratory-annotations" / "e46a-correction" ), ) ) app.include_router( build_e46b_temporal_motion_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46b" / "temporal-motion" ), ) ) app.include_router( build_e46c_full_replay_world_tracks_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46c" / "full-replay-world-tracks" ), e26_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e10" / "worker-results" ), ) ) app.include_router( build_e46d_temporal_failure_audit_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46d" / "temporal-failure-audits" ), ) ) app.include_router( build_m4_threat_replay_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m4" / "replay-threat" ), repository_root_provider=lambda: REPOSITORY_ROOT, camera_frame_provider=( session_recorded_camera_frame_service.extract if session_recorded_camera_frame_service is not None else None ), ) ) app.include_router( build_m48_object_quality_router( pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48" / "object-quality-packs" ), workflow_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "laboratory-annotations" / "m48-object-quality" ), truth_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48" / "object-truth-seals" ), result_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48" / "object-quality-results" ), small_static_result_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48" / "small-static-passage-regression-results" ), static_occupancy_result_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48" / "static-occupancy-qualification-results" ), camera_frame_provider=( session_recorded_camera_frame_service.extract if session_recorded_camera_frame_service is not None else None ), camera_playback_provider=( _m48_recorded_camera_playback_source if session_recorded_camera_frame_service is not None else None ), spatial_evidence_provider=m48_raw_evidence_reader, evaluation_runner=LABORATORY_RUNNER, evaluation_receipt_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "laboratory-run-receipts" ), ) ) app.include_router( build_m48r3_static_occupancy_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48" / "static-occupancy-shadow-results" ), repository_root_provider=lambda: REPOSITORY_ROOT, camera_frame_provider=( session_recorded_camera_frame_service.extract if session_recorded_camera_frame_service is not None else None ), ) ) app.include_router( build_m49_tgs_fail_closed_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m49" / "tgs-fail-closed-results" ), ) ) app.include_router( build_m49_tgs_full_shadow_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m49" / "tgs-full-shadow-results" ), ) ) app.include_router( build_vegetation_shadow_lab_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "lab-v1-vegetation" / "results" ), canonical_recording_provider=_canonical_lab_recording_source, camera_frame_provider=( session_recorded_camera_frame_service.extract if session_recorded_camera_frame_service is not None else None ), jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs", rerun_overlay_cache_root=(session_store.data_dir / "laboratory-rerun-overlays"), ffmpeg_path=_ffmpeg, ) ) app.include_router( build_vegetation_benchmark_lab_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "lab-v1-vegetation-benchmark" / "results" ), ) ) app.include_router( build_m49_physical_safety_playback_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m49" / "physical-safety-playback-results" ), ) ) app.include_router( build_m48s_fixed_class_detector_lab_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48s-semantic-shadow" / "fixed-class-detector-lab-results" ), repository_root_provider=lambda: REPOSITORY_ROOT, camera_frame_provider=( session_recorded_camera_frame_service.extract if session_recorded_camera_frame_service is not None else None ), ) ) app.include_router( build_m48t_risk_quality_lab_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48t-risk-quality" / "lab-results" ), native_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48t-risk-quality" / "native-lab-results" ), ) ) app.include_router( build_e47_semantic_slam_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e47" / "semantic-slam-results" ), ) ) app.include_router( build_e46e_ready_stack_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46e" / "ready-stack-results" ), ) ) app.include_router( build_e46f_dashcam_bakeoff_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46f" / "dashcam-bakeoff-results" ), e46e_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46e" / "ready-stack-results" ), ) ) app.include_router( build_e46g_rectified_detector_bakeoff_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46g" / "results" ) ) ) app.include_router( build_e46h_full_rectified_front_replay_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46h" / "results" ) ) ) app.include_router( build_e46i_grounding_dino_full_replay_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46i" / "results" ) ) ) app.include_router( build_e46j_raw_fisheye_realtime_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46j" / "results" ) ) ) app.include_router( build_l34_annotation_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "right-yolox-truth-island-freeze" ), truth_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results" ), evaluation_pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs" ), annotation_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "laboratory-annotations" / "l34" ), ) ) app.include_router( build_l34a_assisted_yolox_error_audit_router( audit_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "assisted-yolox-error-audits" ), l34_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "right-yolox-truth-island-freeze" ), truth_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results" ), evaluation_pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs" ), ) ) app.include_router( build_l34b_nested_box_consolidation_router( shadow_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "nested-box-consolidation-shadows" ), l34_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "right-yolox-truth-island-freeze" ), truth_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results" ), evaluation_pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs" ), ) ) app.include_router( build_l34c_tile_seam_stitch_router( shadow_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "tile-seam-stitch-shadows" ), l34_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "right-yolox-truth-island-freeze" ), truth_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results" ), evaluation_pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs" ), ) ) app.include_router( build_l34d_cumulative_postprocessing_router( candidate_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "cumulative-postprocessing-candidates" ), l34_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "right-yolox-truth-island-freeze" ), truth_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results" ), evaluation_pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs" ), ) ) app.include_router( build_l34d_blind_annotation_router( candidate_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "cumulative-postprocessing-candidates" ), l34_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "right-yolox-truth-island-freeze" ), truth_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results" ), evaluation_pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs" ), annotation_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "laboratory-annotations" / "l34d-blind" ), ) ) app.include_router( build_l34e_self_review_diagnostic_router( diagnostic_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "self-review-diagnostics" ), candidate_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "cumulative-postprocessing-candidates" ), l34_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "right-yolox-truth-island-freeze" ), truth_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results" ), evaluation_pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs" ), ) ) app.include_router( build_l34f_adjudication_router( diagnostic_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "self-review-diagnostics" ), candidate_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "cumulative-postprocessing-candidates" ), l34_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "right-yolox-truth-island-freeze" ), truth_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results" ), evaluation_pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e2" / "evaluation-packs" ), adjudication_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "laboratory-annotations" / "l34f-adjudication" ), result_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "adjudicated-references" ), ) ) app.include_router( build_e30_review_router( materialization_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "materializations" ), review_pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "review-packs" ), ) ) app.include_router( build_e40_case_review_router( e40_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e40" / "results" ), operator_review_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e40" / "operator-reviews" ), ) ) app.include_router( build_e30_engineering_router( generation_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "engineering-generations" ), materialization_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "materializations" ), review_pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "review-packs" ), ) ) app.include_router( build_e30_human_review_router( materialization_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "materializations" ), review_pack_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "review-packs" ), engineering_generation_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "engineering-generations" ), draft_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "human-review-drafts" ), generation_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e30" / "human-review-generations" ), ) ) app.include_router( build_system_telemetry_router( root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system", ) ) app.include_router( build_artifact_health_router( runtime_root_provider=lambda: REPOSITORY_ROOT / ".runtime", e44_results_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e44" / "results" ), e50_results_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e50" / "results" ), ) ) app.include_router( build_compute_contour_router( root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system", ) ) app.include_router(build_simulation_world_provider_router()) app.include_router( build_simulation_projects_router( store=simulation_project_store, service=simulation_project_service, ) ) frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist" app.include_router( build_viewer_diagnostics_router( expected_ui_build_id=lambda: frontend_build_id(frontend_dist), ) ) if frontend_dist.is_dir(): app.mount( "/", ControlStationStaticFiles(directory=frontend_dist, html=True), name="frontend", )