feat(observatory): ship modular AI inference labs

2026-09-04 17:59:05 +03:00
parent eff60e490a
commit cada687173
145 changed files with 17651 additions and 1667 deletions
@@ -42,11 +42,19 @@ from k1link.observatory import (
    ObservatoryRunPreparationLedger,
    load_observatory_run_preparation_ledger,
 )
+from k1link.observatory.composition_runs import CompositionRunError, CompositionRunStore
+from k1link.observatory.domain_ontology import (
+    ObservatoryDomainOntology,
+    ObservatoryOntologyError,
+)
+from k1link.observatory.lab_view_profiles import LabViewProfileError, LabViewProfileStore
 from k1link.observatory.m49_queue_binding import (
    M49QueueBindingConfig,
    M49QueueBindingError,
    M49RecordedQueueBindingService,
 )
+from k1link.observatory.modular_composition import CompositionError, ModuleRegistry
+from k1link.observatory.modular_composition_store import ModularCompositionStore
 from k1link.observatory.portable_publication_reconciler import (
    PortablePublicationReconciler,
 )
@@ -196,6 +204,7 @@ from k1link.web.map_api import (
    build_map_router,
 )
 from k1link.web.map_view_api import build_map_view_router
+from k1link.web.modular_observatory_api import build_modular_observatory_router
 from k1link.web.observatory_api import build_observatory_router
 from k1link.web.observatory_worker_api import (
    ObservatoryWorkerAuthentication,
@@ -288,6 +297,37 @@ plugin_environment = load_installed_device_plugins(REPOSITORY_ROOT)
 plugin_catalog: DevicePluginCatalog = plugin_environment.catalog
 plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher
 session_store = SessionStore(REPOSITORY_ROOT)
+OBSERVATORY_AI_COMPOSITIONS: ModularCompositionStore | None
+OBSERVATORY_AI_COMPOSITION_RUNS: CompositionRunStore | None
+OBSERVATORY_DOMAIN_ONTOLOGY: ObservatoryDomainOntology | None
+OBSERVATORY_LAB_VIEW_PROFILES: LabViewProfileStore | None
+try:
+    OBSERVATORY_DOMAIN_ONTOLOGY = ObservatoryDomainOntology.from_file(
+        REPOSITORY_ROOT / "config" / "observatory-domain-ontology.json"
+    )
+    OBSERVATORY_AI_COMPOSITIONS = ModularCompositionStore(
+        session_store.data_dir / "observatory-ai-compositions",
+        ModuleRegistry.from_file(REPOSITORY_ROOT / "config" / "observatory-ai-modules.json"),
+    )
+    OBSERVATORY_AI_COMPOSITION_RUNS = CompositionRunStore(
+        session_store.data_dir / "observatory-ai-composition-runs"
+    )
+    OBSERVATORY_LAB_VIEW_PROFILES = LabViewProfileStore(
+        session_store.data_dir / "observatory-lab-view-profiles"
+    )
+except (
+    CompositionError,
+    CompositionRunError,
+    LabViewProfileError,
+    ObservatoryOntologyError,
+    OSError,
+    ValueError,
+):
+    # A modular catalog failure cannot disable recordings, existing LABs or Legacy.
+    OBSERVATORY_AI_COMPOSITIONS = None
+    OBSERVATORY_AI_COMPOSITION_RUNS = None
+    OBSERVATORY_DOMAIN_ONTOLOGY = None
+    OBSERVATORY_LAB_VIEW_PROFILES = None

 OBSERVATORY_PORTABLE_DEFINITION_REGISTRY: PortableRunDefinitionRegistry | None
 OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR: str | None
@@ -322,9 +362,7 @@ def _resolve_observatory_calculation_profile(
    summary: SessionSummary,
 ) -> dict[str, object] | None:
    if OBSERVATORY_LABORATORY_SETUP_REGISTRY is not None:
-        legacy = OBSERVATORY_LABORATORY_SETUP_REGISTRY.observatory_calculation_profile(
-            summary
-        )
+        legacy = OBSERVATORY_LABORATORY_SETUP_REGISTRY.observatory_calculation_profile(summary)
        if legacy is not None:
            return legacy
    if (
@@ -437,47 +475,30 @@ try:
            or "portable definition registry is unavailable"
        )
    if OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None:
-        raise PortableWorkerIntegrationError(
-            "portable calculation profile registry is unavailable"
-        )
+        raise PortableWorkerIntegrationError("portable calculation profile registry is unavailable")
    if OBSERVATORY_PORTABLE_RESULT_VALIDATORS is None:
-        raise PortableWorkerIntegrationError(
-            "portable result validator registry is unavailable"
-        )
+        raise PortableWorkerIntegrationError("portable result validator registry is unavailable")
    if OBSERVATORY_RECORDED_JOB_QUEUE is None:
        raise PortableWorkerIntegrationError(
-            OBSERVATORY_RECORDED_JOB_QUEUE_ERROR
-            or "Observatory recorded-job queue is unavailable"
+            OBSERVATORY_RECORDED_JOB_QUEUE_ERROR or "Observatory recorded-job queue is unavailable"
        )
    if session_artifact_gateway is None:
-        raise PortableWorkerIntegrationError(
-            "central artifact store is not configured"
-        )
+        raise PortableWorkerIntegrationError("central artifact store is not configured")
    if session_artifact_gateway.status().central_status != "ready":
-        raise PortableWorkerIntegrationError(
-            "central artifact store is unavailable"
-        )
-    OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS = (
-        PortableWorkerStorageRoots.from_environment(
-            artifact_store_root=session_artifact_gateway.store.root,
-        )
+        raise PortableWorkerIntegrationError("central artifact store is unavailable")
+    OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS = PortableWorkerStorageRoots.from_environment(
+        artifact_store_root=session_artifact_gateway.store.root,
    )
-    OBSERVATORY_PORTABLE_WORKER_INTEGRATION = (
-        build_portable_observatory_worker_integration(
-            queue=OBSERVATORY_RECORDED_JOB_QUEUE,
-            session_store=session_store,
-            media_inspector=session_recorded_media_inspector,
-            definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
-            artifact_store=session_artifact_gateway.store,
-            calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
-            validators=OBSERVATORY_PORTABLE_RESULT_VALIDATORS,
-            source_cas_root=(
-                OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.source_cas_root
-            ),
-            result_staging_root=(
-                OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.result_staging_root
-            ),
-        )
+    OBSERVATORY_PORTABLE_WORKER_INTEGRATION = build_portable_observatory_worker_integration(
+        queue=OBSERVATORY_RECORDED_JOB_QUEUE,
+        session_store=session_store,
+        media_inspector=session_recorded_media_inspector,
+        definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
+        artifact_store=session_artifact_gateway.store,
+        calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
+        validators=OBSERVATORY_PORTABLE_RESULT_VALIDATORS,
+        source_cas_root=(OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.source_cas_root),
+        result_staging_root=(OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.result_staging_root),
    )
    OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = None
 except (PortableWorkerIntegrationError, OSError, ValueError) as exc:
@@ -487,10 +508,7 @@ except (PortableWorkerIntegrationError, OSError, ValueError) as exc:
    OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = str(exc)
 OBSERVATORY_PUBLICATION_RECONCILER = (
    None
-    if (
-        OBSERVATORY_RECORDED_JOB_QUEUE is None
-        or OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None
-    )
+    if (OBSERVATORY_RECORDED_JOB_QUEUE is None or OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None)
    else PortablePublicationReconciler(
        queue=OBSERVATORY_RECORDED_JOB_QUEUE,
        artifact_transport=OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport,
@@ -499,12 +517,10 @@ OBSERVATORY_PUBLICATION_RECONCILER = (
 )
 OBSERVATORY_WORKER_API_GATE_ENABLED = OBSERVATORY_WORKER_LOCAL_ENABLED
 OBSERVATORY_WORKER_CLAIM_LEASE_READY = (
-    OBSERVATORY_WORKER_API_GATE_ENABLED
-    and OBSERVATORY_RECORDED_JOB_QUEUE is not None
+    OBSERVATORY_WORKER_API_GATE_ENABLED and OBSERVATORY_RECORDED_JOB_QUEUE is not None
 )
 OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = (
-    OBSERVATORY_WORKER_API_GATE_ENABLED
-    and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
+    OBSERVATORY_WORKER_API_GATE_ENABLED and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
 )
 OBSERVATORY_WORKER_DISPATCH_READY = (
    OBSERVATORY_WORKER_CLAIM_LEASE_READY
@@ -527,17 +543,13 @@ else:
        )
    if OBSERVATORY_WORKER_AUTHENTICATION_ERROR is not None:
        worker_api_errors.append(
-            "authentication unavailable: "
-            f"{OBSERVATORY_WORKER_AUTHENTICATION_ERROR}"
+            f"authentication unavailable: {OBSERVATORY_WORKER_AUTHENTICATION_ERROR}"
        )
    if OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR is not None:
        worker_api_errors.append(
-            "integration unavailable: "
-            f"{OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR}"
+            f"integration unavailable: {OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR}"
        )
-    OBSERVATORY_WORKER_API_ERROR = "Worker pull API is disabled; " + "; ".join(
-        worker_api_errors
-    )
+    OBSERVATORY_WORKER_API_ERROR = "Worker pull API is disabled; " + "; ".join(worker_api_errors)
 OBSERVATORY_PORTABLE_BINDING_SERVICE: PortableRecordedQueueBindingService | None
 OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableSetupProjector | None
 OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
@@ -554,7 +566,8 @@ try:
        and OBSERVATORY_PORTABLE_CALCULATION_PROFILES is not None
    ):
        OBSERVATORY_PORTABLE_RESULT_CACHE = PortableResultCache(
-            sessions=session_store, artifacts=session_artifact_gateway.store,
+            sessions=session_store,
+            artifacts=session_artifact_gateway.store,
            queue=OBSERVATORY_RECORDED_JOB_QUEUE,
            definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
            calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
@@ -566,7 +579,8 @@ try:
        definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
        queue=OBSERVATORY_RECORDED_JOB_QUEUE,
        published_result_available=(
-            None if OBSERVATORY_PORTABLE_RESULT_CACHE is None
+            None
+            if OBSERVATORY_PORTABLE_RESULT_CACHE is None
            else OBSERVATORY_PORTABLE_RESULT_CACHE.available
        ),
    )
@@ -874,10 +888,19 @@ async def _portable_result_publication_reconciler() -> None:
        await asyncio.sleep(15.0)


+async def _recorded_blueprint_resource_reaper() -> None:
+    from k1link.viewer.recorded import recorded_blueprint_sessions
+
+    while True:
+        await asyncio.sleep(30.0)
+        await asyncio.to_thread(recorded_blueprint_sessions.expire)
+
+
@asynccontextmanager
 async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
    reconciler: asyncio.Task[None] | None = None
    publication_reconciler: asyncio.Task[None] | None = None
+    blueprint_reaper: asyncio.Task[None] | None = None
    try:
        configure_scanner_diagnostics(session_store.data_dir / "logs")
        session_recording_preparation_manager.start()
@@ -891,11 +914,17 @@ async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
        # expensive on field captures. Start it immediately in the background
        # instead of holding the ASGI startup gate.
        reconciler = asyncio.create_task(_recording_preparation_reconciler())
-        publication_reconciler = asyncio.create_task(
-            _portable_result_publication_reconciler()
-        )
+        publication_reconciler = asyncio.create_task(_portable_result_publication_reconciler())
+        blueprint_reaper = asyncio.create_task(_recorded_blueprint_resource_reaper())
        yield
    finally:
+        from k1link.viewer.recorded import recorded_blueprint_sessions
+
+        if blueprint_reaper is not None:
+            blueprint_reaper.cancel()
+            with suppress(asyncio.CancelledError):
+                await blueprint_reaper
+        await asyncio.to_thread(recorded_blueprint_sessions.close)
        await map_gateway_proxy.close()
        if reconciler is not None:
            reconciler.cancel()
@@ -1066,7 +1095,9 @@ if session_artifact_gateway is not None and _ffmpeg is not None:
                media=session_recorded_media_inspector,
                recording_source=_canonical_lab_recording_source,
                ffmpeg_path=_ffmpeg,
-            )
+            ),
+            composition_runs=OBSERVATORY_AI_COMPOSITION_RUNS,
+            queue=OBSERVATORY_RECORDED_JOB_QUEUE,
        )
    )

@@ -1128,6 +1159,23 @@ app.include_router(
        ),
    )
 )
+if (
+    OBSERVATORY_AI_COMPOSITIONS is not None
+    and OBSERVATORY_AI_COMPOSITION_RUNS is not None
+    and OBSERVATORY_DOMAIN_ONTOLOGY is not None
+):
+    app.include_router(
+        build_modular_observatory_router(
+            store=session_store,
+            compositions=OBSERVATORY_AI_COMPOSITIONS,
+            composition_runs=OBSERVATORY_AI_COMPOSITION_RUNS,
+            ontology=OBSERVATORY_DOMAIN_ONTOLOGY,
+            definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
+            binding=OBSERVATORY_PORTABLE_BINDING_SERVICE,
+            queue=OBSERVATORY_RECORDED_JOB_QUEUE,
+            view_profiles=OBSERVATORY_LAB_VIEW_PROFILES,
+        )
+    )
 if OBSERVATORY_WORKER_DISPATCH_READY:
    assert OBSERVATORY_RECORDED_JOB_QUEUE is not None
    assert OBSERVATORY_WORKER_AUTHENTICATION is not None
@@ -1136,12 +1184,8 @@ if OBSERVATORY_WORKER_DISPATCH_READY:
        build_observatory_worker_router(
            OBSERVATORY_RECORDED_JOB_QUEUE,
            authentication=OBSERVATORY_WORKER_AUTHENTICATION,
-            artifact_transport=(
-                OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport
-            ),
-            result_publisher=(
-                OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher
-            ),
+            artifact_transport=(OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport),
+            result_publisher=(OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher),
        )
    )
 app.include_router(