refactor(platform): harden LAB evidence and telemetry

2026-08-05 10:03:37 +03:00
parent 67bd96868f
commit 4c763bd8aa
42 changed files with 936 additions and 272 deletions
@@ -55,6 +55,7 @@ from k1link.compute.e40_perception_product_gate import (
    E40PerceptionProductGateError,
    read_e40_perception_product_gate,
 )
+from k1link.laboratory import LaboratoryEvidenceDefinition, LaboratoryEvidenceRegistry
 from k1link.web.l3_pointpillars_visual_api import latest_l3_visual_identity
 from k1link.web.l32_pointpillars_camera_review_api import latest_l32_identity
 from k1link.web.l33_camera_first_detector_review_api import latest_l33_identity
@@ -72,24 +73,6 @@ _E37_RESULT_ID = re.compile(r"^e37-ravnoves-acceptance-[a-f0-9]{64}$")
 _E38_RESULT_ID = re.compile(r"^e38-perception-baseline-[a-f0-9]{64}$")
 _E39_RESULT_ID = re.compile(r"^e39-perception-refinement-[a-f0-9]{64}$")
 _E40_RESULT_ID = re.compile(r"^e40-perception-product-gate-[a-f0-9]{64}$")
-_E46_RESULT_ID = re.compile(r"^e46-detector-truth-island-[a-f0-9]{64}$")
-_E46A_RESULT_ID = re.compile(r"^e46a-ai-engineering-preannotation-[a-f0-9]{64}$")
-_E46B_RESULT_ID = re.compile(r"^e46b-temporal-motion-[a-f0-9]{64}$")
-_E46C_RESULT_ID = re.compile(r"^e46c-full-replay-world-tracks-[a-f0-9]{64}$")
-_E46D_RESULT_ID = re.compile(r"^e46d-temporal-failure-audit-[a-f0-9]{64}$")
-_E46E_RESULT_ID = re.compile(r"^e46e-ready-stack-[a-f0-9]{64}$")
-_E46F_RESULT_ID = re.compile(r"^e46f-dashcam-bakeoff-[a-f0-9]{64}$")
-_E46G_RESULT_ID = re.compile(r"^e46g-rectified-detector-bakeoff-[a-f0-9]{64}$")
-_E46H_RESULT_ID = re.compile(r"^e46h-full-rectified-front-replay-[a-f0-9]{64}$")
-_E46I_RESULT_ID = re.compile(r"^e46i-grounding-dino-full-replay-[a-f0-9]{64}$")
-_E46J_RESULT_ID = re.compile(r"^e46j-raw-fisheye-realtime-[a-f0-9]{64}$")
-_L34_RESULT_ID = re.compile(r"^l34-right-yolox-truth-island-freeze-[a-f0-9]{64}$")
-_L34A_RESULT_ID = re.compile(r"^l34a-assisted-yolox-error-audit-[a-f0-9]{64}$")
-_L34B_RESULT_ID = re.compile(r"^l34b-nested-box-consolidation-shadow-[a-f0-9]{64}$")
-_L34C_RESULT_ID = re.compile(r"^l34c-tile-seam-stitch-shadow-[a-f0-9]{64}$")
-_L34D_RESULT_ID = re.compile(r"^l34d-cumulative-postprocessing-candidate-[a-f0-9]{64}$")
-_L34E_RESULT_ID = re.compile(r"^l34e-self-review-diagnostic-[a-f0-9]{64}$")
-_L34F_RESULT_ID = re.compile(r"^l34f-adjudicated-reference-[a-f0-9]{64}$")

 RootProvider = Callable[[], Path | None]

@@ -299,6 +282,50 @@ def _advanced_index(
    }


+def _registry_index_specs(
+    registry: LaboratoryEvidenceRegistry,
+    runtime_root_provider: RootProvider,
+) -> tuple[_AdvancedIndexSpec, ...]:
+    return tuple(
+        (
+            definition.work_id,
+            _evidence_root_provider(definition, runtime_root_provider),
+            definition.result_id_pattern,
+            definition.document_name,
+            definition.result_schema_version,
+        )
+        for definition in registry.definitions
+    )
+
+
+def _evidence_root_provider(
+    definition: LaboratoryEvidenceDefinition,
+    runtime_root_provider: RootProvider,
+) -> RootProvider:
+    def result_root_provider() -> Path | None:
+        configured_runtime_root = runtime_root_provider()
+        if configured_runtime_root is None:
+            return None
+        runtime_candidate = configured_runtime_root.expanduser().absolute()
+        if runtime_candidate.is_symlink():
+            return None
+        try:
+            runtime_root = runtime_candidate.resolve(strict=True)
+        except OSError:
+            return None
+        candidate = definition.result_root(runtime_root)
+        if candidate.exists():
+            try:
+                resolved = candidate.resolve(strict=True)
+            except OSError:
+                return None
+            if resolved != candidate.absolute() or not resolved.is_relative_to(runtime_root):
+                return None
+        return candidate
+
+    return result_root_provider
+
+
 def _object(value: object, label: str) -> dict[str, Any]:
    if not isinstance(value, dict):
        raise ValueError(f"{label} is invalid")
@@ -786,6 +813,8 @@ def _empty_catalog(configured: bool) -> dict[str, object]:

 def build_advanced_laboratory_router(
    *,
+    evidence_registry: LaboratoryEvidenceRegistry | None = None,
+    evidence_runtime_root_provider: RootProvider = lambda: None,
    e31_root_provider: RootProvider = lambda: None,
    e32_root_provider: RootProvider = lambda: None,
    e33_root_provider: RootProvider = lambda: None,
@@ -795,24 +824,6 @@ def build_advanced_laboratory_router(
    e38_root_provider: RootProvider = lambda: None,
    e39_root_provider: RootProvider = lambda: None,
    e40_root_provider: RootProvider = lambda: None,
-    e46_root_provider: RootProvider = lambda: None,
-    e46a_root_provider: RootProvider = lambda: None,
-    e46b_root_provider: RootProvider = lambda: None,
-    e46c_root_provider: RootProvider = lambda: None,
-    e46d_root_provider: RootProvider = lambda: None,
-    e46e_root_provider: RootProvider = lambda: None,
-    e46f_root_provider: RootProvider = lambda: None,
-    e46g_root_provider: RootProvider = lambda: None,
-    e46h_root_provider: RootProvider = lambda: None,
-    e46i_root_provider: RootProvider = lambda: None,
-    e46j_root_provider: RootProvider = lambda: None,
-    l34_root_provider: RootProvider = lambda: None,
-    l34a_root_provider: RootProvider = lambda: None,
-    l34b_root_provider: RootProvider = lambda: None,
-    l34c_root_provider: RootProvider = lambda: None,
-    l34d_root_provider: RootProvider = lambda: None,
-    l34e_root_provider: RootProvider = lambda: None,
-    l34f_root_provider: RootProvider = lambda: None,
    l3_visual_root_provider: RootProvider = lambda: None,
    l31_ravnoves_root_provider: RootProvider = lambda: None,
    l32_camera_review_root_provider: RootProvider = lambda: None,
@@ -822,8 +833,8 @@ def build_advanced_laboratory_router(

    @router.get("/advanced-index")
    def list_advanced_results() -> dict[str, object]:
-        result = _advanced_index(
-            (
+        if evidence_registry is None:
+            specs: tuple[_AdvancedIndexSpec, ...] = (
                (
                    "e31-source-binding",
                    e31_root_provider,
@@ -887,134 +898,13 @@ def build_advanced_laboratory_router(
                    "manifest.json",
                    "missioncore.e40-perception-product-gate/v1",
                ),
-                (
-                    "e46-detector-truth-island",
-                    e46_root_provider,
-                    _E46_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.e46-detector-truth-island/v1",
-                ),
-                (
-                    "e46a-ai-engineering-preannotation",
-                    e46a_root_provider,
-                    _E46A_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.e46a-ai-engineering-preannotation/v1",
-                ),
-                (
-                    "e46b-temporal-motion",
-                    e46b_root_provider,
-                    _E46B_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.e46b-temporal-motion/v1",
-                ),
-                (
-                    "e46c-full-replay-world-tracks",
-                    e46c_root_provider,
-                    _E46C_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.e46c-full-replay-world-tracks/v1",
-                ),
-                (
-                    "e46d-temporal-failure-audit",
-                    e46d_root_provider,
-                    _E46D_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.e46d-temporal-failure-audit/v1",
-                ),
-                (
-                    "e46e-ready-stack",
-                    e46e_root_provider,
-                    _E46E_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.e46e-ready-stack-result/v1",
-                ),
-                (
-                    "e46f-dashcam-bakeoff",
-                    e46f_root_provider,
-                    _E46F_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.e46f-dashcam-bakeoff-result/v1",
-                ),
-                (
-                    "e46g-rectified-detector-bakeoff",
-                    e46g_root_provider,
-                    _E46G_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.e46g-rectified-detector-bakeoff-result/v1",
-                ),
-                (
-                    "e46h-full-rectified-front-replay",
-                    e46h_root_provider,
-                    _E46H_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.e46h-full-rectified-front-replay-result/v1",
-                ),
-                (
-                    "e46i-grounding-dino-full-replay",
-                    e46i_root_provider,
-                    _E46I_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.e46i-grounding-dino-full-replay-result/v1",
-                ),
-                (
-                    "e46j-raw-fisheye-realtime",
-                    e46j_root_provider,
-                    _E46J_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.e46j-raw-fisheye-realtime-result/v1",
-                ),
-                (
-                    "l34-right-yolox-truth-island-freeze",
-                    l34_root_provider,
-                    _L34_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.l34-right-yolox-truth-island-freeze/v1",
-                ),
-                (
-                    "l34a-assisted-yolox-error-audit",
-                    l34a_root_provider,
-                    _L34A_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.l34a-assisted-yolox-error-audit/v1",
-                ),
-                (
-                    "l34b-nested-box-consolidation-shadow",
-                    l34b_root_provider,
-                    _L34B_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.l34b-nested-box-consolidation-shadow/v1",
-                ),
-                (
-                    "l34c-tile-seam-stitch-shadow",
-                    l34c_root_provider,
-                    _L34C_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.l34c-tile-seam-stitch-shadow/v1",
-                ),
-                (
-                    "l34d-cumulative-postprocessing-candidate",
-                    l34d_root_provider,
-                    _L34D_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.l34d-cumulative-postprocessing-candidate/v1",
-                ),
-                (
-                    "l34e-self-review-diagnostic",
-                    l34e_root_provider,
-                    _L34E_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.l34e-self-review-diagnostic/v1",
-                ),
-                (
-                    "l34f-adjudicated-reference",
-                    l34f_root_provider,
-                    _L34F_RESULT_ID,
-                    "manifest.json",
-                    "missioncore.l34f-adjudicated-reference/v1",
-                ),
            )
-        )
+        else:
+            specs = _registry_index_specs(
+                evidence_registry,
+                evidence_runtime_root_provider,
+            )
+        result = _advanced_index(specs)
        items = result["items"]
        if not isinstance(items, list):
            raise RuntimeError("advanced LAB index items are invalid")
@@ -23,6 +23,7 @@ from k1link.compute import (
    RecordedPerceptionOverlayMux,
    RecordedPerceptionOverlayStore,
 )
+from k1link.laboratory import LaboratoryEvidenceRegistry
 from k1link.sessions import (
    MaterializedRecording,
    RecordedMediaInspector,
@@ -133,6 +134,9 @@ from k1link.web.viewer_diagnostics_api import build_viewer_diagnostics_router

 REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
 INVALID_REQUEST_DETAIL = "Некорректные параметры запроса."
+LABORATORY_EVIDENCE_REGISTRY = LaboratoryEvidenceRegistry.from_directory(
+    REPOSITORY_ROOT / "config" / "laboratories"
+)


 def _resolve_media_tool(name: str) -> Path | None:
@@ -557,6 +561,10 @@ app.include_router(
 )
 app.include_router(
    build_advanced_laboratory_router(
+        evidence_registry=LABORATORY_EVIDENCE_REGISTRY,
+        evidence_runtime_root_provider=lambda: (
+            REPOSITORY_ROOT / ".runtime" / "compute-experiments"
+        ),
        e31_root_provider=lambda: (
            REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e31" / "source-qualifications"
        ),
@@ -584,96 +592,6 @@ app.include_router(
        e40_root_provider=lambda: (
            REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e40" / "results"
        ),
-        e46_root_provider=lambda: (
-            REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results"
-        ),
-        e46a_root_provider=lambda: (
-            REPOSITORY_ROOT
-            / ".runtime"
-            / "compute-experiments"
-            / "e46a"
-            / "ai-engineering-preannotations"
-        ),
-        e46b_root_provider=lambda: (
-            REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46b" / "temporal-motion"
-        ),
-        e46c_root_provider=lambda: (
-            REPOSITORY_ROOT
-            / ".runtime"
-            / "compute-experiments"
-            / "e46c"
-            / "full-replay-world-tracks"
-        ),
-        e46d_root_provider=lambda: (
-            REPOSITORY_ROOT
-            / ".runtime"
-            / "compute-experiments"
-            / "e46d"
-            / "temporal-failure-audits"
-        ),
-        e46e_root_provider=lambda: (
-            REPOSITORY_ROOT
-            / ".runtime"
-            / "compute-experiments"
-            / "e46e"
-            / "ready-stack-results"
-        ),
-        e46f_root_provider=lambda: (
-            REPOSITORY_ROOT
-            / ".runtime"
-            / "compute-experiments"
-            / "e46f"
-            / "dashcam-bakeoff-results"
-        ),
-        e46g_root_provider=lambda: (
-            REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46g" / "results"
-        ),
-        e46h_root_provider=lambda: (
-            REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46h" / "results"
-        ),
-        e46i_root_provider=lambda: (
-            REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46i" / "results"
-        ),
-        e46j_root_provider=lambda: (
-            REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46j" / "results"
-        ),
-        l34_root_provider=lambda: (
-            REPOSITORY_ROOT
-            / ".runtime"
-            / "compute-experiments"
-            / "l3"
-            / "right-yolox-truth-island-freeze"
-        ),
-        l34a_root_provider=lambda: (
-            REPOSITORY_ROOT
-            / ".runtime"
-            / "compute-experiments"
-            / "l3"
-            / "assisted-yolox-error-audits"
-        ),
-        l34b_root_provider=lambda: (
-            REPOSITORY_ROOT
-            / ".runtime"
-            / "compute-experiments"
-            / "l3"
-            / "nested-box-consolidation-shadows"
-        ),
-        l34c_root_provider=lambda: (
-            REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "tile-seam-stitch-shadows"
-        ),
-        l34d_root_provider=lambda: (
-            REPOSITORY_ROOT
-            / ".runtime"
-            / "compute-experiments"
-            / "l3"
-            / "cumulative-postprocessing-candidates"
-        ),
-        l34e_root_provider=lambda: (
-            REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "self-review-diagnostics"
-        ),
-        l34f_root_provider=lambda: (
-            REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "adjudicated-references"
-        ),
        l3_visual_root_provider=lambda: (
            REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "visual-audits"
        ),
@@ -313,6 +313,15 @@ elseif ((Get-Service -Name "telegraf").Status -ne "Running") {
    [TimeSpan]::FromSeconds(20)
  )
 }
+& sc.exe failure telegraf reset= 86400 `
+  actions= restart/5000/restart/30000/restart/60000 | Out-Null
+if ($LASTEXITCODE -ne 0) {
+  throw "Failed to configure Telegraf service recovery actions"
+}
+& sc.exe failureflag telegraf 1 | Out-Null
+if ($LASTEXITCODE -ne 0) {
+  throw "Failed to enable Telegraf recovery for non-crash failures"
+}
 $resolved = @(
  [Net.Dns]::GetHostAddresses([string]$payload.mqtt_host) |
    ForEach-Object { $_.IPAddressToString } |
@@ -326,6 +335,7 @@ $resolved = @(
  mqtt_port = [int]$payload.mqtt_port
  resolved_addresses = $resolved
  broker_reachable = $true
+  recovery_configured = $true
  changed = $changed
 } | ConvertTo-Json -Depth 5 -Compress
 """.strip()
@@ -587,6 +597,7 @@ def apply_worker_network(
        or document.get("mqtt_port") != target.mqtt_port
        or document.get("service_status") != "Running"
        or document.get("broker_reachable") is not True
+        or document.get("recovery_configured") is not True
    ):
        raise NetworkOperationError(
            "Worker не подтвердил применённый сетевой профиль."