From 4c763bd8aabae687d10d62e09fb442374a18f4f5 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Wed, 5 Aug 2026 10:03:37 +0300 Subject: [PATCH] refactor(platform): harden LAB evidence and telemetry --- config/laboratories/e31-source-binding.json | 10 + config/laboratories/e32-track-geometry.json | 10 + config/laboratories/e33-worker-shadow.json | 10 + config/laboratories/e34-temporal-layer.json | 10 + .../e35-degradation-recovery.json | 10 + .../laboratories/e37-ravnoves-acceptance.json | 10 + .../laboratories/e38-perception-baseline.json | 10 + .../e39-perception-refinement.json | 10 + .../e40-perception-product-gate.json | 10 + .../e46-detector-truth-island.json | 10 + .../e46a-ai-engineering-preannotation.json | 10 + config/laboratories/e46b-temporal-motion.json | 10 + .../e46c-full-replay-world-tracks.json | 10 + .../e46d-temporal-failure-audit.json | 10 + config/laboratories/e46e-ready-stack.json | 10 + config/laboratories/e46f-dashcam-bakeoff.json | 10 + .../e46g-rectified-detector-bakeoff.json | 10 + .../e46h-full-rectified-front-replay.json | 10 + .../e46i-grounding-dino-full-replay.json | 10 + .../e46j-raw-fisheye-realtime.json | 10 + .../l34-right-yolox-truth-island-freeze.json | 10 + .../l34a-assisted-yolox-error-audit.json | 10 + .../l34b-nested-box-consolidation-shadow.json | 10 + .../l34c-tile-seam-stitch-shadow.json | 10 + ...d-cumulative-postprocessing-candidate.json | 10 + .../l34e-self-review-diagnostic.json | 10 + .../l34f-adjudicated-reference.json | 10 + deploy/telemetry-plane/README.md | 6 + .../Install-NdcMissionCoreTelegraf.ps1 | 10 + .../Update-NdcMissionCoreTelegraf.ps1 | 20 ++ ...1-local-compute-contour-telemetry-plane.md | 33 ++- ...figurative-laboratory-evidence-registry.md | 67 ++++++ src/k1link/laboratory/__init__.py | 15 ++ src/k1link/laboratory/evidence_registry.py | 187 +++++++++++++++ src/k1link/web/advanced_laboratory_api.py | 220 +++++------------- src/k1link/web/app.py | 98 +------- src/k1link/web/compute_contour_network.py | 11 + tests/test_advanced_laboratory_api.py | 83 ++++++- tests/test_compute_contour_network.py | 3 + .../test_l34_right_yolox_truth_island_api.py | 18 +- tests/test_laboratory_evidence_registry.py | 135 +++++++++++ tests/test_telegraf_deployment_contracts.py | 32 +++ 42 files changed, 936 insertions(+), 272 deletions(-) create mode 100644 config/laboratories/e31-source-binding.json create mode 100644 config/laboratories/e32-track-geometry.json create mode 100644 config/laboratories/e33-worker-shadow.json create mode 100644 config/laboratories/e34-temporal-layer.json create mode 100644 config/laboratories/e35-degradation-recovery.json create mode 100644 config/laboratories/e37-ravnoves-acceptance.json create mode 100644 config/laboratories/e38-perception-baseline.json create mode 100644 config/laboratories/e39-perception-refinement.json create mode 100644 config/laboratories/e40-perception-product-gate.json create mode 100644 config/laboratories/e46-detector-truth-island.json create mode 100644 config/laboratories/e46a-ai-engineering-preannotation.json create mode 100644 config/laboratories/e46b-temporal-motion.json create mode 100644 config/laboratories/e46c-full-replay-world-tracks.json create mode 100644 config/laboratories/e46d-temporal-failure-audit.json create mode 100644 config/laboratories/e46e-ready-stack.json create mode 100644 config/laboratories/e46f-dashcam-bakeoff.json create mode 100644 config/laboratories/e46g-rectified-detector-bakeoff.json create mode 100644 config/laboratories/e46h-full-rectified-front-replay.json create mode 100644 config/laboratories/e46i-grounding-dino-full-replay.json create mode 100644 config/laboratories/e46j-raw-fisheye-realtime.json create mode 100644 config/laboratories/l34-right-yolox-truth-island-freeze.json create mode 100644 config/laboratories/l34a-assisted-yolox-error-audit.json create mode 100644 config/laboratories/l34b-nested-box-consolidation-shadow.json create mode 100644 config/laboratories/l34c-tile-seam-stitch-shadow.json create mode 100644 config/laboratories/l34d-cumulative-postprocessing-candidate.json create mode 100644 config/laboratories/l34e-self-review-diagnostic.json create mode 100644 config/laboratories/l34f-adjudicated-reference.json create mode 100644 docs/adr/0036-configurative-laboratory-evidence-registry.md create mode 100644 src/k1link/laboratory/__init__.py create mode 100644 src/k1link/laboratory/evidence_registry.py create mode 100644 tests/test_laboratory_evidence_registry.py create mode 100644 tests/test_telegraf_deployment_contracts.py diff --git a/config/laboratories/e31-source-binding.json b/config/laboratories/e31-source-binding.json new file mode 100644 index 0000000..c973425 --- /dev/null +++ b/config/laboratories/e31-source-binding.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e31-source-binding", + "evidence": { + "runtime_relative_root": "e31/source-qualifications", + "result_id_prefix": "e31-source-qualification", + "document_name": "manifest.json", + "schema_version": "missioncore.e31-source-qualification/v1" + } +} diff --git a/config/laboratories/e32-track-geometry.json b/config/laboratories/e32-track-geometry.json new file mode 100644 index 0000000..b53e54b --- /dev/null +++ b/config/laboratories/e32-track-geometry.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e32-track-geometry", + "evidence": { + "runtime_relative_root": "e32/results", + "result_id_prefix": "e32-track-geometry", + "document_name": "manifest.json", + "schema_version": "missioncore.e32-track-geometry-replay/v1" + } +} diff --git a/config/laboratories/e33-worker-shadow.json b/config/laboratories/e33-worker-shadow.json new file mode 100644 index 0000000..77577b5 --- /dev/null +++ b/config/laboratories/e33-worker-shadow.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e33-worker-shadow", + "evidence": { + "runtime_relative_root": "e33/results", + "result_id_prefix": "e33-worker-shadow", + "document_name": "result.json", + "schema_version": "missioncore.e33-worker-shadow-result/v1" + } +} diff --git a/config/laboratories/e34-temporal-layer.json b/config/laboratories/e34-temporal-layer.json new file mode 100644 index 0000000..377cab1 --- /dev/null +++ b/config/laboratories/e34-temporal-layer.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e34-temporal-layer", + "evidence": { + "runtime_relative_root": "e34/results", + "result_id_prefix": "e34-temporal-occupied", + "document_name": "manifest.json", + "schema_version": "missioncore.e34-temporal-occupied-result/v1" + } +} diff --git a/config/laboratories/e35-degradation-recovery.json b/config/laboratories/e35-degradation-recovery.json new file mode 100644 index 0000000..14c8d0f --- /dev/null +++ b/config/laboratories/e35-degradation-recovery.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e35-degradation-recovery", + "evidence": { + "runtime_relative_root": "e35/results", + "result_id_prefix": "e35-degradation-recovery", + "document_name": "manifest.json", + "schema_version": "missioncore.e35-degradation-result/v1" + } +} diff --git a/config/laboratories/e37-ravnoves-acceptance.json b/config/laboratories/e37-ravnoves-acceptance.json new file mode 100644 index 0000000..4b22481 --- /dev/null +++ b/config/laboratories/e37-ravnoves-acceptance.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e37-ravnoves-acceptance", + "evidence": { + "runtime_relative_root": "e37/results", + "result_id_prefix": "e37-ravnoves-acceptance", + "document_name": "manifest.json", + "schema_version": "missioncore.e37-acceptance-contract/v1" + } +} diff --git a/config/laboratories/e38-perception-baseline.json b/config/laboratories/e38-perception-baseline.json new file mode 100644 index 0000000..8512d64 --- /dev/null +++ b/config/laboratories/e38-perception-baseline.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e38-perception-baseline", + "evidence": { + "runtime_relative_root": "e38/results", + "result_id_prefix": "e38-perception-baseline", + "document_name": "manifest.json", + "schema_version": "missioncore.e38-perception-baseline/v1" + } +} diff --git a/config/laboratories/e39-perception-refinement.json b/config/laboratories/e39-perception-refinement.json new file mode 100644 index 0000000..0fb669e --- /dev/null +++ b/config/laboratories/e39-perception-refinement.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e39-perception-refinement", + "evidence": { + "runtime_relative_root": "e39/results", + "result_id_prefix": "e39-perception-refinement", + "document_name": "manifest.json", + "schema_version": "missioncore.e39-perception-refinement/v1" + } +} diff --git a/config/laboratories/e40-perception-product-gate.json b/config/laboratories/e40-perception-product-gate.json new file mode 100644 index 0000000..1978896 --- /dev/null +++ b/config/laboratories/e40-perception-product-gate.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e40-perception-product-gate", + "evidence": { + "runtime_relative_root": "e40/results", + "result_id_prefix": "e40-perception-product-gate", + "document_name": "manifest.json", + "schema_version": "missioncore.e40-perception-product-gate/v1" + } +} diff --git a/config/laboratories/e46-detector-truth-island.json b/config/laboratories/e46-detector-truth-island.json new file mode 100644 index 0000000..e2b9c5f --- /dev/null +++ b/config/laboratories/e46-detector-truth-island.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e46-detector-truth-island", + "evidence": { + "runtime_relative_root": "e46/results", + "result_id_prefix": "e46-detector-truth-island", + "document_name": "manifest.json", + "schema_version": "missioncore.e46-detector-truth-island/v1" + } +} diff --git a/config/laboratories/e46a-ai-engineering-preannotation.json b/config/laboratories/e46a-ai-engineering-preannotation.json new file mode 100644 index 0000000..55c9223 --- /dev/null +++ b/config/laboratories/e46a-ai-engineering-preannotation.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e46a-ai-engineering-preannotation", + "evidence": { + "runtime_relative_root": "e46a/ai-engineering-preannotations", + "result_id_prefix": "e46a-ai-engineering-preannotation", + "document_name": "manifest.json", + "schema_version": "missioncore.e46a-ai-engineering-preannotation/v1" + } +} diff --git a/config/laboratories/e46b-temporal-motion.json b/config/laboratories/e46b-temporal-motion.json new file mode 100644 index 0000000..5bb864e --- /dev/null +++ b/config/laboratories/e46b-temporal-motion.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e46b-temporal-motion", + "evidence": { + "runtime_relative_root": "e46b/temporal-motion", + "result_id_prefix": "e46b-temporal-motion", + "document_name": "manifest.json", + "schema_version": "missioncore.e46b-temporal-motion/v1" + } +} diff --git a/config/laboratories/e46c-full-replay-world-tracks.json b/config/laboratories/e46c-full-replay-world-tracks.json new file mode 100644 index 0000000..bbd96c0 --- /dev/null +++ b/config/laboratories/e46c-full-replay-world-tracks.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e46c-full-replay-world-tracks", + "evidence": { + "runtime_relative_root": "e46c/full-replay-world-tracks", + "result_id_prefix": "e46c-full-replay-world-tracks", + "document_name": "manifest.json", + "schema_version": "missioncore.e46c-full-replay-world-tracks/v1" + } +} diff --git a/config/laboratories/e46d-temporal-failure-audit.json b/config/laboratories/e46d-temporal-failure-audit.json new file mode 100644 index 0000000..a0d352e --- /dev/null +++ b/config/laboratories/e46d-temporal-failure-audit.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e46d-temporal-failure-audit", + "evidence": { + "runtime_relative_root": "e46d/temporal-failure-audits", + "result_id_prefix": "e46d-temporal-failure-audit", + "document_name": "manifest.json", + "schema_version": "missioncore.e46d-temporal-failure-audit/v1" + } +} diff --git a/config/laboratories/e46e-ready-stack.json b/config/laboratories/e46e-ready-stack.json new file mode 100644 index 0000000..cffd768 --- /dev/null +++ b/config/laboratories/e46e-ready-stack.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e46e-ready-stack", + "evidence": { + "runtime_relative_root": "e46e/ready-stack-results", + "result_id_prefix": "e46e-ready-stack", + "document_name": "manifest.json", + "schema_version": "missioncore.e46e-ready-stack-result/v1" + } +} diff --git a/config/laboratories/e46f-dashcam-bakeoff.json b/config/laboratories/e46f-dashcam-bakeoff.json new file mode 100644 index 0000000..08fcb1a --- /dev/null +++ b/config/laboratories/e46f-dashcam-bakeoff.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e46f-dashcam-bakeoff", + "evidence": { + "runtime_relative_root": "e46f/dashcam-bakeoff-results", + "result_id_prefix": "e46f-dashcam-bakeoff", + "document_name": "manifest.json", + "schema_version": "missioncore.e46f-dashcam-bakeoff-result/v1" + } +} diff --git a/config/laboratories/e46g-rectified-detector-bakeoff.json b/config/laboratories/e46g-rectified-detector-bakeoff.json new file mode 100644 index 0000000..5a46814 --- /dev/null +++ b/config/laboratories/e46g-rectified-detector-bakeoff.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e46g-rectified-detector-bakeoff", + "evidence": { + "runtime_relative_root": "e46g/results", + "result_id_prefix": "e46g-rectified-detector-bakeoff", + "document_name": "manifest.json", + "schema_version": "missioncore.e46g-rectified-detector-bakeoff-result/v1" + } +} diff --git a/config/laboratories/e46h-full-rectified-front-replay.json b/config/laboratories/e46h-full-rectified-front-replay.json new file mode 100644 index 0000000..f13d09f --- /dev/null +++ b/config/laboratories/e46h-full-rectified-front-replay.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e46h-full-rectified-front-replay", + "evidence": { + "runtime_relative_root": "e46h/results", + "result_id_prefix": "e46h-full-rectified-front-replay", + "document_name": "manifest.json", + "schema_version": "missioncore.e46h-full-rectified-front-replay-result/v1" + } +} diff --git a/config/laboratories/e46i-grounding-dino-full-replay.json b/config/laboratories/e46i-grounding-dino-full-replay.json new file mode 100644 index 0000000..4a0883e --- /dev/null +++ b/config/laboratories/e46i-grounding-dino-full-replay.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e46i-grounding-dino-full-replay", + "evidence": { + "runtime_relative_root": "e46i/results", + "result_id_prefix": "e46i-grounding-dino-full-replay", + "document_name": "manifest.json", + "schema_version": "missioncore.e46i-grounding-dino-full-replay-result/v1" + } +} diff --git a/config/laboratories/e46j-raw-fisheye-realtime.json b/config/laboratories/e46j-raw-fisheye-realtime.json new file mode 100644 index 0000000..3fa1352 --- /dev/null +++ b/config/laboratories/e46j-raw-fisheye-realtime.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "e46j-raw-fisheye-realtime", + "evidence": { + "runtime_relative_root": "e46j/results", + "result_id_prefix": "e46j-raw-fisheye-realtime", + "document_name": "manifest.json", + "schema_version": "missioncore.e46j-raw-fisheye-realtime-result/v1" + } +} diff --git a/config/laboratories/l34-right-yolox-truth-island-freeze.json b/config/laboratories/l34-right-yolox-truth-island-freeze.json new file mode 100644 index 0000000..eb4a139 --- /dev/null +++ b/config/laboratories/l34-right-yolox-truth-island-freeze.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "l34-right-yolox-truth-island-freeze", + "evidence": { + "runtime_relative_root": "l3/right-yolox-truth-island-freeze", + "result_id_prefix": "l34-right-yolox-truth-island-freeze", + "document_name": "manifest.json", + "schema_version": "missioncore.l34-right-yolox-truth-island-freeze/v1" + } +} diff --git a/config/laboratories/l34a-assisted-yolox-error-audit.json b/config/laboratories/l34a-assisted-yolox-error-audit.json new file mode 100644 index 0000000..37bc9cd --- /dev/null +++ b/config/laboratories/l34a-assisted-yolox-error-audit.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "l34a-assisted-yolox-error-audit", + "evidence": { + "runtime_relative_root": "l3/assisted-yolox-error-audits", + "result_id_prefix": "l34a-assisted-yolox-error-audit", + "document_name": "manifest.json", + "schema_version": "missioncore.l34a-assisted-yolox-error-audit/v1" + } +} diff --git a/config/laboratories/l34b-nested-box-consolidation-shadow.json b/config/laboratories/l34b-nested-box-consolidation-shadow.json new file mode 100644 index 0000000..83d52b8 --- /dev/null +++ b/config/laboratories/l34b-nested-box-consolidation-shadow.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "l34b-nested-box-consolidation-shadow", + "evidence": { + "runtime_relative_root": "l3/nested-box-consolidation-shadows", + "result_id_prefix": "l34b-nested-box-consolidation-shadow", + "document_name": "manifest.json", + "schema_version": "missioncore.l34b-nested-box-consolidation-shadow/v1" + } +} diff --git a/config/laboratories/l34c-tile-seam-stitch-shadow.json b/config/laboratories/l34c-tile-seam-stitch-shadow.json new file mode 100644 index 0000000..c402fad --- /dev/null +++ b/config/laboratories/l34c-tile-seam-stitch-shadow.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "l34c-tile-seam-stitch-shadow", + "evidence": { + "runtime_relative_root": "l3/tile-seam-stitch-shadows", + "result_id_prefix": "l34c-tile-seam-stitch-shadow", + "document_name": "manifest.json", + "schema_version": "missioncore.l34c-tile-seam-stitch-shadow/v1" + } +} diff --git a/config/laboratories/l34d-cumulative-postprocessing-candidate.json b/config/laboratories/l34d-cumulative-postprocessing-candidate.json new file mode 100644 index 0000000..7a1131c --- /dev/null +++ b/config/laboratories/l34d-cumulative-postprocessing-candidate.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "l34d-cumulative-postprocessing-candidate", + "evidence": { + "runtime_relative_root": "l3/cumulative-postprocessing-candidates", + "result_id_prefix": "l34d-cumulative-postprocessing-candidate", + "document_name": "manifest.json", + "schema_version": "missioncore.l34d-cumulative-postprocessing-candidate/v1" + } +} diff --git a/config/laboratories/l34e-self-review-diagnostic.json b/config/laboratories/l34e-self-review-diagnostic.json new file mode 100644 index 0000000..1b6fbfb --- /dev/null +++ b/config/laboratories/l34e-self-review-diagnostic.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "l34e-self-review-diagnostic", + "evidence": { + "runtime_relative_root": "l3/self-review-diagnostics", + "result_id_prefix": "l34e-self-review-diagnostic", + "document_name": "manifest.json", + "schema_version": "missioncore.l34e-self-review-diagnostic/v1" + } +} diff --git a/config/laboratories/l34f-adjudicated-reference.json b/config/laboratories/l34f-adjudicated-reference.json new file mode 100644 index 0000000..53228f8 --- /dev/null +++ b/config/laboratories/l34f-adjudicated-reference.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "l34f-adjudicated-reference", + "evidence": { + "runtime_relative_root": "l3/adjudicated-references", + "result_id_prefix": "l34f-adjudicated-reference", + "document_name": "manifest.json", + "schema_version": "missioncore.l34f-adjudicated-reference/v1" + } +} diff --git a/deploy/telemetry-plane/README.md b/deploy/telemetry-plane/README.md index ceeaf3d..1c01af1 100644 --- a/deploy/telemetry-plane/README.md +++ b/deploy/telemetry-plane/README.md @@ -115,6 +115,12 @@ configuration and rolls back if the service does not return to `Running`. MQTT credentials are scoped to the service environment and must not be passed on a command line or stored in the repository. +Both install and update configure Windows Service Control Manager recovery for +Telegraf: restart after 5 seconds, then 30 seconds, then 60 seconds, with the failure +counter reset after one day. The compute-contour worker network transition applies +the same policy to already-provisioned agents and fails closed unless the worker +confirms it. + The same host service reads the existing perception worker's loopback `/health` contract with `Get-NdcMissionCorePipelineTelemetry.ps1` and publishes nine stage-keyed snapshots to the contour's `pipeline` topic. The perception container diff --git a/deploy/telemetry-plane/telegraf/Install-NdcMissionCoreTelegraf.ps1 b/deploy/telemetry-plane/telegraf/Install-NdcMissionCoreTelegraf.ps1 index febd23b..2bb1145 100644 --- a/deploy/telemetry-plane/telegraf/Install-NdcMissionCoreTelegraf.ps1 +++ b/deploy/telemetry-plane/telegraf/Install-NdcMissionCoreTelegraf.ps1 @@ -128,6 +128,15 @@ try { throw "Failed to apply the NDC service display name" } Set-Service -Name $serviceName -StartupType Automatic + & sc.exe failure $serviceName reset= 86400 ` + actions= restart/5000/restart/30000/restart/60000 | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Failed to configure Telegraf service recovery actions" + } + & sc.exe failureflag $serviceName 1 | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Failed to enable Telegraf recovery for non-crash failures" + } Start-Service -Name $serviceName $service = Get-Service -Name $serviceName $service.WaitForStatus([ServiceProcess.ServiceControllerStatus]::Running, [TimeSpan]::FromSeconds(20)) @@ -140,6 +149,7 @@ try { DisplayName = $service.DisplayName Status = $service.Status.ToString() StartType = $service.StartType.ToString() + RecoveryConfigured = $true Configuration = $configurationPath PipelineCollector = $collectorPath PipelineJournal = $PipelineJournal diff --git a/deploy/telemetry-plane/telegraf/Update-NdcMissionCoreTelegraf.ps1 b/deploy/telemetry-plane/telegraf/Update-NdcMissionCoreTelegraf.ps1 index 3788892..67949bc 100644 --- a/deploy/telemetry-plane/telegraf/Update-NdcMissionCoreTelegraf.ps1 +++ b/deploy/telemetry-plane/telegraf/Update-NdcMissionCoreTelegraf.ps1 @@ -118,8 +118,27 @@ try { [ServiceProcess.ServiceControllerStatus]::Running, [TimeSpan]::FromSeconds(20) ) + & sc.exe failure $serviceName reset= 86400 ` + actions= restart/5000/restart/30000/restart/60000 | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Failed to configure Telegraf service recovery actions" + } + & sc.exe failureflag $serviceName 1 | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Failed to enable Telegraf recovery for non-crash failures" + } } catch { + Stop-Service -Name $serviceName -Force -ErrorAction SilentlyContinue + $service = Get-Service -Name $serviceName + if ($service.Status -ne [ServiceProcess.ServiceControllerStatus]::Stopped) { + $service.WaitForStatus( + [ServiceProcess.ServiceControllerStatus]::Stopped, + [TimeSpan]::FromSeconds(20) + ) + } + Set-ItemProperty -Path $serviceRegistryPath -Name Environment ` + -Type MultiString -Value $serviceEnvironmentBefore Copy-Item -LiteralPath $backupPath -Destination $configurationPath if ($hadCollector) { Copy-Item -LiteralPath $backupCollectorPath -Destination $collectorPath @@ -135,6 +154,7 @@ try { SchemaVersion = "missioncore.telemetry-agent-config-result/v1" ServiceName = $serviceName Status = (Get-Service -Name $serviceName).Status.ToString() + RecoveryConfigured = $true Configuration = $configurationPath Backup = $backupPath PipelineCollector = $collectorPath diff --git a/docs/adr/0031-local-compute-contour-telemetry-plane.md b/docs/adr/0031-local-compute-contour-telemetry-plane.md index 1b6be85..3dfa67b 100644 --- a/docs/adr/0031-local-compute-contour-telemetry-plane.md +++ b/docs/adr/0031-local-compute-contour-telemetry-plane.md @@ -16,10 +16,9 @@ The first local contour is active: - Mission Core on canonical port `8000` uses this adapter for live System telemetry and reports the source as `agent-mqtt`; - SSH is retained only for explicit profile diagnostics and bootstrap checks. -- the selected contour owns a managed network profile: Worker 006 uses - `desktop-opj8j04.local`, resolves the broker through the operator Mac's stable - `.local` name, and keeps the current private Docker bind address as a separate - reviewed value; +- the selected contour owns a managed network profile: Worker 006 keeps its node + identity separate from its SSH address and from the broker endpoint; every saved + endpoint and private Docker bind address is probed before an explicit transition; - the existing contour settings can probe DNS/TCP from both Mac and Worker, apply the listener address to the local broker, and apply the endpoint to the Telegraf Windows service without exposing or replacing its scoped credential. @@ -81,11 +80,27 @@ The direct SSH/PowerShell probe remains a bounded bootstrap and diagnostic fallb the existing Worker 006 profile. It is not the target live telemetry transport. Mesh access points belonging to one LAN are not modeled as separate broker addresses. -Roaming keeps the stable hostname contract. A move to another LAN changes the reviewed -private bind address, not the agent identity or MQTT topic identity. Applying a saved -profile is explicit, verifies DNS and TCP reachability, and rolls back the affected -configuration when verification fails. MQTT without TLS cannot be configured for a -public endpoint or wildcard listener. +The broker endpoint may be a hostname only when the Worker can resolve it; Mission Core +does not assume that macOS `.local` discovery is portable to Windows. A move to another +LAN changes the reviewed endpoint and private bind address, not the agent identity or +MQTT topic identity. Applying a saved profile is explicit, verifies DNS and TCP +reachability, and rolls back the affected configuration when verification fails. MQTT +without TLS cannot be configured for a public endpoint or wildcard listener. + +## 2026-08-05 recovery correction + +Worker 006 stopped publishing because its Telegraf MQTT output still referenced the +operator Mac through a `.local` name that Windows no longer resolved. Telegraf exited +on the output connection error, and the Windows service had no configured recovery +actions. The reviewed compute-contour transition rebound the broker and worker profile +to the currently verified private LAN endpoint without replacing the scoped MQTT +credential; fresh agent, hardware and pipeline telemetry then returned. + +Install, update and existing-agent network transitions now configure the same bounded +Windows recovery policy: restart after 5 seconds, 30 seconds and 60 seconds, resetting +the failure counter after one day and applying recovery to non-crash failures. This +does not make an invalid endpoint valid; it keeps transient broker or network loss from +silently leaving the telemetry agent stopped after connectivity returns. ## Why these components diff --git a/docs/adr/0036-configurative-laboratory-evidence-registry.md b/docs/adr/0036-configurative-laboratory-evidence-registry.md new file mode 100644 index 0000000..f3ff618 --- /dev/null +++ b/docs/adr/0036-configurative-laboratory-evidence-registry.md @@ -0,0 +1,67 @@ +# ADR 0036: Configurative laboratory evidence registry + +Date: 2026-08-05 +Status: accepted and implemented as the first migration slice + +## Context + +The advanced LAB index grew by adding one result-id regular expression, one root +provider argument, one application wiring block and one tuple for every experiment. +That made the Mission Core control plane structurally dependent on an expanding list +of experiment names. It also made a new frozen evidence source look like a new product +runtime pipeline even when the UI only needed read-only result discovery. + +The target product architecture has a stricter dependency direction: + +```text +contracts -> providers -> graphs -> experiments +``` + +Mission Core owns configurations, runs, evidence, health, telemetry and operator +control. Experiment code may invoke production graphs and freeze evidence, but the +core must not import an experiment to discover that evidence. + +## Decision + +Advanced LAB evidence sources are declared as one versioned JSON definition per +`work_id` under `config/laboratories/`. Each definition contains only: + +- a repository-runtime-relative evidence root; +- a digest-bound result-id prefix; +- the bounded identity document name; +- the exact identity-document schema version. + +`LaboratoryEvidenceRegistry` discovers these files without importing experiment code. +It rejects unknown keys, unsafe or non-normalized paths, symlinked definitions, +oversized files, invalid identifiers and duplicate work IDs, roots or result prefixes. +The existing advanced index keeps its previous fail-closed evidence validation: +identity digest, schema, creation time and read-only/no-command authority must still +match before a result is published. + +The production advanced index now obtains all 27 standard evidence sources through +this registry. Adding another standard source therefore adds data, not another router +parameter or application wiring branch. + +## Explicit non-goals + +This first slice configures evidence discovery; it does not claim that the historical +experiment implementations are already reusable production graphs. E31-E40 retain +specialized read-model projectors behind their current adapters while those payloads +are converted to canonical contracts. L3/L31/L32/L33 retain bounded legacy identity +adapters because their stored evidence does not yet use the common identity document. + +The registry does not select detectors, trackers, planners or command paths and cannot +grant navigation or safety authority. Future executable LAB definitions must reference +versioned provider/graph contracts; they must not contain arbitrary Python imports or +shell commands. + +## Consequences + +- LAB growth no longer expands the product composition root for standard evidence. +- Evidence location and identity contracts are reviewable, diffable configuration. +- A malformed or ambiguous definition prevents startup instead of silently exposing + the wrong evidence. +- The remaining manual frontend fetch/render map and specialized backend projectors + are visible migration debt, not the intended extension mechanism. +- The next architecture slice can add provider and graph registries against the + canonical perception contracts without making experiments dependencies of core. diff --git a/src/k1link/laboratory/__init__.py b/src/k1link/laboratory/__init__.py new file mode 100644 index 0000000..51498c2 --- /dev/null +++ b/src/k1link/laboratory/__init__.py @@ -0,0 +1,15 @@ +"""Configuration contracts for Mission Core laboratory evidence.""" + +from k1link.laboratory.evidence_registry import ( + LABORATORY_EVIDENCE_DEFINITION_SCHEMA, + LaboratoryEvidenceDefinition, + LaboratoryEvidenceRegistry, + LaboratoryRegistryError, +) + +__all__ = [ + "LABORATORY_EVIDENCE_DEFINITION_SCHEMA", + "LaboratoryEvidenceDefinition", + "LaboratoryEvidenceRegistry", + "LaboratoryRegistryError", +] diff --git a/src/k1link/laboratory/evidence_registry.py b/src/k1link/laboratory/evidence_registry.py new file mode 100644 index 0000000..e175a32 --- /dev/null +++ b/src/k1link/laboratory/evidence_registry.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Final + +LABORATORY_EVIDENCE_DEFINITION_SCHEMA: Final = "missioncore.laboratory-evidence-definition/v1" +_DEFINITION_MAX_BYTES: Final = 16 * 1024 +_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$") +_SCHEMA_VERSION = re.compile(r"^missioncore\.[a-z0-9.-]+/v[1-9][0-9]*$") +_TOP_LEVEL_KEYS: Final = frozenset({"schema_version", "work_id", "evidence"}) +_EVIDENCE_KEYS: Final = frozenset( + {"runtime_relative_root", "result_id_prefix", "document_name", "schema_version"} +) + + +class LaboratoryRegistryError(ValueError): + """Raised when a LAB evidence definition is unsafe or ambiguous.""" + + +@dataclass(frozen=True, slots=True) +class LaboratoryEvidenceDefinition: + work_id: str + runtime_relative_root: PurePosixPath + result_id_prefix: str + document_name: str + result_schema_version: str + + def __post_init__(self) -> None: + _identifier(self.work_id, "work_id") + _identifier(self.result_id_prefix, "result_id_prefix") + _document_name(self.document_name) + _schema_version(self.result_schema_version) + if not isinstance(self.runtime_relative_root, PurePosixPath): + raise LaboratoryRegistryError("runtime_relative_root must be a POSIX path") + _relative_root(str(self.runtime_relative_root)) + + @property + def result_id_pattern(self) -> re.Pattern[str]: + return re.compile(rf"^{re.escape(self.result_id_prefix)}-[a-f0-9]{{64}}$") + + def result_root(self, runtime_root: Path) -> Path: + return runtime_root.joinpath(*self.runtime_relative_root.parts) + + +@dataclass(frozen=True, slots=True) +class LaboratoryEvidenceRegistry: + definitions: tuple[LaboratoryEvidenceDefinition, ...] + + def __post_init__(self) -> None: + if not isinstance(self.definitions, tuple) or not all( + isinstance(definition, LaboratoryEvidenceDefinition) for definition in self.definitions + ): + raise LaboratoryRegistryError("LAB definitions must be an immutable tuple") + _reject_duplicates(self.definitions) + + @classmethod + def from_directory(cls, root: Path) -> LaboratoryEvidenceRegistry: + definition_root = _real_directory(root, "LAB definition root") + definitions = tuple( + _read_definition(path) + for path in sorted(definition_root.glob("*.json"), key=lambda item: item.name) + ) + return cls(definitions=definitions) + + +def _real_directory(path: Path, label: str) -> Path: + candidate = path.expanduser().absolute() + if candidate.is_symlink(): + raise LaboratoryRegistryError(f"{label} must not be a symlink") + try: + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise LaboratoryRegistryError(f"{label} does not exist") from exc + if not resolved.is_dir(): + raise LaboratoryRegistryError(f"{label} must be a directory") + return resolved + + +def _read_definition(path: Path) -> LaboratoryEvidenceDefinition: + if path.is_symlink() or not path.is_file(): + raise LaboratoryRegistryError(f"LAB definition must be a regular file: {path.name}") + if path.stat().st_size > _DEFINITION_MAX_BYTES: + raise LaboratoryRegistryError(f"LAB definition is too large: {path.name}") + try: + payload: object = json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + raise LaboratoryRegistryError(f"LAB definition is unreadable: {path.name}") from exc + document = _object(payload, f"LAB definition {path.name}") + _exact_keys(document, _TOP_LEVEL_KEYS, f"LAB definition {path.name}") + if document["schema_version"] != LABORATORY_EVIDENCE_DEFINITION_SCHEMA: + raise LaboratoryRegistryError(f"LAB definition schema is invalid: {path.name}") + work_id = _identifier(document["work_id"], "work_id") + if path.name != f"{work_id}.json": + raise LaboratoryRegistryError(f"LAB definition filename must match work_id: {path.name}") + evidence = _object(document["evidence"], f"LAB evidence {work_id}") + _exact_keys(evidence, _EVIDENCE_KEYS, f"LAB evidence {work_id}") + result_id_prefix = _identifier(evidence["result_id_prefix"], "result_id_prefix") + document_name = _document_name(evidence["document_name"]) + result_schema_version = _schema_version(evidence["schema_version"]) + return LaboratoryEvidenceDefinition( + work_id=work_id, + runtime_relative_root=_relative_root(evidence["runtime_relative_root"]), + result_id_prefix=result_id_prefix, + document_name=document_name, + result_schema_version=result_schema_version, + ) + + +def _object(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise LaboratoryRegistryError(f"{label} must be an object") + return value + + +def _exact_keys(document: dict[str, object], expected: frozenset[str], label: str) -> None: + actual = frozenset(document) + if actual != expected: + missing = sorted(expected - actual) + unexpected = sorted(actual - expected) + raise LaboratoryRegistryError( + f"{label} keys are invalid; missing={missing}, unexpected={unexpected}" + ) + + +def _text(value: object, label: str) -> str: + if not isinstance(value, str) or not value.strip() or value != value.strip(): + raise LaboratoryRegistryError(f"{label} must be a non-empty trimmed string") + return value + + +def _identifier(value: object, label: str) -> str: + text = _text(value, label) + if _IDENTIFIER.fullmatch(text) is None: + raise LaboratoryRegistryError(f"{label} is invalid") + return text + + +def _schema_version(value: object) -> str: + text = _text(value, "evidence schema_version") + if _SCHEMA_VERSION.fullmatch(text) is None: + raise LaboratoryRegistryError("evidence schema_version is invalid") + return text + + +def _document_name(value: object) -> str: + text = _text(value, "document_name") + candidate = PurePosixPath(text) + if ( + candidate.is_absolute() + or len(candidate.parts) != 1 + or candidate.name != text + or candidate.suffix != ".json" + ): + raise LaboratoryRegistryError("document_name must be one JSON filename") + return text + + +def _relative_root(value: object) -> PurePosixPath: + text = _text(value, "runtime_relative_root") + if "\\" in text: + raise LaboratoryRegistryError("runtime_relative_root must use POSIX separators") + candidate = PurePosixPath(text) + if ( + candidate.is_absolute() + or not candidate.parts + or text == "." + or str(candidate) != text + or any(part in {"", ".", ".."} for part in candidate.parts) + ): + raise LaboratoryRegistryError("runtime_relative_root must be a normalized relative path") + return candidate + + +def _reject_duplicates(definitions: tuple[LaboratoryEvidenceDefinition, ...]) -> None: + dimensions = { + "work_id": [definition.work_id for definition in definitions], + "result_id_prefix": [definition.result_id_prefix for definition in definitions], + "runtime_relative_root": [ + str(definition.runtime_relative_root) for definition in definitions + ], + } + for label, values in dimensions.items(): + if len(values) != len(set(values)): + raise LaboratoryRegistryError(f"duplicate LAB {label}") diff --git a/src/k1link/web/advanced_laboratory_api.py b/src/k1link/web/advanced_laboratory_api.py index acbcc56..8d66cf7 100644 --- a/src/k1link/web/advanced_laboratory_api.py +++ b/src/k1link/web/advanced_laboratory_api.py @@ -55,6 +55,7 @@ from k1link.compute.e40_perception_product_gate import ( E40PerceptionProductGateError, read_e40_perception_product_gate, ) +from k1link.laboratory import LaboratoryEvidenceDefinition, LaboratoryEvidenceRegistry from k1link.web.l3_pointpillars_visual_api import latest_l3_visual_identity from k1link.web.l32_pointpillars_camera_review_api import latest_l32_identity from k1link.web.l33_camera_first_detector_review_api import latest_l33_identity @@ -72,24 +73,6 @@ _E37_RESULT_ID = re.compile(r"^e37-ravnoves-acceptance-[a-f0-9]{64}$") _E38_RESULT_ID = re.compile(r"^e38-perception-baseline-[a-f0-9]{64}$") _E39_RESULT_ID = re.compile(r"^e39-perception-refinement-[a-f0-9]{64}$") _E40_RESULT_ID = re.compile(r"^e40-perception-product-gate-[a-f0-9]{64}$") -_E46_RESULT_ID = re.compile(r"^e46-detector-truth-island-[a-f0-9]{64}$") -_E46A_RESULT_ID = re.compile(r"^e46a-ai-engineering-preannotation-[a-f0-9]{64}$") -_E46B_RESULT_ID = re.compile(r"^e46b-temporal-motion-[a-f0-9]{64}$") -_E46C_RESULT_ID = re.compile(r"^e46c-full-replay-world-tracks-[a-f0-9]{64}$") -_E46D_RESULT_ID = re.compile(r"^e46d-temporal-failure-audit-[a-f0-9]{64}$") -_E46E_RESULT_ID = re.compile(r"^e46e-ready-stack-[a-f0-9]{64}$") -_E46F_RESULT_ID = re.compile(r"^e46f-dashcam-bakeoff-[a-f0-9]{64}$") -_E46G_RESULT_ID = re.compile(r"^e46g-rectified-detector-bakeoff-[a-f0-9]{64}$") -_E46H_RESULT_ID = re.compile(r"^e46h-full-rectified-front-replay-[a-f0-9]{64}$") -_E46I_RESULT_ID = re.compile(r"^e46i-grounding-dino-full-replay-[a-f0-9]{64}$") -_E46J_RESULT_ID = re.compile(r"^e46j-raw-fisheye-realtime-[a-f0-9]{64}$") -_L34_RESULT_ID = re.compile(r"^l34-right-yolox-truth-island-freeze-[a-f0-9]{64}$") -_L34A_RESULT_ID = re.compile(r"^l34a-assisted-yolox-error-audit-[a-f0-9]{64}$") -_L34B_RESULT_ID = re.compile(r"^l34b-nested-box-consolidation-shadow-[a-f0-9]{64}$") -_L34C_RESULT_ID = re.compile(r"^l34c-tile-seam-stitch-shadow-[a-f0-9]{64}$") -_L34D_RESULT_ID = re.compile(r"^l34d-cumulative-postprocessing-candidate-[a-f0-9]{64}$") -_L34E_RESULT_ID = re.compile(r"^l34e-self-review-diagnostic-[a-f0-9]{64}$") -_L34F_RESULT_ID = re.compile(r"^l34f-adjudicated-reference-[a-f0-9]{64}$") RootProvider = Callable[[], Path | None] @@ -299,6 +282,50 @@ def _advanced_index( } +def _registry_index_specs( + registry: LaboratoryEvidenceRegistry, + runtime_root_provider: RootProvider, +) -> tuple[_AdvancedIndexSpec, ...]: + return tuple( + ( + definition.work_id, + _evidence_root_provider(definition, runtime_root_provider), + definition.result_id_pattern, + definition.document_name, + definition.result_schema_version, + ) + for definition in registry.definitions + ) + + +def _evidence_root_provider( + definition: LaboratoryEvidenceDefinition, + runtime_root_provider: RootProvider, +) -> RootProvider: + def result_root_provider() -> Path | None: + configured_runtime_root = runtime_root_provider() + if configured_runtime_root is None: + return None + runtime_candidate = configured_runtime_root.expanduser().absolute() + if runtime_candidate.is_symlink(): + return None + try: + runtime_root = runtime_candidate.resolve(strict=True) + except OSError: + return None + candidate = definition.result_root(runtime_root) + if candidate.exists(): + try: + resolved = candidate.resolve(strict=True) + except OSError: + return None + if resolved != candidate.absolute() or not resolved.is_relative_to(runtime_root): + return None + return candidate + + return result_root_provider + + def _object(value: object, label: str) -> dict[str, Any]: if not isinstance(value, dict): raise ValueError(f"{label} is invalid") @@ -786,6 +813,8 @@ def _empty_catalog(configured: bool) -> dict[str, object]: def build_advanced_laboratory_router( *, + evidence_registry: LaboratoryEvidenceRegistry | None = None, + evidence_runtime_root_provider: RootProvider = lambda: None, e31_root_provider: RootProvider = lambda: None, e32_root_provider: RootProvider = lambda: None, e33_root_provider: RootProvider = lambda: None, @@ -795,24 +824,6 @@ def build_advanced_laboratory_router( e38_root_provider: RootProvider = lambda: None, e39_root_provider: RootProvider = lambda: None, e40_root_provider: RootProvider = lambda: None, - e46_root_provider: RootProvider = lambda: None, - e46a_root_provider: RootProvider = lambda: None, - e46b_root_provider: RootProvider = lambda: None, - e46c_root_provider: RootProvider = lambda: None, - e46d_root_provider: RootProvider = lambda: None, - e46e_root_provider: RootProvider = lambda: None, - e46f_root_provider: RootProvider = lambda: None, - e46g_root_provider: RootProvider = lambda: None, - e46h_root_provider: RootProvider = lambda: None, - e46i_root_provider: RootProvider = lambda: None, - e46j_root_provider: RootProvider = lambda: None, - l34_root_provider: RootProvider = lambda: None, - l34a_root_provider: RootProvider = lambda: None, - l34b_root_provider: RootProvider = lambda: None, - l34c_root_provider: RootProvider = lambda: None, - l34d_root_provider: RootProvider = lambda: None, - l34e_root_provider: RootProvider = lambda: None, - l34f_root_provider: RootProvider = lambda: None, l3_visual_root_provider: RootProvider = lambda: None, l31_ravnoves_root_provider: RootProvider = lambda: None, l32_camera_review_root_provider: RootProvider = lambda: None, @@ -822,8 +833,8 @@ def build_advanced_laboratory_router( @router.get("/advanced-index") def list_advanced_results() -> dict[str, object]: - result = _advanced_index( - ( + if evidence_registry is None: + specs: tuple[_AdvancedIndexSpec, ...] = ( ( "e31-source-binding", e31_root_provider, @@ -887,134 +898,13 @@ def build_advanced_laboratory_router( "manifest.json", "missioncore.e40-perception-product-gate/v1", ), - ( - "e46-detector-truth-island", - e46_root_provider, - _E46_RESULT_ID, - "manifest.json", - "missioncore.e46-detector-truth-island/v1", - ), - ( - "e46a-ai-engineering-preannotation", - e46a_root_provider, - _E46A_RESULT_ID, - "manifest.json", - "missioncore.e46a-ai-engineering-preannotation/v1", - ), - ( - "e46b-temporal-motion", - e46b_root_provider, - _E46B_RESULT_ID, - "manifest.json", - "missioncore.e46b-temporal-motion/v1", - ), - ( - "e46c-full-replay-world-tracks", - e46c_root_provider, - _E46C_RESULT_ID, - "manifest.json", - "missioncore.e46c-full-replay-world-tracks/v1", - ), - ( - "e46d-temporal-failure-audit", - e46d_root_provider, - _E46D_RESULT_ID, - "manifest.json", - "missioncore.e46d-temporal-failure-audit/v1", - ), - ( - "e46e-ready-stack", - e46e_root_provider, - _E46E_RESULT_ID, - "manifest.json", - "missioncore.e46e-ready-stack-result/v1", - ), - ( - "e46f-dashcam-bakeoff", - e46f_root_provider, - _E46F_RESULT_ID, - "manifest.json", - "missioncore.e46f-dashcam-bakeoff-result/v1", - ), - ( - "e46g-rectified-detector-bakeoff", - e46g_root_provider, - _E46G_RESULT_ID, - "manifest.json", - "missioncore.e46g-rectified-detector-bakeoff-result/v1", - ), - ( - "e46h-full-rectified-front-replay", - e46h_root_provider, - _E46H_RESULT_ID, - "manifest.json", - "missioncore.e46h-full-rectified-front-replay-result/v1", - ), - ( - "e46i-grounding-dino-full-replay", - e46i_root_provider, - _E46I_RESULT_ID, - "manifest.json", - "missioncore.e46i-grounding-dino-full-replay-result/v1", - ), - ( - "e46j-raw-fisheye-realtime", - e46j_root_provider, - _E46J_RESULT_ID, - "manifest.json", - "missioncore.e46j-raw-fisheye-realtime-result/v1", - ), - ( - "l34-right-yolox-truth-island-freeze", - l34_root_provider, - _L34_RESULT_ID, - "manifest.json", - "missioncore.l34-right-yolox-truth-island-freeze/v1", - ), - ( - "l34a-assisted-yolox-error-audit", - l34a_root_provider, - _L34A_RESULT_ID, - "manifest.json", - "missioncore.l34a-assisted-yolox-error-audit/v1", - ), - ( - "l34b-nested-box-consolidation-shadow", - l34b_root_provider, - _L34B_RESULT_ID, - "manifest.json", - "missioncore.l34b-nested-box-consolidation-shadow/v1", - ), - ( - "l34c-tile-seam-stitch-shadow", - l34c_root_provider, - _L34C_RESULT_ID, - "manifest.json", - "missioncore.l34c-tile-seam-stitch-shadow/v1", - ), - ( - "l34d-cumulative-postprocessing-candidate", - l34d_root_provider, - _L34D_RESULT_ID, - "manifest.json", - "missioncore.l34d-cumulative-postprocessing-candidate/v1", - ), - ( - "l34e-self-review-diagnostic", - l34e_root_provider, - _L34E_RESULT_ID, - "manifest.json", - "missioncore.l34e-self-review-diagnostic/v1", - ), - ( - "l34f-adjudicated-reference", - l34f_root_provider, - _L34F_RESULT_ID, - "manifest.json", - "missioncore.l34f-adjudicated-reference/v1", - ), ) - ) + else: + specs = _registry_index_specs( + evidence_registry, + evidence_runtime_root_provider, + ) + result = _advanced_index(specs) items = result["items"] if not isinstance(items, list): raise RuntimeError("advanced LAB index items are invalid") diff --git a/src/k1link/web/app.py b/src/k1link/web/app.py index 0178e73..40cf13a 100644 --- a/src/k1link/web/app.py +++ b/src/k1link/web/app.py @@ -23,6 +23,7 @@ from k1link.compute import ( RecordedPerceptionOverlayMux, RecordedPerceptionOverlayStore, ) +from k1link.laboratory import LaboratoryEvidenceRegistry from k1link.sessions import ( MaterializedRecording, RecordedMediaInspector, @@ -133,6 +134,9 @@ from k1link.web.viewer_diagnostics_api import build_viewer_diagnostics_router REPOSITORY_ROOT = Path(__file__).resolve().parents[3] INVALID_REQUEST_DETAIL = "Некорректные параметры запроса." +LABORATORY_EVIDENCE_REGISTRY = LaboratoryEvidenceRegistry.from_directory( + REPOSITORY_ROOT / "config" / "laboratories" +) def _resolve_media_tool(name: str) -> Path | None: @@ -557,6 +561,10 @@ app.include_router( ) app.include_router( build_advanced_laboratory_router( + evidence_registry=LABORATORY_EVIDENCE_REGISTRY, + evidence_runtime_root_provider=lambda: ( + REPOSITORY_ROOT / ".runtime" / "compute-experiments" + ), e31_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e31" / "source-qualifications" ), @@ -584,96 +592,6 @@ app.include_router( e40_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e40" / "results" ), - e46_root_provider=lambda: ( - REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46" / "results" - ), - e46a_root_provider=lambda: ( - REPOSITORY_ROOT - / ".runtime" - / "compute-experiments" - / "e46a" - / "ai-engineering-preannotations" - ), - e46b_root_provider=lambda: ( - REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46b" / "temporal-motion" - ), - e46c_root_provider=lambda: ( - REPOSITORY_ROOT - / ".runtime" - / "compute-experiments" - / "e46c" - / "full-replay-world-tracks" - ), - e46d_root_provider=lambda: ( - REPOSITORY_ROOT - / ".runtime" - / "compute-experiments" - / "e46d" - / "temporal-failure-audits" - ), - e46e_root_provider=lambda: ( - REPOSITORY_ROOT - / ".runtime" - / "compute-experiments" - / "e46e" - / "ready-stack-results" - ), - e46f_root_provider=lambda: ( - REPOSITORY_ROOT - / ".runtime" - / "compute-experiments" - / "e46f" - / "dashcam-bakeoff-results" - ), - e46g_root_provider=lambda: ( - REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46g" / "results" - ), - e46h_root_provider=lambda: ( - REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46h" / "results" - ), - e46i_root_provider=lambda: ( - REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46i" / "results" - ), - e46j_root_provider=lambda: ( - REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e46j" / "results" - ), - l34_root_provider=lambda: ( - REPOSITORY_ROOT - / ".runtime" - / "compute-experiments" - / "l3" - / "right-yolox-truth-island-freeze" - ), - l34a_root_provider=lambda: ( - REPOSITORY_ROOT - / ".runtime" - / "compute-experiments" - / "l3" - / "assisted-yolox-error-audits" - ), - l34b_root_provider=lambda: ( - REPOSITORY_ROOT - / ".runtime" - / "compute-experiments" - / "l3" - / "nested-box-consolidation-shadows" - ), - l34c_root_provider=lambda: ( - REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "tile-seam-stitch-shadows" - ), - l34d_root_provider=lambda: ( - REPOSITORY_ROOT - / ".runtime" - / "compute-experiments" - / "l3" - / "cumulative-postprocessing-candidates" - ), - l34e_root_provider=lambda: ( - REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "self-review-diagnostics" - ), - l34f_root_provider=lambda: ( - REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "adjudicated-references" - ), l3_visual_root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "l3" / "visual-audits" ), diff --git a/src/k1link/web/compute_contour_network.py b/src/k1link/web/compute_contour_network.py index 9c96c12..0c21ca0 100644 --- a/src/k1link/web/compute_contour_network.py +++ b/src/k1link/web/compute_contour_network.py @@ -313,6 +313,15 @@ elseif ((Get-Service -Name "telegraf").Status -ne "Running") { [TimeSpan]::FromSeconds(20) ) } +& sc.exe failure telegraf reset= 86400 ` + actions= restart/5000/restart/30000/restart/60000 | Out-Null +if ($LASTEXITCODE -ne 0) { + throw "Failed to configure Telegraf service recovery actions" +} +& sc.exe failureflag telegraf 1 | Out-Null +if ($LASTEXITCODE -ne 0) { + throw "Failed to enable Telegraf recovery for non-crash failures" +} $resolved = @( [Net.Dns]::GetHostAddresses([string]$payload.mqtt_host) | ForEach-Object { $_.IPAddressToString } | @@ -326,6 +335,7 @@ $resolved = @( mqtt_port = [int]$payload.mqtt_port resolved_addresses = $resolved broker_reachable = $true + recovery_configured = $true changed = $changed } | ConvertTo-Json -Depth 5 -Compress """.strip() @@ -587,6 +597,7 @@ def apply_worker_network( or document.get("mqtt_port") != target.mqtt_port or document.get("service_status") != "Running" or document.get("broker_reachable") is not True + or document.get("recovery_configured") is not True ): raise NetworkOperationError( "Worker не подтвердил применённый сетевой профиль." diff --git a/tests/test_advanced_laboratory_api.py b/tests/test_advanced_laboratory_api.py index 8932f2b..d323878 100644 --- a/tests/test_advanced_laboratory_api.py +++ b/tests/test_advanced_laboratory_api.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from pathlib import Path +from pathlib import Path, PurePosixPath from types import SimpleNamespace from fastapi import APIRouter @@ -9,6 +9,7 @@ from fastapi.routing import APIRoute from pytest import MonkeyPatch import k1link.web.advanced_laboratory_api as advanced_api +from k1link.laboratory import LaboratoryEvidenceDefinition, LaboratoryEvidenceRegistry from k1link.web.advanced_laboratory_api import build_advanced_laboratory_router @@ -24,6 +25,26 @@ def _endpoint(router: APIRouter, path: str) -> object: raise AssertionError(f"GET {path} route is missing") +def _evidence_registry( + root: Path, + *, + work_id: str, + result_id_prefix: str, + schema_version: str, +) -> LaboratoryEvidenceRegistry: + return LaboratoryEvidenceRegistry( + definitions=( + LaboratoryEvidenceDefinition( + work_id=work_id, + runtime_relative_root=PurePosixPath(root.name), + result_id_prefix=result_id_prefix, + document_name="manifest.json", + result_schema_version=schema_version, + ), + ) + ) + + def test_advanced_catalogs_are_empty_when_not_configured() -> None: router = build_advanced_laboratory_router() @@ -53,6 +74,26 @@ def test_advanced_index_is_empty_when_not_configured() -> None: } +def test_registry_index_does_not_follow_a_runtime_symlink(tmp_path: Path) -> None: + actual = tmp_path / "actual" + actual.mkdir() + linked = tmp_path / "linked" + linked.symlink_to(actual, target_is_directory=True) + registry = _evidence_registry( + linked, + work_id="linked-lab", + result_id_prefix="linked-result", + schema_version="missioncore.linked-result/v1", + ) + router = build_advanced_laboratory_router( + evidence_registry=registry, + evidence_runtime_root_provider=lambda: tmp_path, + ) + route = _endpoint(router, "/api/v1/laboratory/advanced-index") + + assert route()["items"] == [] # type: ignore[index,operator] + + def test_advanced_index_publishes_l34b_shadow_identity(tmp_path: Path) -> None: root = tmp_path / "l34b" digest = "d" * 64 @@ -82,7 +123,13 @@ def test_advanced_index_publishes_l34b_shadow_identity(tmp_path: Path) -> None: encoding="utf-8", ) router = build_advanced_laboratory_router( - l34b_root_provider=lambda: root, + evidence_registry=_evidence_registry( + root, + work_id="l34b-nested-box-consolidation-shadow", + result_id_prefix="l34b-nested-box-consolidation-shadow", + schema_version="missioncore.l34b-nested-box-consolidation-shadow/v1", + ), + evidence_runtime_root_provider=lambda: root.parent, ) route = _endpoint(router, "/api/v1/laboratory/advanced-index") @@ -123,7 +170,13 @@ def test_advanced_index_publishes_l34c_shadow_identity(tmp_path: Path) -> None: encoding="utf-8", ) router = build_advanced_laboratory_router( - l34c_root_provider=lambda: root, + evidence_registry=_evidence_registry( + root, + work_id="l34c-tile-seam-stitch-shadow", + result_id_prefix="l34c-tile-seam-stitch-shadow", + schema_version="missioncore.l34c-tile-seam-stitch-shadow/v1", + ), + evidence_runtime_root_provider=lambda: root.parent, ) route = _endpoint(router, "/api/v1/laboratory/advanced-index") @@ -166,7 +219,13 @@ def test_advanced_index_publishes_l34d_candidate_identity(tmp_path: Path) -> Non encoding="utf-8", ) router = build_advanced_laboratory_router( - l34d_root_provider=lambda: root, + evidence_registry=_evidence_registry( + root, + work_id="l34d-cumulative-postprocessing-candidate", + result_id_prefix="l34d-cumulative-postprocessing-candidate", + schema_version="missioncore.l34d-cumulative-postprocessing-candidate/v1", + ), + evidence_runtime_root_provider=lambda: root.parent, ) route = _endpoint(router, "/api/v1/laboratory/advanced-index") @@ -207,7 +266,13 @@ def test_advanced_index_publishes_l34e_diagnostic_identity(tmp_path: Path) -> No encoding="utf-8", ) router = build_advanced_laboratory_router( - l34e_root_provider=lambda: root, + evidence_registry=_evidence_registry( + root, + work_id="l34e-self-review-diagnostic", + result_id_prefix="l34e-self-review-diagnostic", + schema_version="missioncore.l34e-self-review-diagnostic/v1", + ), + evidence_runtime_root_provider=lambda: root.parent, ) route = _endpoint(router, "/api/v1/laboratory/advanced-index") @@ -250,7 +315,13 @@ def test_advanced_index_publishes_l34f_adjudicated_reference_identity( encoding="utf-8", ) router = build_advanced_laboratory_router( - l34f_root_provider=lambda: root, + evidence_registry=_evidence_registry( + root, + work_id="l34f-adjudicated-reference", + result_id_prefix="l34f-adjudicated-reference", + schema_version="missioncore.l34f-adjudicated-reference/v1", + ), + evidence_runtime_root_provider=lambda: root.parent, ) route = _endpoint(router, "/api/v1/laboratory/advanced-index") diff --git a/tests/test_compute_contour_network.py b/tests/test_compute_contour_network.py index 5003e84..2aa0ea6 100644 --- a/tests/test_compute_contour_network.py +++ b/tests/test_compute_contour_network.py @@ -111,6 +111,7 @@ def test_worker_apply_uses_private_resolved_endpoint_and_confirms_service( "mqtt_port": 1883, "resolved_addresses": ["192.168.68.56"], "broker_reachable": True, + "recovery_configured": True, } monkeypatch.setattr(network, "_run_worker_powershell", run) @@ -121,6 +122,8 @@ def test_worker_apply_uses_private_resolved_endpoint_and_confirms_service( assert result["ready"] is True assert "__PAYLOAD_BASE64__" not in captured["script"] assert "MISSIONCORE_MQTT_PASSWORD" not in captured["script"] + assert "restart/5000/restart/30000/restart/60000" in captured["script"] + assert "failureflag telegraf 1" in captured["script"] assert captured["timeout"] == 35 diff --git a/tests/test_l34_right_yolox_truth_island_api.py b/tests/test_l34_right_yolox_truth_island_api.py index 6f40c47..a98dc37 100644 --- a/tests/test_l34_right_yolox_truth_island_api.py +++ b/tests/test_l34_right_yolox_truth_island_api.py @@ -4,7 +4,7 @@ import hashlib import json import struct import zlib -from pathlib import Path +from pathlib import Path, PurePosixPath import pytest from fastapi import APIRouter, HTTPException @@ -22,6 +22,7 @@ from k1link.compute.l34_right_yolox_truth_island_freeze import ( L34_PREDICTIONS_NAME, L34_REPORT_NAME, ) +from k1link.laboratory import LaboratoryEvidenceDefinition, LaboratoryEvidenceRegistry from k1link.web.advanced_laboratory_api import build_advanced_laboratory_router from k1link.web.l34_annotation_api import ( L34AnnotationCreateRequest, @@ -579,7 +580,20 @@ def test_l34_annotation_sessions_are_revisioned_and_not_truth( def test_advanced_index_includes_l34_by_real_creation_time(tmp_path: Path) -> None: result = _write_result(tmp_path) router = build_advanced_laboratory_router( - l34_root_provider=lambda: tmp_path, + evidence_registry=LaboratoryEvidenceRegistry( + definitions=( + LaboratoryEvidenceDefinition( + work_id="l34-right-yolox-truth-island-freeze", + runtime_relative_root=PurePosixPath(tmp_path.name), + result_id_prefix="l34-right-yolox-truth-island-freeze", + document_name="manifest.json", + result_schema_version=( + "missioncore.l34-right-yolox-truth-island-freeze/v1" + ), + ), + ) + ), + evidence_runtime_root_provider=lambda: tmp_path.parent, ) index = _endpoint(router, "/api/v1/laboratory/advanced-index")() diff --git a/tests/test_laboratory_evidence_registry.py b/tests/test_laboratory_evidence_registry.py new file mode 100644 index 0000000..abc0b54 --- /dev/null +++ b/tests/test_laboratory_evidence_registry.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import json +from pathlib import Path, PurePosixPath + +import pytest + +from k1link.laboratory import ( + LaboratoryEvidenceDefinition, + LaboratoryEvidenceRegistry, + LaboratoryRegistryError, +) + + +def _definition( + work_id: str, + *, + root: str | None = None, + prefix: str | None = None, + extra: dict[str, object] | None = None, +) -> dict[str, object]: + document: dict[str, object] = { + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": work_id, + "evidence": { + "runtime_relative_root": root or f"{work_id}/results", + "result_id_prefix": prefix or work_id, + "document_name": "manifest.json", + "schema_version": f"missioncore.{work_id}/v1", + }, + } + if extra is not None: + document.update(extra) + return document + + +def _write(root: Path, work_id: str, document: dict[str, object]) -> Path: + path = root / f"{work_id}.json" + path.write_text(json.dumps(document), encoding="utf-8") + return path + + +def test_registry_loads_sorted_definitions_and_resolves_beneath_runtime_root( + tmp_path: Path, +) -> None: + definitions = tmp_path / "definitions" + definitions.mkdir() + _write(definitions, "z-last", _definition("z-last")) + _write( + definitions, + "a-first", + _definition("a-first", root="family/a-first/results"), + ) + + registry = LaboratoryEvidenceRegistry.from_directory(definitions) + + assert [item.work_id for item in registry.definitions] == ["a-first", "z-last"] + first = registry.definitions[0] + assert first.result_root(tmp_path / "runtime") == ( + tmp_path / "runtime" / "family" / "a-first" / "results" + ) + assert first.result_id_pattern.fullmatch(f"a-first-{'a' * 64}") is not None + assert first.result_id_pattern.fullmatch("a-first-not-a-digest") is None + + +@pytest.mark.parametrize("unsafe_root", ["../escape", "/absolute", "a\\b", "."]) +def test_registry_rejects_unsafe_runtime_roots( + tmp_path: Path, + unsafe_root: str, +) -> None: + definitions = tmp_path / "definitions" + definitions.mkdir() + _write(definitions, "unsafe-lab", _definition("unsafe-lab", root=unsafe_root)) + + with pytest.raises(LaboratoryRegistryError, match="runtime_relative_root"): + LaboratoryEvidenceRegistry.from_directory(definitions) + + +def test_registry_rejects_unknown_keys(tmp_path: Path) -> None: + definitions = tmp_path / "definitions" + definitions.mkdir() + _write( + definitions, + "strict-lab", + _definition("strict-lab", extra={"renderer": "hard-coded"}), + ) + + with pytest.raises(LaboratoryRegistryError, match=r"unexpected=\['renderer'\]"): + LaboratoryEvidenceRegistry.from_directory(definitions) + + +def test_registry_rejects_symlinked_definitions(tmp_path: Path) -> None: + definitions = tmp_path / "definitions" + definitions.mkdir() + outside = tmp_path / "outside.json" + outside.write_text(json.dumps(_definition("linked-lab")), encoding="utf-8") + (definitions / "linked-lab.json").symlink_to(outside) + + with pytest.raises(LaboratoryRegistryError, match="regular file"): + LaboratoryEvidenceRegistry.from_directory(definitions) + + +def test_registry_rejects_ambiguous_result_prefixes(tmp_path: Path) -> None: + definitions = tmp_path / "definitions" + definitions.mkdir() + _write(definitions, "first-lab", _definition("first-lab", prefix="shared-result")) + _write(definitions, "second-lab", _definition("second-lab", prefix="shared-result")) + + with pytest.raises(LaboratoryRegistryError, match="duplicate LAB result_id_prefix"): + LaboratoryEvidenceRegistry.from_directory(definitions) + + +def test_definition_constructor_cannot_bypass_path_validation() -> None: + with pytest.raises(LaboratoryRegistryError, match="runtime_relative_root"): + LaboratoryEvidenceDefinition( + work_id="unsafe-lab", + runtime_relative_root=PurePosixPath("../escape"), + result_id_prefix="unsafe-result", + document_name="manifest.json", + result_schema_version="missioncore.unsafe-result/v1", + ) + + +def test_product_registry_declares_every_advanced_evidence_source() -> None: + repository_root = Path(__file__).resolve().parents[1] + registry = LaboratoryEvidenceRegistry.from_directory( + repository_root / "config" / "laboratories" + ) + + assert len(registry.definitions) == 27 + assert {item.work_id for item in registry.definitions} >= { + "e31-source-binding", + "e46j-raw-fisheye-realtime", + "l34f-adjudicated-reference", + } diff --git a/tests/test_telegraf_deployment_contracts.py b/tests/test_telegraf_deployment_contracts.py new file mode 100644 index 0000000..3018705 --- /dev/null +++ b/tests/test_telegraf_deployment_contracts.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +TELEGRAF_ROOT = REPOSITORY_ROOT / "deploy" / "telemetry-plane" / "telegraf" +RECOVERY_ACTIONS = "restart/5000/restart/30000/restart/60000" + + +def test_install_configures_bounded_service_recovery() -> None: + script = (TELEGRAF_ROOT / "Install-NdcMissionCoreTelegraf.ps1").read_text(encoding="utf-8") + + assert "sc.exe failure $serviceName reset= 86400" in script + assert RECOVERY_ACTIONS in script + assert "sc.exe failureflag $serviceName 1" in script + assert "RecoveryConfigured = $true" in script + + +def test_update_configures_recovery_inside_rollback_boundary() -> None: + script = (TELEGRAF_ROOT / "Update-NdcMissionCoreTelegraf.ps1").read_text(encoding="utf-8") + + recovery = script.index("sc.exe failure $serviceName reset= 86400") + rollback = script.index("Stop-Service -Name $serviceName -Force", recovery) + restore_environment = script.index( + "-Type MultiString -Value $serviceEnvironmentBefore", + rollback, + ) + restart = script.index("Start-Service -Name $serviceName", restore_environment) + assert recovery < rollback < restore_environment < restart + assert RECOVERY_ACTIONS in script + assert "sc.exe failureflag $serviceName 1" in script + assert "RecoveryConfigured = $true" in script