From 611853d3af0c49d1726fca1860a1073001bb4a58 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Wed, 29 Jul 2026 14:57:34 +0300 Subject: [PATCH] fix(worker): deploy persistent runtime companions --- deploy/telemetry-plane/README.md | 19 +++-- .../Update-NdcMissionCorePerceptionRunner.ps1 | 42 ++++++++++ ...pdate-NdcMissionCorePersistentLauncher.ps1 | 81 ++++++++++++++++--- docs/01_IMPLEMENTATION_PLAN.md | 2 +- ...1-local-compute-contour-telemetry-plane.md | 27 +++++-- tests/test_worker_deployment_contracts.py | 46 +++++++++++ 6 files changed, 193 insertions(+), 24 deletions(-) create mode 100644 tests/test_worker_deployment_contracts.py diff --git a/deploy/telemetry-plane/README.md b/deploy/telemetry-plane/README.md index 20551f5..ceeaf3d 100644 --- a/deploy/telemetry-plane/README.md +++ b/deploy/telemetry-plane/README.md @@ -136,13 +136,18 @@ change inference control flow. When the mounted perception runner itself changes, use `Update-NdcMissionCorePerceptionRunner.ps1` with exact predecessor and candidate -digests for both the runner and its standalone telemetry module. It backs up the -mounted files, restarts the same container, accepts only a ready health document with -stage metrics and a ready native journal transport, and restores the predecessor on -failure. Update the token-stdin launcher separately with -`Update-NdcMissionCorePersistentLauncher.ps1`; this keeps the operational -contour/agent/node identity explicit and reviewable without changing the K1 command -sequence. +digests for the runner, its `e15_shadow_runtime.py` companion and its standalone +telemetry module. The three files form one deployment unit: updating the runner +without the matching runtime is rejected by the digest-gated transaction. It backs +up the mounted files, restarts the same container, accepts only a ready health +document with stage metrics and a ready native journal transport, and restores the +predecessor set on failure. Update the token-stdin launcher separately with +`Update-NdcMissionCorePersistentLauncher.ps1`. The launcher update always installs +and digest-checks `MissionCoreDockerNames.ps1` in the same transaction because the +launcher dot-sources that companion contract before any worker request. Both files +are restored (or a previously absent companion is removed) if validation fails. This +keeps the operational contour/agent/node identity explicit and reviewable without +changing the K1 command sequence. The stack and agent are intentionally not started by repository tests. Provisioning a machine is a separate, explicit operation. diff --git a/deploy/telemetry-plane/telegraf/Update-NdcMissionCorePerceptionRunner.ps1 b/deploy/telemetry-plane/telegraf/Update-NdcMissionCorePerceptionRunner.ps1 index 6800a72..ed4297e 100644 --- a/deploy/telemetry-plane/telegraf/Update-NdcMissionCorePerceptionRunner.ps1 +++ b/deploy/telemetry-plane/telegraf/Update-NdcMissionCorePerceptionRunner.ps1 @@ -11,6 +11,12 @@ param( [Parameter(Mandatory = $true)] [string]$ExpectedPipelineTelemetrySha256, [string]$ExpectedPipelineTelemetryPredecessorSha256 = "absent", + [Parameter(Mandatory = $true)] + [string]$RuntimeCandidate, + [Parameter(Mandatory = $true)] + [string]$ExpectedRuntimeSha256, + [Parameter(Mandatory = $true)] + [string]$ExpectedRuntimePredecessorSha256, [string]$ContainerName = "ndc-mission-core-perception-worker", [string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002" ) @@ -18,10 +24,13 @@ param( $ErrorActionPreference = "Stop" $target = Join-Path $RunnerRoot "run_e15_shadow_inference.py" $telemetryTarget = Join-Path $RunnerRoot "pipeline_telemetry.py" +$runtimeTarget = Join-Path $RunnerRoot "e15_shadow_runtime.py" $expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant() $expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant() $expectedTelemetry = $ExpectedPipelineTelemetrySha256.ToLowerInvariant() $expectedTelemetryPredecessor = $ExpectedPipelineTelemetryPredecessorSha256.ToLowerInvariant() +$expectedRuntime = $ExpectedRuntimeSha256.ToLowerInvariant() +$expectedRuntimePredecessor = $ExpectedRuntimePredecessorSha256.ToLowerInvariant() $predecessor = ( Get-FileHash -Algorithm SHA256 -LiteralPath $target ).Hash.ToLowerInvariant() @@ -50,6 +59,18 @@ else { if ($telemetryPredecessor -ne $expectedTelemetryPredecessor) { throw "Pipeline telemetry predecessor digest changed" } +$runtimePredecessor = ( + Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget +).Hash.ToLowerInvariant() +if ($runtimePredecessor -ne $expectedRuntimePredecessor) { + throw "E15 runtime predecessor digest changed" +} +$runtimeCandidateDigest = ( + Get-FileHash -Algorithm SHA256 -LiteralPath $RuntimeCandidate +).Hash.ToLowerInvariant() +if ($runtimeCandidateDigest -ne $expectedRuntime) { + throw "E15 runtime candidate digest changed" +} $backup = Join-Path $RunnerRoot ( "run_e15_shadow_inference.py.rollback-" + [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") + @@ -62,6 +83,12 @@ $telemetryBackup = Join-Path $RunnerRoot ( "-" + $telemetryPredecessor.Substring(0, [Math]::Min(12, $telemetryPredecessor.Length)) ) +$runtimeBackup = Join-Path $RunnerRoot ( + "e15_shadow_runtime.py.rollback-" + + [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") + + "-" + + $runtimePredecessor.Substring(0, 12) +) $healthCode = "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:18020/health',timeout=2).read().decode())" function Wait-PerceptionHealth { @@ -100,7 +127,10 @@ Copy-Item -LiteralPath $target -Destination $backup if ($hadTelemetryModule) { Copy-Item -LiteralPath $telemetryTarget -Destination $telemetryBackup } +Copy-Item -LiteralPath $runtimeTarget -Destination $runtimeBackup try { + Copy-Item -LiteralPath $RuntimeCandidate ` + -Destination $runtimeTarget -Force Copy-Item -LiteralPath $PipelineTelemetryCandidate ` -Destination $telemetryTarget -Force Copy-Item -LiteralPath $Candidate -Destination $target -Force @@ -116,6 +146,12 @@ try { ) { throw "Pipeline telemetry replacement digest changed" } + if ( + (Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget).Hash.ToLowerInvariant() ` + -ne $expectedRuntime + ) { + throw "E15 runtime replacement digest changed" + } docker restart --time 20 $ContainerName | Out-Null if ($LASTEXITCODE -ne 0) { throw "Perception container restart failed" @@ -127,6 +163,7 @@ try { } catch { Copy-Item -LiteralPath $backup -Destination $target -Force + Copy-Item -LiteralPath $runtimeBackup -Destination $runtimeTarget -Force if ($hadTelemetryModule) { Copy-Item -LiteralPath $telemetryBackup -Destination $telemetryTarget -Force } @@ -153,6 +190,11 @@ catch { PipelineTelemetryCandidateSha256 = ( Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget ).Hash.ToLowerInvariant() + RuntimePredecessorSha256 = $runtimePredecessor + RuntimeCandidateSha256 = ( + Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget + ).Hash.ToLowerInvariant() + RuntimeBackup = $runtimeBackup PipelineTelemetryBackup = if ($hadTelemetryModule) { $telemetryBackup } else { $null } Backup = $backup Health = $health diff --git a/deploy/telemetry-plane/telegraf/Update-NdcMissionCorePersistentLauncher.ps1 b/deploy/telemetry-plane/telegraf/Update-NdcMissionCorePersistentLauncher.ps1 index 94b8a03..a935d4d 100644 --- a/deploy/telemetry-plane/telegraf/Update-NdcMissionCorePersistentLauncher.ps1 +++ b/deploy/telemetry-plane/telegraf/Update-NdcMissionCorePersistentLauncher.ps1 @@ -6,13 +6,23 @@ param( [string]$ExpectedPredecessorSha256, [Parameter(Mandatory = $true)] [string]$ExpectedCandidateSha256, + [Parameter(Mandatory = $true)] + [string]$DockerNamesCandidate, + [Parameter(Mandatory = $true)] + [string]$ExpectedDockerNamesSha256, + [string]$ExpectedDockerNamesPredecessorSha256 = "absent", [string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002" ) $ErrorActionPreference = "Stop" $target = Join-Path $RunnerRoot "Invoke-E15PersistentShadowRun.ps1" +$dockerNamesTarget = Join-Path $RunnerRoot "MissionCoreDockerNames.ps1" $expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant() $expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant() +$expectedDockerNames = $ExpectedDockerNamesSha256.ToLowerInvariant() +$expectedDockerNamesPredecessor = ( + $ExpectedDockerNamesPredecessorSha256.ToLowerInvariant() +) $predecessor = ( Get-FileHash -Algorithm SHA256 -LiteralPath $target ).Hash.ToLowerInvariant() @@ -25,15 +35,47 @@ if ( ) { throw "Persistent launcher candidate digest changed" } +$dockerNamesCandidateDigest = ( + Get-FileHash -Algorithm SHA256 -LiteralPath $DockerNamesCandidate +).Hash.ToLowerInvariant() +if ($dockerNamesCandidateDigest -ne $expectedDockerNames) { + throw "Docker names candidate digest changed" +} +$hadDockerNames = Test-Path -LiteralPath $dockerNamesTarget -PathType Leaf +$dockerNamesPredecessor = if ($hadDockerNames) { + ( + Get-FileHash -Algorithm SHA256 -LiteralPath $dockerNamesTarget + ).Hash.ToLowerInvariant() +} +else { + "absent" +} +if ($dockerNamesPredecessor -ne $expectedDockerNamesPredecessor) { + throw "Docker names predecessor digest changed" +} $backup = Join-Path $RunnerRoot ( "Invoke-E15PersistentShadowRun.ps1.rollback-" + [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") + "-" + $predecessor.Substring(0, 12) ) +$dockerNamesBackup = Join-Path $RunnerRoot ( + "MissionCoreDockerNames.ps1.rollback-" + + [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") + + "-" + + $dockerNamesPredecessor.Substring( + 0, + [Math]::Min(12, $dockerNamesPredecessor.Length) + ) +) Copy-Item -LiteralPath $target -Destination $backup +if ($hadDockerNames) { + Copy-Item -LiteralPath $dockerNamesTarget -Destination $dockerNamesBackup +} try { + Copy-Item -LiteralPath $DockerNamesCandidate ` + -Destination $dockerNamesTarget -Force Copy-Item -LiteralPath $Candidate -Destination $target -Force $installed = ( Get-FileHash -Algorithm SHA256 -LiteralPath $target @@ -41,19 +83,35 @@ try { if ($installed -ne $expectedCandidate) { throw "Persistent launcher replacement digest changed" } - $tokens = $null - $parseErrors = $null - [Management.Automation.Language.Parser]::ParseFile( - $target, - [ref]$tokens, - [ref]$parseErrors - ) | Out-Null - if (@($parseErrors).Count -ne 0) { - throw "Persistent launcher candidate has PowerShell parse errors" + $installedDockerNames = ( + Get-FileHash -Algorithm SHA256 -LiteralPath $dockerNamesTarget + ).Hash.ToLowerInvariant() + if ($installedDockerNames -ne $expectedDockerNames) { + throw "Docker names replacement digest changed" + } + foreach ($scriptPath in @($dockerNamesTarget, $target)) { + $tokens = $null + $parseErrors = $null + [Management.Automation.Language.Parser]::ParseFile( + $scriptPath, + [ref]$tokens, + [ref]$parseErrors + ) | Out-Null + if (@($parseErrors).Count -ne 0) { + throw "Persistent launcher deployment has PowerShell parse errors" + } } } catch { Copy-Item -LiteralPath $backup -Destination $target -Force + if ($hadDockerNames) { + Copy-Item -LiteralPath $dockerNamesBackup ` + -Destination $dockerNamesTarget -Force + } + else { + Remove-Item -LiteralPath $dockerNamesTarget ` + -Force -ErrorAction SilentlyContinue + } throw } @@ -63,5 +121,10 @@ catch { CandidateSha256 = ( Get-FileHash -Algorithm SHA256 -LiteralPath $target ).Hash.ToLowerInvariant() + DockerNamesPredecessorSha256 = $dockerNamesPredecessor + DockerNamesCandidateSha256 = ( + Get-FileHash -Algorithm SHA256 -LiteralPath $dockerNamesTarget + ).Hash.ToLowerInvariant() + DockerNamesBackup = if ($hadDockerNames) { $dockerNamesBackup } else { $null } Backup = $backup } | ConvertTo-Json -Compress diff --git a/docs/01_IMPLEMENTATION_PLAN.md b/docs/01_IMPLEMENTATION_PLAN.md index 901efed..d80b7ad 100644 --- a/docs/01_IMPLEMENTATION_PLAN.md +++ b/docs/01_IMPLEMENTATION_PLAN.md @@ -11,7 +11,7 @@ Each gate produces evidence and an explicit GO, PAUSE or BLOCKED result. | Prediction/evaluation boundary | GO — truth-free E41 predictor package/result are physically separate from E37 and reproduce E40 predictions exactly; the visible evaluator runs only after inference. | | Source-scoped quality | PAUSE — E40 visible engineering evaluation is 84.2466% presence, 84.2466% geometry association and 94.5205% freshness, with 11 high-severity failures, complete accounting and zero false-free claims. This is not a blind accuracy gate. | | Structural regressions | GO (bounded) — E42 passes predictor identity/order/chunk invariance and PointSlab order/SE(3) checks. Raw-producer and cross-route invariance remain unproved. | -| Native pipeline telemetry | GO for producer/normalizer contract — lifecycle events and JSONL/MQTT sink boundaries exist and an E41 smoke run records real stage accounting. Durable Worker 006 MQTT wiring/deployment remains pending. | +| Native pipeline telemetry | GO (durable Worker 006) — lifecycle events and JSONL/MQTT sink boundaries exist; the persistent worker writes to its existing D:-backed publish mount, host Telegraf tails with saved offset and publishes through the existing authenticated QoS 1 path. Real request `durable-native-acceptance-20260729-115418` produced native run `started → failed` after reaching the unavailable live-source boundary, and both documents were observed through broker → normalizer → Timescale query API with the same run/request identity. No broker credential or additional container entered the inference runtime. | | Evidence storage | MEASURED — E44 finds 525,471,092 logical bytes, 312,753,179 unique-content bytes and 1.680146× amplification across 14 E30–E40 roots. Exact-content references/deduplication precede any format migration. | | Future transfer | PREREGISTERED — E43 freezes same-K1/mount/calibration/firmware, required streams, connected-component split and independent label reveal. Capture and labels do not yet exist. | | E31 binding sensitivity | MEASURED — E45 closes accounting for 87/87 accepted correspondences and finds no material monotonic residual association with represented image radius, rig speed or pose age. It does not supply calibration-target truth or outer-fisheye coverage. | diff --git a/docs/adr/0031-local-compute-contour-telemetry-plane.md b/docs/adr/0031-local-compute-contour-telemetry-plane.md index debfca3..1b6be85 100644 --- a/docs/adr/0031-local-compute-contour-telemetry-plane.md +++ b/docs/adr/0031-local-compute-contour-telemetry-plane.md @@ -29,13 +29,26 @@ The existing Worker 006 containers were renamed in place to container identities were preserved, so this namespace migration did not restart the inference or perception runtimes. -The compute-side native pipeline contract is now implemented. It emits -`missioncore.agent-pipeline-telemetry/v1` lifecycle documents through an injected -transport, and the telemetry normalizer preserves source, method, stage and stage-state -identities in the normalized series key. The E41 runner has exercised the JSONL -evidence sink against the immutable predictor package. This proves producer and -normalizer compatibility; it does not claim that the durable Worker 006 process has -been wired to the MQTT sink or deployed with this code. +The compute-side native pipeline contract is implemented and deployed on Worker 006. +It emits `missioncore.agent-pipeline-telemetry/v1` lifecycle documents through an +injected transport, and the telemetry normalizer preserves source, method, stage and +stage-state identities in the normalized series key. The E41 runner first exercised +the JSONL evidence sink against the immutable predictor package. + +The durable path was then accepted on 2026-07-29 with real persistent-worker request +`durable-native-acceptance-20260729-115418`. The request reached the actual live-source +boundary and terminated because no K1 shadow source was open. Its native run +`started` and `failed` documents were appended by the worker, tailed by host Telegraf, +published on the contour pipeline topic and returned by the normalized Timescale query +API with the same run/request identity. This is transport and lifecycle acceptance, +not a successful perception-result or stage-performance claim. + +The acceptance also closed two deployment-unit defects found by the real request: +the token-stdin launcher now installs its required `MissionCoreDockerNames.ps1` +companion, and the perception runner updater atomically digest-gates +`run_e15_shadow_inference.py`, `e15_shadow_runtime.py` and +`pipeline_telemetry.py`. The existing perception container is still the only +inference worker; broker credentials remain on the host Telegraf service. ## Decision diff --git a/tests/test_worker_deployment_contracts.py b/tests/test_worker_deployment_contracts.py new file mode 100644 index 0000000..a8a2df9 --- /dev/null +++ b/tests/test_worker_deployment_contracts.py @@ -0,0 +1,46 @@ +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] + + +def test_persistent_launcher_deploys_required_docker_names_companion() -> None: + deployment = ( + REPOSITORY_ROOT + / "deploy" + / "telemetry-plane" + / "telegraf" + / "Update-NdcMissionCorePersistentLauncher.ps1" + ).read_text(encoding="utf-8") + launcher = ( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "Invoke-E15PersistentShadowRun.ps1" + ).read_text(encoding="utf-8") + + assert '. (Join-Path $PSScriptRoot "MissionCoreDockerNames.ps1")' in launcher + assert "[string]$DockerNamesCandidate" in deployment + assert "[string]$ExpectedDockerNamesSha256" in deployment + assert "[string]$ExpectedDockerNamesPredecessorSha256" in deployment + assert '$dockerNamesPredecessor = if ($hadDockerNames)' in deployment + assert "Copy-Item -LiteralPath $DockerNamesCandidate" in deployment + assert "Remove-Item -LiteralPath $dockerNamesTarget" in deployment + + +def test_perception_runner_deploys_required_runtime_companion() -> None: + deployment = ( + REPOSITORY_ROOT + / "deploy" + / "telemetry-plane" + / "telegraf" + / "Update-NdcMissionCorePerceptionRunner.ps1" + ).read_text(encoding="utf-8") + + assert "[string]$RuntimeCandidate" in deployment + assert "[string]$ExpectedRuntimeSha256" in deployment + assert "[string]$ExpectedRuntimePredecessorSha256" in deployment + assert '$runtimeTarget = Join-Path $RunnerRoot "e15_shadow_runtime.py"' in deployment + assert "Copy-Item -LiteralPath $RuntimeCandidate" in deployment + assert "Copy-Item -LiteralPath $runtimeBackup" in deployment + assert "RuntimeCandidateSha256" in deployment