fix(worker): deploy persistent runtime companions

This commit is contained in:
DCCONSTRUCTIONS
2026-07-29 14:57:34 +03:00
parent 9db39347bd
commit 611853d3af
6 changed files with 193 additions and 24 deletions
+12 -7
View File
@@ -136,13 +136,18 @@ change inference control flow.
When the mounted perception runner itself changes, use When the mounted perception runner itself changes, use
`Update-NdcMissionCorePerceptionRunner.ps1` with exact predecessor and candidate `Update-NdcMissionCorePerceptionRunner.ps1` with exact predecessor and candidate
digests for both the runner and its standalone telemetry module. It backs up the digests for the runner, its `e15_shadow_runtime.py` companion and its standalone
mounted files, restarts the same container, accepts only a ready health document with telemetry module. The three files form one deployment unit: updating the runner
stage metrics and a ready native journal transport, and restores the predecessor on without the matching runtime is rejected by the digest-gated transaction. It backs
failure. Update the token-stdin launcher separately with up the mounted files, restarts the same container, accepts only a ready health
`Update-NdcMissionCorePersistentLauncher.ps1`; this keeps the operational document with stage metrics and a ready native journal transport, and restores the
contour/agent/node identity explicit and reviewable without changing the K1 command predecessor set on failure. Update the token-stdin launcher separately with
sequence. `Update-NdcMissionCorePersistentLauncher.ps1`. The launcher update always installs
and digest-checks `MissionCoreDockerNames.ps1` in the same transaction because the
launcher dot-sources that companion contract before any worker request. Both files
are restored (or a previously absent companion is removed) if validation fails. This
keeps the operational contour/agent/node identity explicit and reviewable without
changing the K1 command sequence.
The stack and agent are intentionally not started by repository tests. Provisioning a The stack and agent are intentionally not started by repository tests. Provisioning a
machine is a separate, explicit operation. machine is a separate, explicit operation.
@@ -11,6 +11,12 @@ param(
[Parameter(Mandatory = $true)] [Parameter(Mandatory = $true)]
[string]$ExpectedPipelineTelemetrySha256, [string]$ExpectedPipelineTelemetrySha256,
[string]$ExpectedPipelineTelemetryPredecessorSha256 = "absent", [string]$ExpectedPipelineTelemetryPredecessorSha256 = "absent",
[Parameter(Mandatory = $true)]
[string]$RuntimeCandidate,
[Parameter(Mandatory = $true)]
[string]$ExpectedRuntimeSha256,
[Parameter(Mandatory = $true)]
[string]$ExpectedRuntimePredecessorSha256,
[string]$ContainerName = "ndc-mission-core-perception-worker", [string]$ContainerName = "ndc-mission-core-perception-worker",
[string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002" [string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002"
) )
@@ -18,10 +24,13 @@ param(
$ErrorActionPreference = "Stop" $ErrorActionPreference = "Stop"
$target = Join-Path $RunnerRoot "run_e15_shadow_inference.py" $target = Join-Path $RunnerRoot "run_e15_shadow_inference.py"
$telemetryTarget = Join-Path $RunnerRoot "pipeline_telemetry.py" $telemetryTarget = Join-Path $RunnerRoot "pipeline_telemetry.py"
$runtimeTarget = Join-Path $RunnerRoot "e15_shadow_runtime.py"
$expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant() $expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant()
$expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant() $expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant()
$expectedTelemetry = $ExpectedPipelineTelemetrySha256.ToLowerInvariant() $expectedTelemetry = $ExpectedPipelineTelemetrySha256.ToLowerInvariant()
$expectedTelemetryPredecessor = $ExpectedPipelineTelemetryPredecessorSha256.ToLowerInvariant() $expectedTelemetryPredecessor = $ExpectedPipelineTelemetryPredecessorSha256.ToLowerInvariant()
$expectedRuntime = $ExpectedRuntimeSha256.ToLowerInvariant()
$expectedRuntimePredecessor = $ExpectedRuntimePredecessorSha256.ToLowerInvariant()
$predecessor = ( $predecessor = (
Get-FileHash -Algorithm SHA256 -LiteralPath $target Get-FileHash -Algorithm SHA256 -LiteralPath $target
).Hash.ToLowerInvariant() ).Hash.ToLowerInvariant()
@@ -50,6 +59,18 @@ else {
if ($telemetryPredecessor -ne $expectedTelemetryPredecessor) { if ($telemetryPredecessor -ne $expectedTelemetryPredecessor) {
throw "Pipeline telemetry predecessor digest changed" throw "Pipeline telemetry predecessor digest changed"
} }
$runtimePredecessor = (
Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget
).Hash.ToLowerInvariant()
if ($runtimePredecessor -ne $expectedRuntimePredecessor) {
throw "E15 runtime predecessor digest changed"
}
$runtimeCandidateDigest = (
Get-FileHash -Algorithm SHA256 -LiteralPath $RuntimeCandidate
).Hash.ToLowerInvariant()
if ($runtimeCandidateDigest -ne $expectedRuntime) {
throw "E15 runtime candidate digest changed"
}
$backup = Join-Path $RunnerRoot ( $backup = Join-Path $RunnerRoot (
"run_e15_shadow_inference.py.rollback-" + "run_e15_shadow_inference.py.rollback-" +
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") + [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
@@ -62,6 +83,12 @@ $telemetryBackup = Join-Path $RunnerRoot (
"-" + "-" +
$telemetryPredecessor.Substring(0, [Math]::Min(12, $telemetryPredecessor.Length)) $telemetryPredecessor.Substring(0, [Math]::Min(12, $telemetryPredecessor.Length))
) )
$runtimeBackup = Join-Path $RunnerRoot (
"e15_shadow_runtime.py.rollback-" +
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
"-" +
$runtimePredecessor.Substring(0, 12)
)
$healthCode = "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:18020/health',timeout=2).read().decode())" $healthCode = "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:18020/health',timeout=2).read().decode())"
function Wait-PerceptionHealth { function Wait-PerceptionHealth {
@@ -100,7 +127,10 @@ Copy-Item -LiteralPath $target -Destination $backup
if ($hadTelemetryModule) { if ($hadTelemetryModule) {
Copy-Item -LiteralPath $telemetryTarget -Destination $telemetryBackup Copy-Item -LiteralPath $telemetryTarget -Destination $telemetryBackup
} }
Copy-Item -LiteralPath $runtimeTarget -Destination $runtimeBackup
try { try {
Copy-Item -LiteralPath $RuntimeCandidate `
-Destination $runtimeTarget -Force
Copy-Item -LiteralPath $PipelineTelemetryCandidate ` Copy-Item -LiteralPath $PipelineTelemetryCandidate `
-Destination $telemetryTarget -Force -Destination $telemetryTarget -Force
Copy-Item -LiteralPath $Candidate -Destination $target -Force Copy-Item -LiteralPath $Candidate -Destination $target -Force
@@ -116,6 +146,12 @@ try {
) { ) {
throw "Pipeline telemetry replacement digest changed" throw "Pipeline telemetry replacement digest changed"
} }
if (
(Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget).Hash.ToLowerInvariant() `
-ne $expectedRuntime
) {
throw "E15 runtime replacement digest changed"
}
docker restart --time 20 $ContainerName | Out-Null docker restart --time 20 $ContainerName | Out-Null
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
throw "Perception container restart failed" throw "Perception container restart failed"
@@ -127,6 +163,7 @@ try {
} }
catch { catch {
Copy-Item -LiteralPath $backup -Destination $target -Force Copy-Item -LiteralPath $backup -Destination $target -Force
Copy-Item -LiteralPath $runtimeBackup -Destination $runtimeTarget -Force
if ($hadTelemetryModule) { if ($hadTelemetryModule) {
Copy-Item -LiteralPath $telemetryBackup -Destination $telemetryTarget -Force Copy-Item -LiteralPath $telemetryBackup -Destination $telemetryTarget -Force
} }
@@ -153,6 +190,11 @@ catch {
PipelineTelemetryCandidateSha256 = ( PipelineTelemetryCandidateSha256 = (
Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget
).Hash.ToLowerInvariant() ).Hash.ToLowerInvariant()
RuntimePredecessorSha256 = $runtimePredecessor
RuntimeCandidateSha256 = (
Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget
).Hash.ToLowerInvariant()
RuntimeBackup = $runtimeBackup
PipelineTelemetryBackup = if ($hadTelemetryModule) { $telemetryBackup } else { $null } PipelineTelemetryBackup = if ($hadTelemetryModule) { $telemetryBackup } else { $null }
Backup = $backup Backup = $backup
Health = $health Health = $health
@@ -6,13 +6,23 @@ param(
[string]$ExpectedPredecessorSha256, [string]$ExpectedPredecessorSha256,
[Parameter(Mandatory = $true)] [Parameter(Mandatory = $true)]
[string]$ExpectedCandidateSha256, [string]$ExpectedCandidateSha256,
[Parameter(Mandatory = $true)]
[string]$DockerNamesCandidate,
[Parameter(Mandatory = $true)]
[string]$ExpectedDockerNamesSha256,
[string]$ExpectedDockerNamesPredecessorSha256 = "absent",
[string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002" [string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002"
) )
$ErrorActionPreference = "Stop" $ErrorActionPreference = "Stop"
$target = Join-Path $RunnerRoot "Invoke-E15PersistentShadowRun.ps1" $target = Join-Path $RunnerRoot "Invoke-E15PersistentShadowRun.ps1"
$dockerNamesTarget = Join-Path $RunnerRoot "MissionCoreDockerNames.ps1"
$expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant() $expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant()
$expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant() $expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant()
$expectedDockerNames = $ExpectedDockerNamesSha256.ToLowerInvariant()
$expectedDockerNamesPredecessor = (
$ExpectedDockerNamesPredecessorSha256.ToLowerInvariant()
)
$predecessor = ( $predecessor = (
Get-FileHash -Algorithm SHA256 -LiteralPath $target Get-FileHash -Algorithm SHA256 -LiteralPath $target
).Hash.ToLowerInvariant() ).Hash.ToLowerInvariant()
@@ -25,15 +35,47 @@ if (
) { ) {
throw "Persistent launcher candidate digest changed" throw "Persistent launcher candidate digest changed"
} }
$dockerNamesCandidateDigest = (
Get-FileHash -Algorithm SHA256 -LiteralPath $DockerNamesCandidate
).Hash.ToLowerInvariant()
if ($dockerNamesCandidateDigest -ne $expectedDockerNames) {
throw "Docker names candidate digest changed"
}
$hadDockerNames = Test-Path -LiteralPath $dockerNamesTarget -PathType Leaf
$dockerNamesPredecessor = if ($hadDockerNames) {
(
Get-FileHash -Algorithm SHA256 -LiteralPath $dockerNamesTarget
).Hash.ToLowerInvariant()
}
else {
"absent"
}
if ($dockerNamesPredecessor -ne $expectedDockerNamesPredecessor) {
throw "Docker names predecessor digest changed"
}
$backup = Join-Path $RunnerRoot ( $backup = Join-Path $RunnerRoot (
"Invoke-E15PersistentShadowRun.ps1.rollback-" + "Invoke-E15PersistentShadowRun.ps1.rollback-" +
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") + [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
"-" + "-" +
$predecessor.Substring(0, 12) $predecessor.Substring(0, 12)
) )
$dockerNamesBackup = Join-Path $RunnerRoot (
"MissionCoreDockerNames.ps1.rollback-" +
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
"-" +
$dockerNamesPredecessor.Substring(
0,
[Math]::Min(12, $dockerNamesPredecessor.Length)
)
)
Copy-Item -LiteralPath $target -Destination $backup Copy-Item -LiteralPath $target -Destination $backup
if ($hadDockerNames) {
Copy-Item -LiteralPath $dockerNamesTarget -Destination $dockerNamesBackup
}
try { try {
Copy-Item -LiteralPath $DockerNamesCandidate `
-Destination $dockerNamesTarget -Force
Copy-Item -LiteralPath $Candidate -Destination $target -Force Copy-Item -LiteralPath $Candidate -Destination $target -Force
$installed = ( $installed = (
Get-FileHash -Algorithm SHA256 -LiteralPath $target Get-FileHash -Algorithm SHA256 -LiteralPath $target
@@ -41,19 +83,35 @@ try {
if ($installed -ne $expectedCandidate) { if ($installed -ne $expectedCandidate) {
throw "Persistent launcher replacement digest changed" throw "Persistent launcher replacement digest changed"
} }
$tokens = $null $installedDockerNames = (
$parseErrors = $null Get-FileHash -Algorithm SHA256 -LiteralPath $dockerNamesTarget
[Management.Automation.Language.Parser]::ParseFile( ).Hash.ToLowerInvariant()
$target, if ($installedDockerNames -ne $expectedDockerNames) {
[ref]$tokens, throw "Docker names replacement digest changed"
[ref]$parseErrors }
) | Out-Null foreach ($scriptPath in @($dockerNamesTarget, $target)) {
if (@($parseErrors).Count -ne 0) { $tokens = $null
throw "Persistent launcher candidate has PowerShell parse errors" $parseErrors = $null
[Management.Automation.Language.Parser]::ParseFile(
$scriptPath,
[ref]$tokens,
[ref]$parseErrors
) | Out-Null
if (@($parseErrors).Count -ne 0) {
throw "Persistent launcher deployment has PowerShell parse errors"
}
} }
} }
catch { catch {
Copy-Item -LiteralPath $backup -Destination $target -Force Copy-Item -LiteralPath $backup -Destination $target -Force
if ($hadDockerNames) {
Copy-Item -LiteralPath $dockerNamesBackup `
-Destination $dockerNamesTarget -Force
}
else {
Remove-Item -LiteralPath $dockerNamesTarget `
-Force -ErrorAction SilentlyContinue
}
throw throw
} }
@@ -63,5 +121,10 @@ catch {
CandidateSha256 = ( CandidateSha256 = (
Get-FileHash -Algorithm SHA256 -LiteralPath $target Get-FileHash -Algorithm SHA256 -LiteralPath $target
).Hash.ToLowerInvariant() ).Hash.ToLowerInvariant()
DockerNamesPredecessorSha256 = $dockerNamesPredecessor
DockerNamesCandidateSha256 = (
Get-FileHash -Algorithm SHA256 -LiteralPath $dockerNamesTarget
).Hash.ToLowerInvariant()
DockerNamesBackup = if ($hadDockerNames) { $dockerNamesBackup } else { $null }
Backup = $backup Backup = $backup
} | ConvertTo-Json -Compress } | ConvertTo-Json -Compress
+1 -1
View File
@@ -11,7 +11,7 @@ Each gate produces evidence and an explicit GO, PAUSE or BLOCKED result.
| Prediction/evaluation boundary | GO — truth-free E41 predictor package/result are physically separate from E37 and reproduce E40 predictions exactly; the visible evaluator runs only after inference. | | Prediction/evaluation boundary | GO — truth-free E41 predictor package/result are physically separate from E37 and reproduce E40 predictions exactly; the visible evaluator runs only after inference. |
| Source-scoped quality | PAUSE — E40 visible engineering evaluation is 84.2466% presence, 84.2466% geometry association and 94.5205% freshness, with 11 high-severity failures, complete accounting and zero false-free claims. This is not a blind accuracy gate. | | Source-scoped quality | PAUSE — E40 visible engineering evaluation is 84.2466% presence, 84.2466% geometry association and 94.5205% freshness, with 11 high-severity failures, complete accounting and zero false-free claims. This is not a blind accuracy gate. |
| Structural regressions | GO (bounded) — E42 passes predictor identity/order/chunk invariance and PointSlab order/SE(3) checks. Raw-producer and cross-route invariance remain unproved. | | Structural regressions | GO (bounded) — E42 passes predictor identity/order/chunk invariance and PointSlab order/SE(3) checks. Raw-producer and cross-route invariance remain unproved. |
| Native pipeline telemetry | GO for producer/normalizer contract — lifecycle events and JSONL/MQTT sink boundaries exist and an E41 smoke run records real stage accounting. Durable Worker 006 MQTT wiring/deployment remains pending. | | Native pipeline telemetry | GO (durable Worker 006) — lifecycle events and JSONL/MQTT sink boundaries exist; the persistent worker writes to its existing D:-backed publish mount, host Telegraf tails with saved offset and publishes through the existing authenticated QoS 1 path. Real request `durable-native-acceptance-20260729-115418` produced native run `started → failed` after reaching the unavailable live-source boundary, and both documents were observed through broker → normalizer → Timescale query API with the same run/request identity. No broker credential or additional container entered the inference runtime. |
| Evidence storage | MEASURED — E44 finds 525,471,092 logical bytes, 312,753,179 unique-content bytes and 1.680146× amplification across 14 E30E40 roots. Exact-content references/deduplication precede any format migration. | | Evidence storage | MEASURED — E44 finds 525,471,092 logical bytes, 312,753,179 unique-content bytes and 1.680146× amplification across 14 E30E40 roots. Exact-content references/deduplication precede any format migration. |
| Future transfer | PREREGISTERED — E43 freezes same-K1/mount/calibration/firmware, required streams, connected-component split and independent label reveal. Capture and labels do not yet exist. | | Future transfer | PREREGISTERED — E43 freezes same-K1/mount/calibration/firmware, required streams, connected-component split and independent label reveal. Capture and labels do not yet exist. |
| E31 binding sensitivity | MEASURED — E45 closes accounting for 87/87 accepted correspondences and finds no material monotonic residual association with represented image radius, rig speed or pose age. It does not supply calibration-target truth or outer-fisheye coverage. | | E31 binding sensitivity | MEASURED — E45 closes accounting for 87/87 accepted correspondences and finds no material monotonic residual association with represented image radius, rig speed or pose age. It does not supply calibration-target truth or outer-fisheye coverage. |
@@ -29,13 +29,26 @@ The existing Worker 006 containers were renamed in place to
container identities were preserved, so this namespace migration did not restart the container identities were preserved, so this namespace migration did not restart the
inference or perception runtimes. inference or perception runtimes.
The compute-side native pipeline contract is now implemented. It emits The compute-side native pipeline contract is implemented and deployed on Worker 006.
`missioncore.agent-pipeline-telemetry/v1` lifecycle documents through an injected It emits `missioncore.agent-pipeline-telemetry/v1` lifecycle documents through an
transport, and the telemetry normalizer preserves source, method, stage and stage-state injected transport, and the telemetry normalizer preserves source, method, stage and
identities in the normalized series key. The E41 runner has exercised the JSONL stage-state identities in the normalized series key. The E41 runner first exercised
evidence sink against the immutable predictor package. This proves producer and the JSONL evidence sink against the immutable predictor package.
normalizer compatibility; it does not claim that the durable Worker 006 process has
been wired to the MQTT sink or deployed with this code. The durable path was then accepted on 2026-07-29 with real persistent-worker request
`durable-native-acceptance-20260729-115418`. The request reached the actual live-source
boundary and terminated because no K1 shadow source was open. Its native run
`started` and `failed` documents were appended by the worker, tailed by host Telegraf,
published on the contour pipeline topic and returned by the normalized Timescale query
API with the same run/request identity. This is transport and lifecycle acceptance,
not a successful perception-result or stage-performance claim.
The acceptance also closed two deployment-unit defects found by the real request:
the token-stdin launcher now installs its required `MissionCoreDockerNames.ps1`
companion, and the perception runner updater atomically digest-gates
`run_e15_shadow_inference.py`, `e15_shadow_runtime.py` and
`pipeline_telemetry.py`. The existing perception container is still the only
inference worker; broker credentials remain on the host Telegraf service.
## Decision ## Decision
+46
View File
@@ -0,0 +1,46 @@
from pathlib import Path
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
def test_persistent_launcher_deploys_required_docker_names_companion() -> None:
deployment = (
REPOSITORY_ROOT
/ "deploy"
/ "telemetry-plane"
/ "telegraf"
/ "Update-NdcMissionCorePersistentLauncher.ps1"
).read_text(encoding="utf-8")
launcher = (
REPOSITORY_ROOT
/ "experiments"
/ "perception"
/ "worker"
/ "Invoke-E15PersistentShadowRun.ps1"
).read_text(encoding="utf-8")
assert '. (Join-Path $PSScriptRoot "MissionCoreDockerNames.ps1")' in launcher
assert "[string]$DockerNamesCandidate" in deployment
assert "[string]$ExpectedDockerNamesSha256" in deployment
assert "[string]$ExpectedDockerNamesPredecessorSha256" in deployment
assert '$dockerNamesPredecessor = if ($hadDockerNames)' in deployment
assert "Copy-Item -LiteralPath $DockerNamesCandidate" in deployment
assert "Remove-Item -LiteralPath $dockerNamesTarget" in deployment
def test_perception_runner_deploys_required_runtime_companion() -> None:
deployment = (
REPOSITORY_ROOT
/ "deploy"
/ "telemetry-plane"
/ "telegraf"
/ "Update-NdcMissionCorePerceptionRunner.ps1"
).read_text(encoding="utf-8")
assert "[string]$RuntimeCandidate" in deployment
assert "[string]$ExpectedRuntimeSha256" in deployment
assert "[string]$ExpectedRuntimePredecessorSha256" in deployment
assert '$runtimeTarget = Join-Path $RunnerRoot "e15_shadow_runtime.py"' in deployment
assert "Copy-Item -LiteralPath $RuntimeCandidate" in deployment
assert "Copy-Item -LiteralPath $runtimeBackup" in deployment
assert "RuntimeCandidateSha256" in deployment