fix(worker): deploy persistent runtime companions
This commit is contained in:
DCCONSTRUCTIONS
@@ -136,13 +136,18 @@ change inference control flow.
|
||||
|
||||
When the mounted perception runner itself changes, use
|
||||
`Update-NdcMissionCorePerceptionRunner.ps1` with exact predecessor and candidate
|
||||
digests for both the runner and its standalone telemetry module. It backs up the
|
||||
mounted files, restarts the same container, accepts only a ready health document with
|
||||
stage metrics and a ready native journal transport, and restores the predecessor on
|
||||
failure. Update the token-stdin launcher separately with
|
||||
`Update-NdcMissionCorePersistentLauncher.ps1`; this keeps the operational
|
||||
contour/agent/node identity explicit and reviewable without changing the K1 command
|
||||
sequence.
|
||||
digests for the runner, its `e15_shadow_runtime.py` companion and its standalone
|
||||
telemetry module. The three files form one deployment unit: updating the runner
|
||||
without the matching runtime is rejected by the digest-gated transaction. It backs
|
||||
up the mounted files, restarts the same container, accepts only a ready health
|
||||
document with stage metrics and a ready native journal transport, and restores the
|
||||
predecessor set on failure. Update the token-stdin launcher separately with
|
||||
`Update-NdcMissionCorePersistentLauncher.ps1`. The launcher update always installs
|
||||
and digest-checks `MissionCoreDockerNames.ps1` in the same transaction because the
|
||||
launcher dot-sources that companion contract before any worker request. Both files
|
||||
are restored (or a previously absent companion is removed) if validation fails. This
|
||||
keeps the operational contour/agent/node identity explicit and reviewable without
|
||||
changing the K1 command sequence.
|
||||
|
||||
The stack and agent are intentionally not started by repository tests. Provisioning a
|
||||
machine is a separate, explicit operation.
|
||||
|
||||
@@ -11,6 +11,12 @@ param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ExpectedPipelineTelemetrySha256,
|
||||
[string]$ExpectedPipelineTelemetryPredecessorSha256 = "absent",
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RuntimeCandidate,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ExpectedRuntimeSha256,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ExpectedRuntimePredecessorSha256,
|
||||
[string]$ContainerName = "ndc-mission-core-perception-worker",
|
||||
[string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002"
|
||||
)
|
||||
@@ -18,10 +24,13 @@ param(
|
||||
$ErrorActionPreference = "Stop"
|
||||
$target = Join-Path $RunnerRoot "run_e15_shadow_inference.py"
|
||||
$telemetryTarget = Join-Path $RunnerRoot "pipeline_telemetry.py"
|
||||
$runtimeTarget = Join-Path $RunnerRoot "e15_shadow_runtime.py"
|
||||
$expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant()
|
||||
$expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant()
|
||||
$expectedTelemetry = $ExpectedPipelineTelemetrySha256.ToLowerInvariant()
|
||||
$expectedTelemetryPredecessor = $ExpectedPipelineTelemetryPredecessorSha256.ToLowerInvariant()
|
||||
$expectedRuntime = $ExpectedRuntimeSha256.ToLowerInvariant()
|
||||
$expectedRuntimePredecessor = $ExpectedRuntimePredecessorSha256.ToLowerInvariant()
|
||||
$predecessor = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $target
|
||||
).Hash.ToLowerInvariant()
|
||||
@@ -50,6 +59,18 @@ else {
|
||||
if ($telemetryPredecessor -ne $expectedTelemetryPredecessor) {
|
||||
throw "Pipeline telemetry predecessor digest changed"
|
||||
}
|
||||
$runtimePredecessor = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget
|
||||
).Hash.ToLowerInvariant()
|
||||
if ($runtimePredecessor -ne $expectedRuntimePredecessor) {
|
||||
throw "E15 runtime predecessor digest changed"
|
||||
}
|
||||
$runtimeCandidateDigest = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $RuntimeCandidate
|
||||
).Hash.ToLowerInvariant()
|
||||
if ($runtimeCandidateDigest -ne $expectedRuntime) {
|
||||
throw "E15 runtime candidate digest changed"
|
||||
}
|
||||
$backup = Join-Path $RunnerRoot (
|
||||
"run_e15_shadow_inference.py.rollback-" +
|
||||
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
|
||||
@@ -62,6 +83,12 @@ $telemetryBackup = Join-Path $RunnerRoot (
|
||||
"-" +
|
||||
$telemetryPredecessor.Substring(0, [Math]::Min(12, $telemetryPredecessor.Length))
|
||||
)
|
||||
$runtimeBackup = Join-Path $RunnerRoot (
|
||||
"e15_shadow_runtime.py.rollback-" +
|
||||
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
|
||||
"-" +
|
||||
$runtimePredecessor.Substring(0, 12)
|
||||
)
|
||||
$healthCode = "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:18020/health',timeout=2).read().decode())"
|
||||
|
||||
function Wait-PerceptionHealth {
|
||||
@@ -100,7 +127,10 @@ Copy-Item -LiteralPath $target -Destination $backup
|
||||
if ($hadTelemetryModule) {
|
||||
Copy-Item -LiteralPath $telemetryTarget -Destination $telemetryBackup
|
||||
}
|
||||
Copy-Item -LiteralPath $runtimeTarget -Destination $runtimeBackup
|
||||
try {
|
||||
Copy-Item -LiteralPath $RuntimeCandidate `
|
||||
-Destination $runtimeTarget -Force
|
||||
Copy-Item -LiteralPath $PipelineTelemetryCandidate `
|
||||
-Destination $telemetryTarget -Force
|
||||
Copy-Item -LiteralPath $Candidate -Destination $target -Force
|
||||
@@ -116,6 +146,12 @@ try {
|
||||
) {
|
||||
throw "Pipeline telemetry replacement digest changed"
|
||||
}
|
||||
if (
|
||||
(Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget).Hash.ToLowerInvariant() `
|
||||
-ne $expectedRuntime
|
||||
) {
|
||||
throw "E15 runtime replacement digest changed"
|
||||
}
|
||||
docker restart --time 20 $ContainerName | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Perception container restart failed"
|
||||
@@ -127,6 +163,7 @@ try {
|
||||
}
|
||||
catch {
|
||||
Copy-Item -LiteralPath $backup -Destination $target -Force
|
||||
Copy-Item -LiteralPath $runtimeBackup -Destination $runtimeTarget -Force
|
||||
if ($hadTelemetryModule) {
|
||||
Copy-Item -LiteralPath $telemetryBackup -Destination $telemetryTarget -Force
|
||||
}
|
||||
@@ -153,6 +190,11 @@ catch {
|
||||
PipelineTelemetryCandidateSha256 = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget
|
||||
).Hash.ToLowerInvariant()
|
||||
RuntimePredecessorSha256 = $runtimePredecessor
|
||||
RuntimeCandidateSha256 = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget
|
||||
).Hash.ToLowerInvariant()
|
||||
RuntimeBackup = $runtimeBackup
|
||||
PipelineTelemetryBackup = if ($hadTelemetryModule) { $telemetryBackup } else { $null }
|
||||
Backup = $backup
|
||||
Health = $health
|
||||
|
||||
@@ -6,13 +6,23 @@ param(
|
||||
[string]$ExpectedPredecessorSha256,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ExpectedCandidateSha256,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$DockerNamesCandidate,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ExpectedDockerNamesSha256,
|
||||
[string]$ExpectedDockerNamesPredecessorSha256 = "absent",
|
||||
[string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$target = Join-Path $RunnerRoot "Invoke-E15PersistentShadowRun.ps1"
|
||||
$dockerNamesTarget = Join-Path $RunnerRoot "MissionCoreDockerNames.ps1"
|
||||
$expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant()
|
||||
$expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant()
|
||||
$expectedDockerNames = $ExpectedDockerNamesSha256.ToLowerInvariant()
|
||||
$expectedDockerNamesPredecessor = (
|
||||
$ExpectedDockerNamesPredecessorSha256.ToLowerInvariant()
|
||||
)
|
||||
$predecessor = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $target
|
||||
).Hash.ToLowerInvariant()
|
||||
@@ -25,15 +35,47 @@ if (
|
||||
) {
|
||||
throw "Persistent launcher candidate digest changed"
|
||||
}
|
||||
$dockerNamesCandidateDigest = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $DockerNamesCandidate
|
||||
).Hash.ToLowerInvariant()
|
||||
if ($dockerNamesCandidateDigest -ne $expectedDockerNames) {
|
||||
throw "Docker names candidate digest changed"
|
||||
}
|
||||
$hadDockerNames = Test-Path -LiteralPath $dockerNamesTarget -PathType Leaf
|
||||
$dockerNamesPredecessor = if ($hadDockerNames) {
|
||||
(
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $dockerNamesTarget
|
||||
).Hash.ToLowerInvariant()
|
||||
}
|
||||
else {
|
||||
"absent"
|
||||
}
|
||||
if ($dockerNamesPredecessor -ne $expectedDockerNamesPredecessor) {
|
||||
throw "Docker names predecessor digest changed"
|
||||
}
|
||||
$backup = Join-Path $RunnerRoot (
|
||||
"Invoke-E15PersistentShadowRun.ps1.rollback-" +
|
||||
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
|
||||
"-" +
|
||||
$predecessor.Substring(0, 12)
|
||||
)
|
||||
$dockerNamesBackup = Join-Path $RunnerRoot (
|
||||
"MissionCoreDockerNames.ps1.rollback-" +
|
||||
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
|
||||
"-" +
|
||||
$dockerNamesPredecessor.Substring(
|
||||
0,
|
||||
[Math]::Min(12, $dockerNamesPredecessor.Length)
|
||||
)
|
||||
)
|
||||
|
||||
Copy-Item -LiteralPath $target -Destination $backup
|
||||
if ($hadDockerNames) {
|
||||
Copy-Item -LiteralPath $dockerNamesTarget -Destination $dockerNamesBackup
|
||||
}
|
||||
try {
|
||||
Copy-Item -LiteralPath $DockerNamesCandidate `
|
||||
-Destination $dockerNamesTarget -Force
|
||||
Copy-Item -LiteralPath $Candidate -Destination $target -Force
|
||||
$installed = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $target
|
||||
@@ -41,19 +83,35 @@ try {
|
||||
if ($installed -ne $expectedCandidate) {
|
||||
throw "Persistent launcher replacement digest changed"
|
||||
}
|
||||
$tokens = $null
|
||||
$parseErrors = $null
|
||||
[Management.Automation.Language.Parser]::ParseFile(
|
||||
$target,
|
||||
[ref]$tokens,
|
||||
[ref]$parseErrors
|
||||
) | Out-Null
|
||||
if (@($parseErrors).Count -ne 0) {
|
||||
throw "Persistent launcher candidate has PowerShell parse errors"
|
||||
$installedDockerNames = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $dockerNamesTarget
|
||||
).Hash.ToLowerInvariant()
|
||||
if ($installedDockerNames -ne $expectedDockerNames) {
|
||||
throw "Docker names replacement digest changed"
|
||||
}
|
||||
foreach ($scriptPath in @($dockerNamesTarget, $target)) {
|
||||
$tokens = $null
|
||||
$parseErrors = $null
|
||||
[Management.Automation.Language.Parser]::ParseFile(
|
||||
$scriptPath,
|
||||
[ref]$tokens,
|
||||
[ref]$parseErrors
|
||||
) | Out-Null
|
||||
if (@($parseErrors).Count -ne 0) {
|
||||
throw "Persistent launcher deployment has PowerShell parse errors"
|
||||
}
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Copy-Item -LiteralPath $backup -Destination $target -Force
|
||||
if ($hadDockerNames) {
|
||||
Copy-Item -LiteralPath $dockerNamesBackup `
|
||||
-Destination $dockerNamesTarget -Force
|
||||
}
|
||||
else {
|
||||
Remove-Item -LiteralPath $dockerNamesTarget `
|
||||
-Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
throw
|
||||
}
|
||||
|
||||
@@ -63,5 +121,10 @@ catch {
|
||||
CandidateSha256 = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $target
|
||||
).Hash.ToLowerInvariant()
|
||||
DockerNamesPredecessorSha256 = $dockerNamesPredecessor
|
||||
DockerNamesCandidateSha256 = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $dockerNamesTarget
|
||||
).Hash.ToLowerInvariant()
|
||||
DockerNamesBackup = if ($hadDockerNames) { $dockerNamesBackup } else { $null }
|
||||
Backup = $backup
|
||||
} | ConvertTo-Json -Compress
|
||||
|
||||
@@ -11,7 +11,7 @@ Each gate produces evidence and an explicit GO, PAUSE or BLOCKED result.
|
||||
| Prediction/evaluation boundary | GO — truth-free E41 predictor package/result are physically separate from E37 and reproduce E40 predictions exactly; the visible evaluator runs only after inference. |
|
||||
| Source-scoped quality | PAUSE — E40 visible engineering evaluation is 84.2466% presence, 84.2466% geometry association and 94.5205% freshness, with 11 high-severity failures, complete accounting and zero false-free claims. This is not a blind accuracy gate. |
|
||||
| Structural regressions | GO (bounded) — E42 passes predictor identity/order/chunk invariance and PointSlab order/SE(3) checks. Raw-producer and cross-route invariance remain unproved. |
|
||||
| Native pipeline telemetry | GO for producer/normalizer contract — lifecycle events and JSONL/MQTT sink boundaries exist and an E41 smoke run records real stage accounting. Durable Worker 006 MQTT wiring/deployment remains pending. |
|
||||
| Native pipeline telemetry | GO (durable Worker 006) — lifecycle events and JSONL/MQTT sink boundaries exist; the persistent worker writes to its existing D:-backed publish mount, host Telegraf tails with saved offset and publishes through the existing authenticated QoS 1 path. Real request `durable-native-acceptance-20260729-115418` produced native run `started → failed` after reaching the unavailable live-source boundary, and both documents were observed through broker → normalizer → Timescale query API with the same run/request identity. No broker credential or additional container entered the inference runtime. |
|
||||
| Evidence storage | MEASURED — E44 finds 525,471,092 logical bytes, 312,753,179 unique-content bytes and 1.680146× amplification across 14 E30–E40 roots. Exact-content references/deduplication precede any format migration. |
|
||||
| Future transfer | PREREGISTERED — E43 freezes same-K1/mount/calibration/firmware, required streams, connected-component split and independent label reveal. Capture and labels do not yet exist. |
|
||||
| E31 binding sensitivity | MEASURED — E45 closes accounting for 87/87 accepted correspondences and finds no material monotonic residual association with represented image radius, rig speed or pose age. It does not supply calibration-target truth or outer-fisheye coverage. |
|
||||
|
||||
@@ -29,13 +29,26 @@ The existing Worker 006 containers were renamed in place to
|
||||
container identities were preserved, so this namespace migration did not restart the
|
||||
inference or perception runtimes.
|
||||
|
||||
The compute-side native pipeline contract is now implemented. It emits
|
||||
`missioncore.agent-pipeline-telemetry/v1` lifecycle documents through an injected
|
||||
transport, and the telemetry normalizer preserves source, method, stage and stage-state
|
||||
identities in the normalized series key. The E41 runner has exercised the JSONL
|
||||
evidence sink against the immutable predictor package. This proves producer and
|
||||
normalizer compatibility; it does not claim that the durable Worker 006 process has
|
||||
been wired to the MQTT sink or deployed with this code.
|
||||
The compute-side native pipeline contract is implemented and deployed on Worker 006.
|
||||
It emits `missioncore.agent-pipeline-telemetry/v1` lifecycle documents through an
|
||||
injected transport, and the telemetry normalizer preserves source, method, stage and
|
||||
stage-state identities in the normalized series key. The E41 runner first exercised
|
||||
the JSONL evidence sink against the immutable predictor package.
|
||||
|
||||
The durable path was then accepted on 2026-07-29 with real persistent-worker request
|
||||
`durable-native-acceptance-20260729-115418`. The request reached the actual live-source
|
||||
boundary and terminated because no K1 shadow source was open. Its native run
|
||||
`started` and `failed` documents were appended by the worker, tailed by host Telegraf,
|
||||
published on the contour pipeline topic and returned by the normalized Timescale query
|
||||
API with the same run/request identity. This is transport and lifecycle acceptance,
|
||||
not a successful perception-result or stage-performance claim.
|
||||
|
||||
The acceptance also closed two deployment-unit defects found by the real request:
|
||||
the token-stdin launcher now installs its required `MissionCoreDockerNames.ps1`
|
||||
companion, and the perception runner updater atomically digest-gates
|
||||
`run_e15_shadow_inference.py`, `e15_shadow_runtime.py` and
|
||||
`pipeline_telemetry.py`. The existing perception container is still the only
|
||||
inference worker; broker credentials remain on the host Telegraf service.
|
||||
|
||||
## Decision
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
from pathlib import Path
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def test_persistent_launcher_deploys_required_docker_names_companion() -> None:
|
||||
deployment = (
|
||||
REPOSITORY_ROOT
|
||||
/ "deploy"
|
||||
/ "telemetry-plane"
|
||||
/ "telegraf"
|
||||
/ "Update-NdcMissionCorePersistentLauncher.ps1"
|
||||
).read_text(encoding="utf-8")
|
||||
launcher = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "Invoke-E15PersistentShadowRun.ps1"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert '. (Join-Path $PSScriptRoot "MissionCoreDockerNames.ps1")' in launcher
|
||||
assert "[string]$DockerNamesCandidate" in deployment
|
||||
assert "[string]$ExpectedDockerNamesSha256" in deployment
|
||||
assert "[string]$ExpectedDockerNamesPredecessorSha256" in deployment
|
||||
assert '$dockerNamesPredecessor = if ($hadDockerNames)' in deployment
|
||||
assert "Copy-Item -LiteralPath $DockerNamesCandidate" in deployment
|
||||
assert "Remove-Item -LiteralPath $dockerNamesTarget" in deployment
|
||||
|
||||
|
||||
def test_perception_runner_deploys_required_runtime_companion() -> None:
|
||||
deployment = (
|
||||
REPOSITORY_ROOT
|
||||
/ "deploy"
|
||||
/ "telemetry-plane"
|
||||
/ "telegraf"
|
||||
/ "Update-NdcMissionCorePerceptionRunner.ps1"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "[string]$RuntimeCandidate" in deployment
|
||||
assert "[string]$ExpectedRuntimeSha256" in deployment
|
||||
assert "[string]$ExpectedRuntimePredecessorSha256" in deployment
|
||||
assert '$runtimeTarget = Join-Path $RunnerRoot "e15_shadow_runtime.py"' in deployment
|
||||
assert "Copy-Item -LiteralPath $RuntimeCandidate" in deployment
|
||||
assert "Copy-Item -LiteralPath $runtimeBackup" in deployment
|
||||
assert "RuntimeCandidateSha256" in deployment
|
||||
Reference in New Issue
Block a user