fix(worker): deploy persistent runtime companions

2026-07-29 14:57:34 +03:00
parent 9db39347bd
commit 611853d3af
6 changed files with 193 additions and 24 deletions
@@ -136,13 +136,18 @@ change inference control flow.

 When the mounted perception runner itself changes, use
 `Update-NdcMissionCorePerceptionRunner.ps1` with exact predecessor and candidate
-digests for both the runner and its standalone telemetry module. It backs up the
-mounted files, restarts the same container, accepts only a ready health document with
-stage metrics and a ready native journal transport, and restores the predecessor on
-failure. Update the token-stdin launcher separately with
-`Update-NdcMissionCorePersistentLauncher.ps1`; this keeps the operational
-contour/agent/node identity explicit and reviewable without changing the K1 command
-sequence.
+digests for the runner, its `e15_shadow_runtime.py` companion and its standalone
+telemetry module. The three files form one deployment unit: updating the runner
+without the matching runtime is rejected by the digest-gated transaction. It backs
+up the mounted files, restarts the same container, accepts only a ready health
+document with stage metrics and a ready native journal transport, and restores the
+predecessor set on failure. Update the token-stdin launcher separately with
+`Update-NdcMissionCorePersistentLauncher.ps1`. The launcher update always installs
+and digest-checks `MissionCoreDockerNames.ps1` in the same transaction because the
+launcher dot-sources that companion contract before any worker request. Both files
+are restored (or a previously absent companion is removed) if validation fails. This
+keeps the operational contour/agent/node identity explicit and reviewable without
+changing the K1 command sequence.

 The stack and agent are intentionally not started by repository tests. Provisioning a
 machine is a separate, explicit operation.
@@ -11,6 +11,12 @@ param(
    [Parameter(Mandatory = $true)]
    [string]$ExpectedPipelineTelemetrySha256,
    [string]$ExpectedPipelineTelemetryPredecessorSha256 = "absent",
+    [Parameter(Mandatory = $true)]
+    [string]$RuntimeCandidate,
+    [Parameter(Mandatory = $true)]
+    [string]$ExpectedRuntimeSha256,
+    [Parameter(Mandatory = $true)]
+    [string]$ExpectedRuntimePredecessorSha256,
    [string]$ContainerName = "ndc-mission-core-perception-worker",
    [string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002"
 )
@@ -18,10 +24,13 @@ param(
 $ErrorActionPreference = "Stop"
 $target = Join-Path $RunnerRoot "run_e15_shadow_inference.py"
 $telemetryTarget = Join-Path $RunnerRoot "pipeline_telemetry.py"
+$runtimeTarget = Join-Path $RunnerRoot "e15_shadow_runtime.py"
 $expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant()
 $expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant()
 $expectedTelemetry = $ExpectedPipelineTelemetrySha256.ToLowerInvariant()
 $expectedTelemetryPredecessor = $ExpectedPipelineTelemetryPredecessorSha256.ToLowerInvariant()
+$expectedRuntime = $ExpectedRuntimeSha256.ToLowerInvariant()
+$expectedRuntimePredecessor = $ExpectedRuntimePredecessorSha256.ToLowerInvariant()
 $predecessor = (
    Get-FileHash -Algorithm SHA256 -LiteralPath $target
 ).Hash.ToLowerInvariant()
@@ -50,6 +59,18 @@ else {
 if ($telemetryPredecessor -ne $expectedTelemetryPredecessor) {
    throw "Pipeline telemetry predecessor digest changed"
 }
+$runtimePredecessor = (
+    Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget
+).Hash.ToLowerInvariant()
+if ($runtimePredecessor -ne $expectedRuntimePredecessor) {
+    throw "E15 runtime predecessor digest changed"
+}
+$runtimeCandidateDigest = (
+    Get-FileHash -Algorithm SHA256 -LiteralPath $RuntimeCandidate
+).Hash.ToLowerInvariant()
+if ($runtimeCandidateDigest -ne $expectedRuntime) {
+    throw "E15 runtime candidate digest changed"
+}
 $backup = Join-Path $RunnerRoot (
    "run_e15_shadow_inference.py.rollback-" +
    [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
@@ -62,6 +83,12 @@ $telemetryBackup = Join-Path $RunnerRoot (
    "-" +
    $telemetryPredecessor.Substring(0, [Math]::Min(12, $telemetryPredecessor.Length))
 )
+$runtimeBackup = Join-Path $RunnerRoot (
+    "e15_shadow_runtime.py.rollback-" +
+    [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
+    "-" +
+    $runtimePredecessor.Substring(0, 12)
+)
 $healthCode = "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:18020/health',timeout=2).read().decode())"

 function Wait-PerceptionHealth {
@@ -100,7 +127,10 @@ Copy-Item -LiteralPath $target -Destination $backup
 if ($hadTelemetryModule) {
    Copy-Item -LiteralPath $telemetryTarget -Destination $telemetryBackup
 }
+Copy-Item -LiteralPath $runtimeTarget -Destination $runtimeBackup
 try {
+    Copy-Item -LiteralPath $RuntimeCandidate `
+        -Destination $runtimeTarget -Force
    Copy-Item -LiteralPath $PipelineTelemetryCandidate `
        -Destination $telemetryTarget -Force
    Copy-Item -LiteralPath $Candidate -Destination $target -Force
@@ -116,6 +146,12 @@ try {
    ) {
        throw "Pipeline telemetry replacement digest changed"
    }
+    if (
+        (Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget).Hash.ToLowerInvariant() `
+            -ne $expectedRuntime
+    ) {
+        throw "E15 runtime replacement digest changed"
+    }
    docker restart --time 20 $ContainerName | Out-Null
    if ($LASTEXITCODE -ne 0) {
        throw "Perception container restart failed"
@@ -127,6 +163,7 @@ try {
 }
 catch {
    Copy-Item -LiteralPath $backup -Destination $target -Force
+    Copy-Item -LiteralPath $runtimeBackup -Destination $runtimeTarget -Force
    if ($hadTelemetryModule) {
        Copy-Item -LiteralPath $telemetryBackup -Destination $telemetryTarget -Force
    }
@@ -153,6 +190,11 @@ catch {
    PipelineTelemetryCandidateSha256 = (
        Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget
    ).Hash.ToLowerInvariant()
+    RuntimePredecessorSha256 = $runtimePredecessor
+    RuntimeCandidateSha256 = (
+        Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget
+    ).Hash.ToLowerInvariant()
+    RuntimeBackup = $runtimeBackup
    PipelineTelemetryBackup = if ($hadTelemetryModule) { $telemetryBackup } else { $null }
    Backup = $backup
    Health = $health
@@ -6,13 +6,23 @@ param(
    [string]$ExpectedPredecessorSha256,
    [Parameter(Mandatory = $true)]
    [string]$ExpectedCandidateSha256,
+    [Parameter(Mandatory = $true)]
+    [string]$DockerNamesCandidate,
+    [Parameter(Mandatory = $true)]
+    [string]$ExpectedDockerNamesSha256,
+    [string]$ExpectedDockerNamesPredecessorSha256 = "absent",
    [string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002"
 )

 $ErrorActionPreference = "Stop"
 $target = Join-Path $RunnerRoot "Invoke-E15PersistentShadowRun.ps1"
+$dockerNamesTarget = Join-Path $RunnerRoot "MissionCoreDockerNames.ps1"
 $expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant()
 $expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant()
+$expectedDockerNames = $ExpectedDockerNamesSha256.ToLowerInvariant()
+$expectedDockerNamesPredecessor = (
+    $ExpectedDockerNamesPredecessorSha256.ToLowerInvariant()
+)
 $predecessor = (
    Get-FileHash -Algorithm SHA256 -LiteralPath $target
 ).Hash.ToLowerInvariant()
@@ -25,15 +35,47 @@ if (
 ) {
    throw "Persistent launcher candidate digest changed"
 }
+$dockerNamesCandidateDigest = (
+    Get-FileHash -Algorithm SHA256 -LiteralPath $DockerNamesCandidate
+).Hash.ToLowerInvariant()
+if ($dockerNamesCandidateDigest -ne $expectedDockerNames) {
+    throw "Docker names candidate digest changed"
+}
+$hadDockerNames = Test-Path -LiteralPath $dockerNamesTarget -PathType Leaf
+$dockerNamesPredecessor = if ($hadDockerNames) {
+    (
+        Get-FileHash -Algorithm SHA256 -LiteralPath $dockerNamesTarget
+    ).Hash.ToLowerInvariant()
+}
+else {
+    "absent"
+}
+if ($dockerNamesPredecessor -ne $expectedDockerNamesPredecessor) {
+    throw "Docker names predecessor digest changed"
+}
 $backup = Join-Path $RunnerRoot (
    "Invoke-E15PersistentShadowRun.ps1.rollback-" +
    [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
    "-" +
    $predecessor.Substring(0, 12)
 )
+$dockerNamesBackup = Join-Path $RunnerRoot (
+    "MissionCoreDockerNames.ps1.rollback-" +
+    [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
+    "-" +
+    $dockerNamesPredecessor.Substring(
+        0,
+        [Math]::Min(12, $dockerNamesPredecessor.Length)
+    )
+)

 Copy-Item -LiteralPath $target -Destination $backup
+if ($hadDockerNames) {
+    Copy-Item -LiteralPath $dockerNamesTarget -Destination $dockerNamesBackup
+}
 try {
+    Copy-Item -LiteralPath $DockerNamesCandidate `
+        -Destination $dockerNamesTarget -Force
    Copy-Item -LiteralPath $Candidate -Destination $target -Force
    $installed = (
        Get-FileHash -Algorithm SHA256 -LiteralPath $target
@@ -41,19 +83,35 @@ try {
    if ($installed -ne $expectedCandidate) {
        throw "Persistent launcher replacement digest changed"
    }
-    $tokens = $null
-    $parseErrors = $null
-    [Management.Automation.Language.Parser]::ParseFile(
-        $target,
-        [ref]$tokens,
-        [ref]$parseErrors
-    ) | Out-Null
-    if (@($parseErrors).Count -ne 0) {
-        throw "Persistent launcher candidate has PowerShell parse errors"
+    $installedDockerNames = (
+        Get-FileHash -Algorithm SHA256 -LiteralPath $dockerNamesTarget
+    ).Hash.ToLowerInvariant()
+    if ($installedDockerNames -ne $expectedDockerNames) {
+        throw "Docker names replacement digest changed"
+    }
+    foreach ($scriptPath in @($dockerNamesTarget, $target)) {
+        $tokens = $null
+        $parseErrors = $null
+        [Management.Automation.Language.Parser]::ParseFile(
+            $scriptPath,
+            [ref]$tokens,
+            [ref]$parseErrors
+        ) | Out-Null
+        if (@($parseErrors).Count -ne 0) {
+            throw "Persistent launcher deployment has PowerShell parse errors"
+        }
    }
 }
 catch {
    Copy-Item -LiteralPath $backup -Destination $target -Force
+    if ($hadDockerNames) {
+        Copy-Item -LiteralPath $dockerNamesBackup `
+            -Destination $dockerNamesTarget -Force
+    }
+    else {
+        Remove-Item -LiteralPath $dockerNamesTarget `
+            -Force -ErrorAction SilentlyContinue
+    }
    throw
 }

@@ -63,5 +121,10 @@ catch {
    CandidateSha256 = (
        Get-FileHash -Algorithm SHA256 -LiteralPath $target
    ).Hash.ToLowerInvariant()
+    DockerNamesPredecessorSha256 = $dockerNamesPredecessor
+    DockerNamesCandidateSha256 = (
+        Get-FileHash -Algorithm SHA256 -LiteralPath $dockerNamesTarget
+    ).Hash.ToLowerInvariant()
+    DockerNamesBackup = if ($hadDockerNames) { $dockerNamesBackup } else { $null }
    Backup = $backup
 } | ConvertTo-Json -Compress