fix(worker): deploy persistent runtime companions

This commit is contained in:
DCCONSTRUCTIONS
2026-07-29 14:57:34 +03:00
parent 9db39347bd
commit 611853d3af
6 changed files with 193 additions and 24 deletions
@@ -11,6 +11,12 @@ param(
[Parameter(Mandatory = $true)]
[string]$ExpectedPipelineTelemetrySha256,
[string]$ExpectedPipelineTelemetryPredecessorSha256 = "absent",
[Parameter(Mandatory = $true)]
[string]$RuntimeCandidate,
[Parameter(Mandatory = $true)]
[string]$ExpectedRuntimeSha256,
[Parameter(Mandatory = $true)]
[string]$ExpectedRuntimePredecessorSha256,
[string]$ContainerName = "ndc-mission-core-perception-worker",
[string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002"
)
@@ -18,10 +24,13 @@ param(
$ErrorActionPreference = "Stop"
$target = Join-Path $RunnerRoot "run_e15_shadow_inference.py"
$telemetryTarget = Join-Path $RunnerRoot "pipeline_telemetry.py"
$runtimeTarget = Join-Path $RunnerRoot "e15_shadow_runtime.py"
$expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant()
$expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant()
$expectedTelemetry = $ExpectedPipelineTelemetrySha256.ToLowerInvariant()
$expectedTelemetryPredecessor = $ExpectedPipelineTelemetryPredecessorSha256.ToLowerInvariant()
$expectedRuntime = $ExpectedRuntimeSha256.ToLowerInvariant()
$expectedRuntimePredecessor = $ExpectedRuntimePredecessorSha256.ToLowerInvariant()
$predecessor = (
Get-FileHash -Algorithm SHA256 -LiteralPath $target
).Hash.ToLowerInvariant()
@@ -50,6 +59,18 @@ else {
if ($telemetryPredecessor -ne $expectedTelemetryPredecessor) {
throw "Pipeline telemetry predecessor digest changed"
}
$runtimePredecessor = (
Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget
).Hash.ToLowerInvariant()
if ($runtimePredecessor -ne $expectedRuntimePredecessor) {
throw "E15 runtime predecessor digest changed"
}
$runtimeCandidateDigest = (
Get-FileHash -Algorithm SHA256 -LiteralPath $RuntimeCandidate
).Hash.ToLowerInvariant()
if ($runtimeCandidateDigest -ne $expectedRuntime) {
throw "E15 runtime candidate digest changed"
}
$backup = Join-Path $RunnerRoot (
"run_e15_shadow_inference.py.rollback-" +
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
@@ -62,6 +83,12 @@ $telemetryBackup = Join-Path $RunnerRoot (
"-" +
$telemetryPredecessor.Substring(0, [Math]::Min(12, $telemetryPredecessor.Length))
)
$runtimeBackup = Join-Path $RunnerRoot (
"e15_shadow_runtime.py.rollback-" +
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
"-" +
$runtimePredecessor.Substring(0, 12)
)
$healthCode = "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:18020/health',timeout=2).read().decode())"
function Wait-PerceptionHealth {
@@ -100,7 +127,10 @@ Copy-Item -LiteralPath $target -Destination $backup
if ($hadTelemetryModule) {
Copy-Item -LiteralPath $telemetryTarget -Destination $telemetryBackup
}
Copy-Item -LiteralPath $runtimeTarget -Destination $runtimeBackup
try {
Copy-Item -LiteralPath $RuntimeCandidate `
-Destination $runtimeTarget -Force
Copy-Item -LiteralPath $PipelineTelemetryCandidate `
-Destination $telemetryTarget -Force
Copy-Item -LiteralPath $Candidate -Destination $target -Force
@@ -116,6 +146,12 @@ try {
) {
throw "Pipeline telemetry replacement digest changed"
}
if (
(Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget).Hash.ToLowerInvariant() `
-ne $expectedRuntime
) {
throw "E15 runtime replacement digest changed"
}
docker restart --time 20 $ContainerName | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Perception container restart failed"
@@ -127,6 +163,7 @@ try {
}
catch {
Copy-Item -LiteralPath $backup -Destination $target -Force
Copy-Item -LiteralPath $runtimeBackup -Destination $runtimeTarget -Force
if ($hadTelemetryModule) {
Copy-Item -LiteralPath $telemetryBackup -Destination $telemetryTarget -Force
}
@@ -153,6 +190,11 @@ catch {
PipelineTelemetryCandidateSha256 = (
Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget
).Hash.ToLowerInvariant()
RuntimePredecessorSha256 = $runtimePredecessor
RuntimeCandidateSha256 = (
Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget
).Hash.ToLowerInvariant()
RuntimeBackup = $runtimeBackup
PipelineTelemetryBackup = if ($hadTelemetryModule) { $telemetryBackup } else { $null }
Backup = $backup
Health = $health