202 lines
7.4 KiB
PowerShell
202 lines
7.4 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$Candidate,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ExpectedPredecessorSha256,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ExpectedCandidateSha256,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$PipelineTelemetryCandidate,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ExpectedPipelineTelemetrySha256,
|
|
[string]$ExpectedPipelineTelemetryPredecessorSha256 = "absent",
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$RuntimeCandidate,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ExpectedRuntimeSha256,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ExpectedRuntimePredecessorSha256,
|
|
[string]$ContainerName = "ndc-mission-core-perception-worker",
|
|
[string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$target = Join-Path $RunnerRoot "run_e15_shadow_inference.py"
|
|
$telemetryTarget = Join-Path $RunnerRoot "pipeline_telemetry.py"
|
|
$runtimeTarget = Join-Path $RunnerRoot "e15_shadow_runtime.py"
|
|
$expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant()
|
|
$expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant()
|
|
$expectedTelemetry = $ExpectedPipelineTelemetrySha256.ToLowerInvariant()
|
|
$expectedTelemetryPredecessor = $ExpectedPipelineTelemetryPredecessorSha256.ToLowerInvariant()
|
|
$expectedRuntime = $ExpectedRuntimeSha256.ToLowerInvariant()
|
|
$expectedRuntimePredecessor = $ExpectedRuntimePredecessorSha256.ToLowerInvariant()
|
|
$predecessor = (
|
|
Get-FileHash -Algorithm SHA256 -LiteralPath $target
|
|
).Hash.ToLowerInvariant()
|
|
if ($predecessor -ne $expectedPredecessor) {
|
|
throw "Runner predecessor digest changed"
|
|
}
|
|
if (
|
|
(Get-FileHash -Algorithm SHA256 -LiteralPath $Candidate).Hash.ToLowerInvariant() `
|
|
-ne $expectedCandidate
|
|
) {
|
|
throw "Runner candidate digest changed"
|
|
}
|
|
$telemetryCandidateDigest = (
|
|
Get-FileHash -Algorithm SHA256 -LiteralPath $PipelineTelemetryCandidate
|
|
).Hash.ToLowerInvariant()
|
|
if ($telemetryCandidateDigest -ne $expectedTelemetry) {
|
|
throw "Pipeline telemetry candidate digest changed"
|
|
}
|
|
$hadTelemetryModule = Test-Path -LiteralPath $telemetryTarget -PathType Leaf
|
|
$telemetryPredecessor = if ($hadTelemetryModule) {
|
|
(Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget).Hash.ToLowerInvariant()
|
|
}
|
|
else {
|
|
"absent"
|
|
}
|
|
if ($telemetryPredecessor -ne $expectedTelemetryPredecessor) {
|
|
throw "Pipeline telemetry predecessor digest changed"
|
|
}
|
|
$runtimePredecessor = (
|
|
Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget
|
|
).Hash.ToLowerInvariant()
|
|
if ($runtimePredecessor -ne $expectedRuntimePredecessor) {
|
|
throw "E15 runtime predecessor digest changed"
|
|
}
|
|
$runtimeCandidateDigest = (
|
|
Get-FileHash -Algorithm SHA256 -LiteralPath $RuntimeCandidate
|
|
).Hash.ToLowerInvariant()
|
|
if ($runtimeCandidateDigest -ne $expectedRuntime) {
|
|
throw "E15 runtime candidate digest changed"
|
|
}
|
|
$backup = Join-Path $RunnerRoot (
|
|
"run_e15_shadow_inference.py.rollback-" +
|
|
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
|
|
"-" +
|
|
$predecessor.Substring(0, 12)
|
|
)
|
|
$telemetryBackup = Join-Path $RunnerRoot (
|
|
"pipeline_telemetry.py.rollback-" +
|
|
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
|
|
"-" +
|
|
$telemetryPredecessor.Substring(0, [Math]::Min(12, $telemetryPredecessor.Length))
|
|
)
|
|
$runtimeBackup = Join-Path $RunnerRoot (
|
|
"e15_shadow_runtime.py.rollback-" +
|
|
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
|
|
"-" +
|
|
$runtimePredecessor.Substring(0, 12)
|
|
)
|
|
$healthCode = "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:18020/health',timeout=2).read().decode())"
|
|
|
|
function Wait-PerceptionHealth {
|
|
param(
|
|
[bool]$RequireStageMetrics,
|
|
[int]$TimeoutSeconds = 150
|
|
)
|
|
|
|
$deadline = [DateTime]::UtcNow.AddSeconds($TimeoutSeconds)
|
|
while ([DateTime]::UtcNow -lt $deadline) {
|
|
Start-Sleep -Seconds 3
|
|
try {
|
|
$healthJson = docker exec $ContainerName python3 -c $healthCode 2>$null
|
|
if ($LASTEXITCODE -ne 0 -or -not $healthJson) {
|
|
continue
|
|
}
|
|
$health = $healthJson | ConvertFrom-Json
|
|
if (
|
|
$health.ok -and
|
|
$health.state -eq "ready" -and
|
|
(-not $RequireStageMetrics -or (
|
|
$health.stage_metrics -and
|
|
$health.native_pipeline_telemetry -and
|
|
$health.native_pipeline_telemetry.ready
|
|
))
|
|
) {
|
|
return $health
|
|
}
|
|
}
|
|
catch {}
|
|
}
|
|
return $null
|
|
}
|
|
|
|
Copy-Item -LiteralPath $target -Destination $backup
|
|
if ($hadTelemetryModule) {
|
|
Copy-Item -LiteralPath $telemetryTarget -Destination $telemetryBackup
|
|
}
|
|
Copy-Item -LiteralPath $runtimeTarget -Destination $runtimeBackup
|
|
try {
|
|
Copy-Item -LiteralPath $RuntimeCandidate `
|
|
-Destination $runtimeTarget -Force
|
|
Copy-Item -LiteralPath $PipelineTelemetryCandidate `
|
|
-Destination $telemetryTarget -Force
|
|
Copy-Item -LiteralPath $Candidate -Destination $target -Force
|
|
if (
|
|
(Get-FileHash -Algorithm SHA256 -LiteralPath $target).Hash.ToLowerInvariant() `
|
|
-ne $expectedCandidate
|
|
) {
|
|
throw "Runner replacement digest changed"
|
|
}
|
|
if (
|
|
(Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget).Hash.ToLowerInvariant() `
|
|
-ne $expectedTelemetry
|
|
) {
|
|
throw "Pipeline telemetry replacement digest changed"
|
|
}
|
|
if (
|
|
(Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget).Hash.ToLowerInvariant() `
|
|
-ne $expectedRuntime
|
|
) {
|
|
throw "E15 runtime replacement digest changed"
|
|
}
|
|
docker restart --time 20 $ContainerName | Out-Null
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Perception container restart failed"
|
|
}
|
|
$health = Wait-PerceptionHealth -RequireStageMetrics $true
|
|
if (-not $health) {
|
|
throw "Stage-instrumented runner did not become ready"
|
|
}
|
|
}
|
|
catch {
|
|
Copy-Item -LiteralPath $backup -Destination $target -Force
|
|
Copy-Item -LiteralPath $runtimeBackup -Destination $runtimeTarget -Force
|
|
if ($hadTelemetryModule) {
|
|
Copy-Item -LiteralPath $telemetryBackup -Destination $telemetryTarget -Force
|
|
}
|
|
else {
|
|
Remove-Item -LiteralPath $telemetryTarget -Force -ErrorAction SilentlyContinue
|
|
}
|
|
docker restart --time 20 $ContainerName | Out-Null
|
|
$rollbackHealth = Wait-PerceptionHealth -RequireStageMetrics $false
|
|
if (-not $rollbackHealth) {
|
|
throw "Candidate failed and predecessor rollback did not become ready"
|
|
}
|
|
throw
|
|
}
|
|
|
|
[ordered]@{
|
|
SchemaVersion = "missioncore.worker-runner-deploy-result/v1"
|
|
Container = $ContainerName
|
|
State = (docker inspect --format "{{.State.Status}}" $ContainerName)
|
|
PredecessorSha256 = $predecessor
|
|
CandidateSha256 = (
|
|
Get-FileHash -Algorithm SHA256 -LiteralPath $target
|
|
).Hash.ToLowerInvariant()
|
|
PipelineTelemetryPredecessorSha256 = $telemetryPredecessor
|
|
PipelineTelemetryCandidateSha256 = (
|
|
Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget
|
|
).Hash.ToLowerInvariant()
|
|
RuntimePredecessorSha256 = $runtimePredecessor
|
|
RuntimeCandidateSha256 = (
|
|
Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeTarget
|
|
).Hash.ToLowerInvariant()
|
|
RuntimeBackup = $runtimeBackup
|
|
PipelineTelemetryBackup = if ($hadTelemetryModule) { $telemetryBackup } else { $null }
|
|
Backup = $backup
|
|
Health = $health
|
|
} | ConvertTo-Json -Depth 12 -Compress
|