feat(telemetry): add bounded native pipeline lifecycle
This commit is contained in:
@@ -6,14 +6,22 @@ param(
|
||||
[string]$ExpectedPredecessorSha256,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ExpectedCandidateSha256,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$PipelineTelemetryCandidate,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ExpectedPipelineTelemetrySha256,
|
||||
[string]$ExpectedPipelineTelemetryPredecessorSha256 = "absent",
|
||||
[string]$ContainerName = "ndc-mission-core-perception-worker",
|
||||
[string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$target = Join-Path $RunnerRoot "run_e15_shadow_inference.py"
|
||||
$telemetryTarget = Join-Path $RunnerRoot "pipeline_telemetry.py"
|
||||
$expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant()
|
||||
$expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant()
|
||||
$expectedTelemetry = $ExpectedPipelineTelemetrySha256.ToLowerInvariant()
|
||||
$expectedTelemetryPredecessor = $ExpectedPipelineTelemetryPredecessorSha256.ToLowerInvariant()
|
||||
$predecessor = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $target
|
||||
).Hash.ToLowerInvariant()
|
||||
@@ -26,12 +34,34 @@ if (
|
||||
) {
|
||||
throw "Runner candidate digest changed"
|
||||
}
|
||||
$telemetryCandidateDigest = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $PipelineTelemetryCandidate
|
||||
).Hash.ToLowerInvariant()
|
||||
if ($telemetryCandidateDigest -ne $expectedTelemetry) {
|
||||
throw "Pipeline telemetry candidate digest changed"
|
||||
}
|
||||
$hadTelemetryModule = Test-Path -LiteralPath $telemetryTarget -PathType Leaf
|
||||
$telemetryPredecessor = if ($hadTelemetryModule) {
|
||||
(Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget).Hash.ToLowerInvariant()
|
||||
}
|
||||
else {
|
||||
"absent"
|
||||
}
|
||||
if ($telemetryPredecessor -ne $expectedTelemetryPredecessor) {
|
||||
throw "Pipeline telemetry predecessor digest changed"
|
||||
}
|
||||
$backup = Join-Path $RunnerRoot (
|
||||
"run_e15_shadow_inference.py.rollback-" +
|
||||
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
|
||||
"-" +
|
||||
$predecessor.Substring(0, 12)
|
||||
)
|
||||
$telemetryBackup = Join-Path $RunnerRoot (
|
||||
"pipeline_telemetry.py.rollback-" +
|
||||
[DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
|
||||
"-" +
|
||||
$telemetryPredecessor.Substring(0, [Math]::Min(12, $telemetryPredecessor.Length))
|
||||
)
|
||||
$healthCode = "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:18020/health',timeout=2).read().decode())"
|
||||
|
||||
function Wait-PerceptionHealth {
|
||||
@@ -52,7 +82,11 @@ function Wait-PerceptionHealth {
|
||||
if (
|
||||
$health.ok -and
|
||||
$health.state -eq "ready" -and
|
||||
(-not $RequireStageMetrics -or $health.stage_metrics)
|
||||
(-not $RequireStageMetrics -or (
|
||||
$health.stage_metrics -and
|
||||
$health.native_pipeline_telemetry -and
|
||||
$health.native_pipeline_telemetry.ready
|
||||
))
|
||||
) {
|
||||
return $health
|
||||
}
|
||||
@@ -63,7 +97,12 @@ function Wait-PerceptionHealth {
|
||||
}
|
||||
|
||||
Copy-Item -LiteralPath $target -Destination $backup
|
||||
if ($hadTelemetryModule) {
|
||||
Copy-Item -LiteralPath $telemetryTarget -Destination $telemetryBackup
|
||||
}
|
||||
try {
|
||||
Copy-Item -LiteralPath $PipelineTelemetryCandidate `
|
||||
-Destination $telemetryTarget -Force
|
||||
Copy-Item -LiteralPath $Candidate -Destination $target -Force
|
||||
if (
|
||||
(Get-FileHash -Algorithm SHA256 -LiteralPath $target).Hash.ToLowerInvariant() `
|
||||
@@ -71,6 +110,12 @@ try {
|
||||
) {
|
||||
throw "Runner replacement digest changed"
|
||||
}
|
||||
if (
|
||||
(Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget).Hash.ToLowerInvariant() `
|
||||
-ne $expectedTelemetry
|
||||
) {
|
||||
throw "Pipeline telemetry replacement digest changed"
|
||||
}
|
||||
docker restart --time 20 $ContainerName | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Perception container restart failed"
|
||||
@@ -82,6 +127,12 @@ try {
|
||||
}
|
||||
catch {
|
||||
Copy-Item -LiteralPath $backup -Destination $target -Force
|
||||
if ($hadTelemetryModule) {
|
||||
Copy-Item -LiteralPath $telemetryBackup -Destination $telemetryTarget -Force
|
||||
}
|
||||
else {
|
||||
Remove-Item -LiteralPath $telemetryTarget -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
docker restart --time 20 $ContainerName | Out-Null
|
||||
$rollbackHealth = Wait-PerceptionHealth -RequireStageMetrics $false
|
||||
if (-not $rollbackHealth) {
|
||||
@@ -98,6 +149,11 @@ catch {
|
||||
CandidateSha256 = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $target
|
||||
).Hash.ToLowerInvariant()
|
||||
PipelineTelemetryPredecessorSha256 = $telemetryPredecessor
|
||||
PipelineTelemetryCandidateSha256 = (
|
||||
Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget
|
||||
).Hash.ToLowerInvariant()
|
||||
PipelineTelemetryBackup = if ($hadTelemetryModule) { $telemetryBackup } else { $null }
|
||||
Backup = $backup
|
||||
Health = $health
|
||||
} | ConvertTo-Json -Depth 12 -Compress
|
||||
|
||||
Reference in New Issue
Block a user