feat(telemetry): expose compute pipeline stage metrics

2026-07-29 01:53:31 +03:00
parent 783aa444ac
commit 79eb4b46f7
19 changed files with 1000 additions and 24 deletions
@@ -0,0 +1,103 @@
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$Candidate,
+    [Parameter(Mandatory = $true)]
+    [string]$ExpectedPredecessorSha256,
+    [Parameter(Mandatory = $true)]
+    [string]$ExpectedCandidateSha256,
+    [string]$ContainerName = "ndc-mission-core-perception-worker",
+    [string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002"
+)
+
+$ErrorActionPreference = "Stop"
+$target = Join-Path $RunnerRoot "run_e15_shadow_inference.py"
+$expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant()
+$expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant()
+$predecessor = (
+    Get-FileHash -Algorithm SHA256 -LiteralPath $target
+).Hash.ToLowerInvariant()
+if ($predecessor -ne $expectedPredecessor) {
+    throw "Runner predecessor digest changed"
+}
+if (
+    (Get-FileHash -Algorithm SHA256 -LiteralPath $Candidate).Hash.ToLowerInvariant() `
+        -ne $expectedCandidate
+) {
+    throw "Runner candidate digest changed"
+}
+$backup = Join-Path $RunnerRoot (
+    "run_e15_shadow_inference.py.rollback-" +
+    [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
+    "-" +
+    $predecessor.Substring(0, 12)
+)
+$healthCode = "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:18020/health',timeout=2).read().decode())"
+
+function Wait-PerceptionHealth {
+    param(
+        [bool]$RequireStageMetrics,
+        [int]$TimeoutSeconds = 150
+    )
+
+    $deadline = [DateTime]::UtcNow.AddSeconds($TimeoutSeconds)
+    while ([DateTime]::UtcNow -lt $deadline) {
+        Start-Sleep -Seconds 3
+        try {
+            $healthJson = docker exec $ContainerName python3 -c $healthCode 2>$null
+            if ($LASTEXITCODE -ne 0 -or -not $healthJson) {
+                continue
+            }
+            $health = $healthJson | ConvertFrom-Json
+            if (
+                $health.ok -and
+                $health.state -eq "ready" -and
+                (-not $RequireStageMetrics -or $health.stage_metrics)
+            ) {
+                return $health
+            }
+        }
+        catch {}
+    }
+    return $null
+}
+
+Copy-Item -LiteralPath $target -Destination $backup
+try {
+    Copy-Item -LiteralPath $Candidate -Destination $target -Force
+    if (
+        (Get-FileHash -Algorithm SHA256 -LiteralPath $target).Hash.ToLowerInvariant() `
+            -ne $expectedCandidate
+    ) {
+        throw "Runner replacement digest changed"
+    }
+    docker restart --time 20 $ContainerName | Out-Null
+    if ($LASTEXITCODE -ne 0) {
+        throw "Perception container restart failed"
+    }
+    $health = Wait-PerceptionHealth -RequireStageMetrics $true
+    if (-not $health) {
+        throw "Stage-instrumented runner did not become ready"
+    }
+}
+catch {
+    Copy-Item -LiteralPath $backup -Destination $target -Force
+    docker restart --time 20 $ContainerName | Out-Null
+    $rollbackHealth = Wait-PerceptionHealth -RequireStageMetrics $false
+    if (-not $rollbackHealth) {
+        throw "Candidate failed and predecessor rollback did not become ready"
+    }
+    throw
+}
+
+[ordered]@{
+    SchemaVersion = "missioncore.worker-runner-deploy-result/v1"
+    Container = $ContainerName
+    State = (docker inspect --format "{{.State.Status}}" $ContainerName)
+    PredecessorSha256 = $predecessor
+    CandidateSha256 = (
+        Get-FileHash -Algorithm SHA256 -LiteralPath $target
+    ).Hash.ToLowerInvariant()
+    Backup = $backup
+    Health = $health
+} | ConvertTo-Json -Depth 12 -Compress