Files
NODEDC_MISSION_CORE/experiments/perception/worker/Invoke-M49TgsIntegratedGraphShadow.ps1
T

456 lines
21 KiB
PowerShell

[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[string]$ReleaseRoot,
[Parameter(Mandatory = $true)]
[string]$CandidateRoot,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[a-f0-9]{64}$")]
[string]$ExpectedArtifactSha256,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[A-Za-z0-9._-]{1,96}$")]
[string]$RunId,
[ValidateRange(1.0, 120.0)]
[double]$SourceRateHz = 12.0,
[string]$OutputRoot = (
"D:\NDC_MISSIONCORE\runtime\results\m49-tgs-integrated-graph-shadow"
)
)
$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue"
$TravelImageTag = "ndc/mission-core-m49-t3-travel:20260826"
$TravelImageId = "sha256:7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f"
$ParityImageTag = "ndc-mission-core-m48t-upstream-parity:1.9.4-cu130"
$ParityImageId = "sha256:ceb13548617e4bd3f619766bfdff00af3fa5160946b367828da6d2233dcdcba0"
$RuntimeImage = (
"nvcr.io/nvidia/tritonserver:26.06-py3@" +
"sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
)
function Assert-LastExitCode([string]$Operation) {
if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" }
}
function Get-Sha256([string]$Path) {
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
}
function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) {
if ($Create -and -not (Test-Path -LiteralPath $Path)) {
$null = New-Item -ItemType Directory -Path $Path
}
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
if (
-not $item.PSIsContainer -or
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
) { throw "$Label must be a real D: directory" }
return $item.FullName
}
function Resolve-DFile([string]$Path, [string]$Label) {
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
if (
$item.PSIsContainer -or
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
) { throw "$Label must be a real D: file" }
return $item.FullName
}
function Convert-ToDockerPath([string]$Path) { return ($Path -replace "\\", "/") }
function Get-Container([string]$Name) {
$rows = @(((& docker inspect $Name) | ConvertFrom-Json))
Assert-LastExitCode "Docker inspection for $Name"
if ($rows.Count -ne 1) { throw "Container identity for $Name is not unique" }
return $rows[0]
}
function Assert-Image([string]$Tag, [string]$ExpectedId) {
$rows = @(((& docker image inspect $Tag) | ConvertFrom-Json))
Assert-LastExitCode "Docker image inspection for $Tag"
if ($rows.Count -ne 1 -or [string]$rows[0].Id -cne $ExpectedId) {
throw "Pinned image identity changed for $Tag"
}
}
function Remove-ExactContainer([string]$Name) {
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$Name$") {
& docker rm --force $Name *> $null
}
}
function Wait-Healthy([string]$Name) {
foreach ($attempt in 1..60) {
Start-Sleep -Seconds 2
$container = Get-Container $Name
if (-not $container.State.Running) {
& docker logs $Name
throw "$Name stopped during startup"
}
if ($container.State.Health.Status -ceq "healthy") { return }
}
throw "$Name did not become healthy"
}
function Wait-SharedReady(
[string]$GraphReady,
[string]$TgsReady,
[string]$GraphName,
[string]$TgsName
) {
$deadline = [DateTimeOffset]::UtcNow.AddMinutes(10)
while (-not ((Test-Path -LiteralPath $GraphReady) -and (Test-Path -LiteralPath $TgsReady))) {
foreach ($name in @($GraphName, $TgsName)) {
$container = Get-Container $name
if (-not $container.State.Running) {
& docker logs $name
throw "$name stopped before shared-start readiness"
}
}
if ([DateTimeOffset]::UtcNow -ge $deadline) {
throw "M49 integrated shared-start readiness timed out"
}
Start-Sleep -Milliseconds 250
}
}
if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
throw "M49 integrated TGS graph shadow is pinned to Worker 006"
}
$release = Resolve-DDirectory $ReleaseRoot "M49 integrated release root" $false
$payload = Resolve-DDirectory (Join-Path $release "payload") "M49 integrated payload" $false
$candidate = Resolve-DDirectory $CandidateRoot "M49 native candidate root" $false
$output = Resolve-DDirectory $OutputRoot "M49 integrated output root" $true
$runCandidate = Join-Path $output $RunId
if (Test-Path -LiteralPath $runCandidate) { throw "M49 integrated output already exists" }
$null = New-Item -ItemType Directory -Path $runCandidate
$runOutput = Resolve-DDirectory $runCandidate "M49 integrated run output" $false
foreach ($directory in @("bin", "control", "graph", "tgs")) {
$null = New-Item -ItemType Directory -Path (Join-Path $runOutput $directory)
}
$releaseDocument = Get-Content -LiteralPath (Join-Path $payload "release.json") -Raw | ConvertFrom-Json
if (
$releaseDocument.schema_version -cne "missioncore.m49-tgs-integrated-graph-worker-release/v1" -or
$releaseDocument.worker_id -cne "worker-006" -or
$releaseDocument.transition -cne "m49-tgs-native-risk-integrated-shadow/v1"
) { throw "M49 integrated release contract changed" }
foreach ($property in $releaseDocument.files.PSObject.Properties) {
$path = Join-Path $payload $property.Name
if ((Get-Sha256 $path) -cne [string]$property.Value.sha256) {
throw "M49 integrated payload digest changed: $($property.Name)"
}
}
$wheelSha256 = [string]$releaseDocument.files."nodedc_mission_core-0.1.0-py3-none-any.whl".sha256
$runnerSha256 = [string]$releaseDocument.files."run_m48s_reference_graph_shadow_worker.py".sha256
$source = [ordered]@{
CameraIndex = (
"D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d" +
"\input\camera\sensor.camera.right\epoch-1\index.jsonl"
)
SourcePack = (
"D:\NDC_MISSIONCORE\runtime\derived" +
"\e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b" +
"\lidar-pack.npz"
)
LocalSurface = (
"D:\NDC_MISSIONCORE\runtime\derived" +
"\k1-local-surface-23762244c8bdb97de26fb721ac957d7a00bc9a63571ac4cfa4be19c4effc7d55" +
"\local-surface.npz"
)
Video = (
"D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs" +
"\right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8.mp4"
)
Mask = (
"D:\NDC_MISSIONCORE\runtime\inputs\e2" +
"\valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2" +
"\mask.png"
)
}
foreach ($entry in $source.GetEnumerator()) {
$null = Resolve-DFile $entry.Value "M49 source $($entry.Key)"
}
if ((Get-Sha256 $source.SourcePack) -cne [string]$releaseDocument.source_pack_sha256) {
throw "RAVNOVES00 source pack digest changed"
}
$nativeConfig = Resolve-DFile (
(Join-Path $payload "rf_detr_large_native_kb4_config.pbtxt")
) "native RF-DETR config"
$nativeEngine = Resolve-DFile (
(Join-Path $candidate "rf-detr-native-uint8.plan")
) "native RF-DETR engine"
if ((Get-Sha256 $nativeEngine) -cne "b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695") {
throw "native RF-DETR engine SHA-256 changed"
}
$modelRoot = Join-Path $runOutput "triton-models"
$modelDirectory = Join-Path $modelRoot "rf_detr_large_native_kb4"
$modelVersionDirectory = Join-Path $modelDirectory "1"
$null = New-Item -ItemType Directory -Path $modelVersionDirectory
Copy-Item -LiteralPath $nativeConfig -Destination (Join-Path $modelDirectory "config.pbtxt")
Copy-Item -LiteralPath $nativeEngine -Destination (Join-Path $modelVersionDirectory "model.plan")
$media = Resolve-DDirectory (
"D:\NDC_MISSIONCORE\runtime\derived\perception-e15-media-pyav180-lz445-v1"
) "PyAV dependency" $false
$opencv = Resolve-DDirectory (
"D:\NDC_MISSIONCORE\runtime\derived\perception-e3-opencv413092-v1\packages"
) "OpenCV dependency" $false
$pillow = Resolve-DDirectory (
"D:\NDC_MISSIONCORE\runtime\derived\perception-p0-env-v1"
) "Pillow dependency" $false
Assert-Image $TravelImageTag $TravelImageId
Assert-Image $ParityImageTag $ParityImageId
& docker image inspect $RuntimeImage *> $null
Assert-LastExitCode "pinned runtime image inspection"
$os = Get-CimInstance Win32_OperatingSystem
$freeMemoryGiB = [double]$os.FreePhysicalMemory / 1MB
if ($freeMemoryGiB -lt 24.0) {
throw ("M49 integrated shadow requires 24 GiB free memory; observed {0:N2} GiB" -f $freeMemoryGiB)
}
$canonicalBefore = Get-Container "ndc-mission-core-triton"
if (-not $canonicalBefore.State.Running -or $canonicalBefore.State.Health.Status -cne "healthy") {
throw "Canonical Mission Core Triton must remain healthy"
}
$canonicalId = [string]$canonicalBefore.Id
$prepareName = "ndc-mission-core-m49-integrated-prepare-$RunId"
$compileName = "ndc-mission-core-m49-integrated-compile-$RunId"
$tritonName = "ndc-mission-core-m49-integrated-triton-$RunId"
$graphName = "ndc-mission-core-m49-integrated-graph-$RunId"
$tgsName = "ndc-mission-core-m49-integrated-tgs-$RunId"
$analyzeName = "ndc-mission-core-m49-integrated-analyze-$RunId"
$evidenceName = "ndc-mission-core-m49-integrated-evidence-$RunId"
$containers = @($prepareName, $compileName, $tritonName, $graphName, $tgsName, $analyzeName, $evidenceName)
foreach ($name in $containers) {
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
throw "M49 integrated container name already exists: $name"
}
}
$started = [DateTimeOffset]::UtcNow
try {
& docker run --rm --name $prepareName --network none --cpus 8 --memory 16g `
--entrypoint python3 `
--volume ((Convert-ToDockerPath $source.SourcePack) + ":/source/lidar-pack.npz:ro") `
--volume ((Convert-ToDockerPath $payload) + ":/release:ro") `
--volume ((Convert-ToDockerPath (Join-Path $runOutput "tgs")) + ":/tgs") `
$ParityImageTag /release/prepare_tgs_full_shadow_inputs.py `
--source-pack /source/lidar-pack.npz `
--config /release/m49-tgs-full-shadow-v1.json `
--output-root /tgs/inputs
Assert-LastExitCode "M49 integrated TGS input preparation"
& docker run --rm --name $compileName --network none --cpus 8 --memory 8g `
--entrypoint /bin/bash `
--volume ((Convert-ToDockerPath $payload) + ":/release:ro") `
--volume ((Convert-ToDockerPath (Join-Path $runOutput "bin")) + ":/out") `
$TravelImageTag /release/build_tgs_full_shadow_binary.sh /out/run_tgs_full_shadow
Assert-LastExitCode "M49 integrated TGS binary build"
& docker create --name $tritonName `
--read-only --security-opt "no-new-privileges:true" --cap-drop ALL `
--pids-limit 512 --shm-size 1g --gpus all `
--tmpfs "/tmp:rw,noexec,nosuid,size=2g" `
--health-cmd "curl --fail --silent http://127.0.0.1:8000/v2/health/ready" `
--health-interval 5s --health-timeout 3s --health-start-period 20s --health-retries 24 `
-v ((Convert-ToDockerPath $modelRoot) + ":/models:ro") `
$RuntimeImage tritonserver --model-repository=/models `
--model-control-mode=explicit --load-model=rf_detr_large_native_kb4 `
--disable-auto-complete-config --strict-readiness=true --exit-on-error=true `
--allow-http=true --allow-grpc=false --allow-metrics=false *> $null
Assert-LastExitCode "M49 integrated Triton creation"
& docker start $tritonName *> $null
Assert-LastExitCode "M49 integrated Triton start"
Wait-Healthy $tritonName
$dockerRelease = Convert-ToDockerPath $payload
$dockerRun = Convert-ToDockerPath $runOutput
$rate = [string]::Format([Globalization.CultureInfo]::InvariantCulture, "{0:R}", $SourceRateHz)
$graphArguments = @(
"create", "--name", $graphName,
"--network", ("container:{0}" -f $tritonName),
"--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
"--pids-limit", "256", "--gpus", "all",
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
"-e", "PYTHONDONTWRITEBYTECODE=1",
"-e", "PYTHONPATH=/release/nodedc_mission_core-0.1.0-py3-none-any.whl:/opt/media:/opt/opencv:/opt/pillow",
"-v", ("{0}:/release:ro" -f $dockerRelease),
"-v", ("{0}:/shared:rw" -f $dockerRun),
"-v", ((Convert-ToDockerPath $media) + ":/opt/media:ro"),
"-v", ((Convert-ToDockerPath $opencv) + ":/opt/opencv:ro"),
"-v", ((Convert-ToDockerPath $pillow) + ":/opt/pillow:ro"),
"-v", ((Convert-ToDockerPath $source.CameraIndex) + ":/source/camera-index.jsonl:ro"),
"-v", ((Convert-ToDockerPath $source.SourcePack) + ":/source/source-pack.npz:ro"),
"-v", ((Convert-ToDockerPath $source.LocalSurface) + ":/source/local-surface.npz:ro"),
"-v", ((Convert-ToDockerPath $source.Video) + ":/source/right.mp4:ro"),
"-v", ((Convert-ToDockerPath $source.Mask) + ":/source/mask.png:ro"),
"--entrypoint", "python3", $RuntimeImage,
"/release/run_m48s_reference_graph_shadow_worker.py",
"--graph-config", "/release/m48n-rf-detr-native-reference-graph-shadow-v0.json",
"--baseline-profile", "/release/m4-recorded-realtime-baseline-v1.json",
"--detector-profile", "/release/rf-detr-large-native-kb4-risk-shadow-v0.json",
"--geometry-profile", "/release/m4-geometry-association-v1.json",
"--temporal-motion-profile", "/release/m4-temporal-motion-v1.json",
"--rolling-map-profile", "/release/m4-rolling-local-map-v1.json",
"--threat-profile", "/release/m4-replay-threat-v3.json",
"--camera-index", "/source/camera-index.jsonl",
"--source-pack", "/source/source-pack.npz",
"--local-surface", "/source/local-surface.npz",
"--video", "/source/right.mp4",
"--valid-fov-mask", "/source/mask.png",
"--triton-origin", "http://127.0.0.1:8000",
"--loops", "1", "--maximum-frames", "4489", "--source-rate-hz", $rate,
"--minimum-delivery-ratio", "1.0",
"--minimum-effective-world-state-fps", "11.209069",
"--maximum-world-state-completion-p95-ms", "125.0",
"--load-purpose", "reserve-gate",
"--runtime-artifact-sha256", $wheelSha256,
"--runner-sha256", $runnerSha256,
"--shared-start-ready-file", "/shared/control/graph.ready",
"--shared-start-file", "/shared/control/start.signal",
"--output", "/shared/graph/result.json",
"--progress", "/shared/graph/progress.jsonl",
"--frame-ledger", "/shared/graph/frames.jsonl"
)
& docker @graphArguments *> $null
Assert-LastExitCode "M49 integrated graph creation"
& docker create --name $tgsName --network none --cpus 16 --memory 24g `
--read-only --security-opt "no-new-privileges:true" --cap-drop ALL `
--pids-limit 256 --tmpfs "/tmp:rw,noexec,nosuid,size=1g" `
-e ("M49_SOURCE_RATE_HZ={0}" -f $rate) `
--entrypoint /bin/bash `
--volume ($dockerRelease + ":/release:ro") `
--volume ($dockerRun + ":/shared:rw") `
$TravelImageTag /release/run_tgs_integrated_shadow.sh *> $null
Assert-LastExitCode "M49 integrated TGS creation"
& docker start $graphName *> $null
Assert-LastExitCode "M49 integrated graph start"
& docker start $tgsName *> $null
Assert-LastExitCode "M49 integrated TGS start"
$graphReady = Join-Path $runOutput "control\graph.ready"
$tgsReady = Join-Path $runOutput "control\tgs.ready"
Wait-SharedReady $graphReady $tgsReady $graphName $tgsName
[DateTimeOffset]::UtcNow.ToString("o") | Set-Content -LiteralPath (
Join-Path $runOutput "control\start.signal"
) -Encoding utf8
$telemetryPath = Join-Path $runOutput "container-telemetry.jsonl"
while ($true) {
$graphState = Get-Container $graphName
$tgsState = Get-Container $tgsName
$running = @()
if ($graphState.State.Running) { $running += $graphName }
if ($tgsState.State.Running) { $running += $tgsName }
if ((Get-Container $tritonName).State.Running) { $running += $tritonName }
if ($running.Count -gt 0) {
$stats = @((& docker stats --no-stream --format "{{json .}}" @running))
Assert-LastExitCode "M49 integrated container telemetry"
foreach ($line in $stats) {
$value = $line | ConvertFrom-Json
$role = if ($value.Name -ceq $graphName) {
"graph"
} elseif ($value.Name -ceq $tgsName) {
"tgs"
} elseif ($value.Name -ceq $tritonName) {
"triton"
} else {
throw "Unknown M49 telemetry container"
}
[ordered]@{
observed_utc = [DateTimeOffset]::UtcNow.ToString("o")
role = $role
name = [string]$value.Name
cpu_percent = [string]$value.CPUPerc
memory_usage = [string]$value.MemUsage
memory_percent = [string]$value.MemPerc
pids = [string]$value.PIDs
} | ConvertTo-Json -Compress | Out-File -LiteralPath $telemetryPath -Encoding utf8 -Append
}
}
if (-not $graphState.State.Running -and -not $tgsState.State.Running) { break }
Start-Sleep -Seconds 1
}
$graphExit = [int](Get-Container $graphName).State.ExitCode
$tgsExit = [int](Get-Container $tgsName).State.ExitCode
$previousErrorAction = $ErrorActionPreference
$ErrorActionPreference = "Continue"
$graphLogs = & docker logs $graphName 2>&1
$tgsLogs = & docker logs $tgsName 2>&1
$ErrorActionPreference = $previousErrorAction
$graphLogs | Set-Content -LiteralPath (Join-Path $runOutput "graph.log") -Encoding utf8
$tgsLogs | Set-Content -LiteralPath (Join-Path $runOutput "tgs.log") -Encoding utf8
if ($graphExit -ne 0) { throw "M49 integrated graph failed with exit code $graphExit" }
if ($tgsExit -ne 0) { throw "M49 integrated TGS failed with exit code $tgsExit" }
& docker run --rm --name $analyzeName --network none --cpus 8 --memory 16g `
--entrypoint python3 `
--volume ($dockerRelease + ":/release:ro") `
--volume ($dockerRun + ":/shared:rw") `
$ParityImageTag /release/build_tgs_full_shadow_evidence.py `
--run-root /shared/tgs `
--config /release/m49-tgs-full-shadow-v1.json `
--output-root /shared/tgs/evidence
Assert-LastExitCode "M49 integrated TGS evidence analysis"
& docker run --rm --name $evidenceName --network none --cpus 4 --memory 8g `
--entrypoint python3 `
--volume ($dockerRelease + ":/release:ro") `
--volume ($dockerRun + ":/shared:rw") `
$ParityImageTag /release/build_tgs_integrated_graph_evidence.py `
--profile /release/m49-tgs-integrated-graph-shadow-v1.json `
--graph-result /shared/graph/result.json `
--graph-frames /shared/graph/frames.jsonl `
--tgs-result /shared/tgs/evidence/result.json `
--tgs-timing /shared/tgs/tgs-full-timing.tsv `
--telemetry /shared/container-telemetry.jsonl `
--output /shared/result.json `
--release-sha256 $ExpectedArtifactSha256
Assert-LastExitCode "M49 integrated evidence gate"
} finally {
foreach ($name in $containers) { Remove-ExactContainer $name }
$canonicalAfter = Get-Container "ndc-mission-core-triton"
if (
[string]$canonicalAfter.Id -cne $canonicalId -or
-not $canonicalAfter.State.Running -or
$canonicalAfter.State.Health.Status -cne "healthy"
) { throw "Canonical Mission Core Triton changed during M49 integrated shadow" }
}
$completed = [DateTimeOffset]::UtcNow
$resultPath = Join-Path $runOutput "result.json"
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
throw "M49 integrated result is missing"
}
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
$summary = [ordered]@{
schema_version = "missioncore.m49-tgs-integrated-graph-worker-summary/v1"
worker_id = "worker-006"
run_id = $RunId
code_revision = [string]$releaseDocument.code_revision
source_rate_hz = $SourceRateHz
started_utc = $started.ToString("o")
completed_utc = $completed.ToString("o")
wall_seconds = [math]::Round(($completed - $started).TotalSeconds, 6)
free_memory_gib_before = [math]::Round($freeMemoryGiB, 6)
result_id = [string]$result.result_id
result_status = [string]$result.status
canonical_triton_id = $canonicalId
canonical_triton_health = "healthy"
gauss_or_playcanvas_action = "none"
durable_worker_action = "none"
navigation_or_actuation_allowed = $false
}
$summary | ConvertTo-Json -Depth 3 | Set-Content -LiteralPath (
Join-Path $runOutput "worker-summary.json"
) -Encoding utf8
$summary | ConvertTo-Json -Depth 3