feat(perception): seal M4.8S reference graph replay
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,287 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ReleaseRoot,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[a-f0-9]{64}$")]
|
||||
[string]$ExpectedWheelSha256,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[A-Za-z0-9._-]{1,96}$")]
|
||||
[string]$RunId,
|
||||
[ValidateRange(1, 10)]
|
||||
[int]$Loops = 1,
|
||||
[ValidateRange(1, 4489)]
|
||||
[int]$MaximumFrames = 120,
|
||||
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m48s-reference-graph-shadow"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode([string]$Operation) {
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-Sha256([string]$Path) {
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Assert-File([string]$Path, [string]$ExpectedSha256, [string]$Label) {
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
throw "$Label must be a regular file"
|
||||
}
|
||||
if ((Get-Sha256 $item.FullName) -cne $ExpectedSha256) {
|
||||
throw "$Label SHA-256 changed"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) {
|
||||
if ($Create -and -not (Test-Path -LiteralPath $Path)) {
|
||||
$null = New-Item -ItemType Directory -Path $Path
|
||||
}
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
if (
|
||||
-not $item.PSIsContainer -or
|
||||
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
||||
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
|
||||
) {
|
||||
throw "$Label must be a real D: directory"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath([string]$Path) {
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
function Get-Container([string]$Name) {
|
||||
$rows = @((& docker inspect $Name) | ConvertFrom-Json)
|
||||
Assert-LastExitCode "Docker inspection for $Name"
|
||||
if ($rows.Count -ne 1) {
|
||||
throw "Container identity for $Name is not unique"
|
||||
}
|
||||
return $rows[0]
|
||||
}
|
||||
|
||||
if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
|
||||
throw "M48S graph shadow is pinned to DESKTOP-OPJ8J04"
|
||||
}
|
||||
|
||||
$release = Resolve-DDirectory $ReleaseRoot "M48S release root" $false
|
||||
$output = Resolve-DDirectory $OutputRoot "M48S output root" $true
|
||||
$runOutput = Join-Path $output $RunId
|
||||
if (Test-Path -LiteralPath $runOutput) {
|
||||
throw "M48S run output already exists"
|
||||
}
|
||||
$null = New-Item -ItemType Directory -Path $runOutput
|
||||
$runOutput = Resolve-DDirectory $runOutput "M48S run output" $false
|
||||
$wheel = Assert-File (
|
||||
Join-Path $release "nodedc_mission_core-0.1.0-py3-none-any.whl"
|
||||
) $ExpectedWheelSha256 "M48S wheel"
|
||||
|
||||
$expectedConfigs = [ordered]@{
|
||||
"m48s-rf-detr-reference-graph-shadow-v0.json" = "e607916c0d2db5a1078bc194fa0e5e1bac1ca336de8daad29359ed0d2791b6cd"
|
||||
"m4-recorded-realtime-baseline-v1.json" = "ea10359339e6cce31b5780a2710299771cab7cc0c1c2a2b56a1621f786b31fa8"
|
||||
"rf-detr-large-risk-shadow-v0.json" = "0c307fd2d19cedd2c9267b6be2effdce82161a719a742f315fcd7a76f7061b08"
|
||||
"m4-geometry-association-v1.json" = "cc666c9389a5e221957faddec89584709b66918d14abaf646f1832e001421999"
|
||||
"m4-temporal-motion-v1.json" = "7130eaee24a95c7d888bf7598010e03e129e1c3ac5b34bcd8401015ff4244b39"
|
||||
"m4-rolling-local-map-v1.json" = "f7e3315eaf6ffaf3aee1e04913933812092cf82bbcc9984c1a6fa2d9250e6784"
|
||||
"m4-replay-threat-v3.json" = "8c3a5aa837da1f028f5998fb504a1381f9b2b68de6420a32160410b6dc0887c7"
|
||||
}
|
||||
foreach ($entry in $expectedConfigs.GetEnumerator()) {
|
||||
$null = Assert-File (Join-Path $release $entry.Key) $entry.Value (
|
||||
"M48S config {0}" -f $entry.Key
|
||||
)
|
||||
}
|
||||
$runner = Get-Item -LiteralPath (
|
||||
Join-Path $release "run_m48s_reference_graph_shadow_worker.py"
|
||||
)
|
||||
if ($runner.PSIsContainer -or ($runner.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
throw "M48S graph runner must be a regular file"
|
||||
}
|
||||
|
||||
$experimentRoot = Resolve-DDirectory (
|
||||
"D:\NDC_MISSIONCORE\runtime\experiments\m48t-fixed-detector-20260825T095425Z"
|
||||
) "M48S RF-DETR experiment root" $false
|
||||
$modelRoot = Resolve-DDirectory (
|
||||
(Join-Path $experimentRoot "triton-models")
|
||||
) "M48S RF-DETR Triton model root" $false
|
||||
$null = Assert-File (
|
||||
(Join-Path $modelRoot "rf_detr_large\1\model.plan")
|
||||
) "986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8" (
|
||||
"RF-DETR TensorRT engine"
|
||||
)
|
||||
$null = Assert-File (
|
||||
(Join-Path $modelRoot "rf_detr_large\config.pbtxt")
|
||||
) "80947cad235e5b000f11aa869a33af0e8c727f07e04046691468df1e171479b6" (
|
||||
"RF-DETR Triton config"
|
||||
)
|
||||
|
||||
$image = "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
|
||||
& docker image inspect $image *> $null
|
||||
Assert-LastExitCode "Pinned M48S image inspection"
|
||||
$historicalTriton = Get-Container "ndc-mission-core-triton"
|
||||
if (-not $historicalTriton.State.Running -or $historicalTriton.State.Health.Status -cne "healthy") {
|
||||
throw "Historical Triton must remain healthy during M48S shadow"
|
||||
}
|
||||
$historicalTritonId = [string]$historicalTriton.Id
|
||||
$tritonName = "ndc-mission-core-m48s-rf-detr-triton-shadow"
|
||||
$graphName = "ndc-mission-core-m48s-reference-graph-shadow"
|
||||
foreach ($name in @($tritonName, $graphName)) {
|
||||
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
|
||||
throw "M48S candidate container $name already exists"
|
||||
}
|
||||
}
|
||||
|
||||
$source = [ordered]@{
|
||||
CameraIndex = "D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d\input\camera\sensor.camera.right\epoch-1\index.jsonl"
|
||||
SourcePack = "D:\NDC_MISSIONCORE\runtime\derived\e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b\lidar-pack.npz"
|
||||
LocalSurface = "D:\NDC_MISSIONCORE\runtime\derived\k1-local-surface-23762244c8bdb97de26fb721ac957d7a00bc9a63571ac4cfa4be19c4effc7d55\local-surface.npz"
|
||||
Video = "D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs\right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8.mp4"
|
||||
Mask = "D:\NDC_MISSIONCORE\runtime\inputs\e2\valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2\mask.png"
|
||||
}
|
||||
foreach ($entry in $source.GetEnumerator()) {
|
||||
if (-not (Test-Path -LiteralPath $entry.Value -PathType Leaf)) {
|
||||
throw "M48S source $($entry.Key) is missing"
|
||||
}
|
||||
}
|
||||
|
||||
$media = Resolve-DDirectory (
|
||||
"D:\NDC_MISSIONCORE\runtime\derived\perception-e15-media-pyav180-lz445-v1"
|
||||
) "PyAV dependency" $false
|
||||
$opencv = Resolve-DDirectory (
|
||||
"D:\NDC_MISSIONCORE\runtime\derived\perception-e3-opencv413092-v1\packages"
|
||||
) "OpenCV dependency" $false
|
||||
$pillow = Resolve-DDirectory (
|
||||
"D:\NDC_MISSIONCORE\runtime\derived\perception-p0-env-v1"
|
||||
) "Pillow dependency" $false
|
||||
|
||||
try {
|
||||
& docker create `
|
||||
--name $tritonName `
|
||||
--read-only `
|
||||
--security-opt "no-new-privileges:true" `
|
||||
--cap-drop ALL `
|
||||
--pids-limit 512 `
|
||||
--shm-size 1g `
|
||||
--gpus all `
|
||||
--tmpfs "/tmp:rw,noexec,nosuid,size=2g" `
|
||||
--health-cmd "curl --fail --silent http://127.0.0.1:8000/v2/health/ready" `
|
||||
--health-interval 5s `
|
||||
--health-timeout 3s `
|
||||
--health-start-period 20s `
|
||||
--health-retries 24 `
|
||||
-v ("{0}:/models:ro" -f (Convert-ToDockerPath $modelRoot)) `
|
||||
$image `
|
||||
tritonserver `
|
||||
--model-repository=/models `
|
||||
--model-control-mode=explicit `
|
||||
--load-model=rf_detr_large `
|
||||
--disable-auto-complete-config `
|
||||
--strict-readiness=true `
|
||||
--exit-on-error=true `
|
||||
--allow-http=true `
|
||||
--allow-grpc=false `
|
||||
--allow-metrics=false *> $null
|
||||
Assert-LastExitCode "M48S Triton creation"
|
||||
& docker start $tritonName *> $null
|
||||
Assert-LastExitCode "M48S Triton start"
|
||||
$ready = $false
|
||||
foreach ($attempt in 1..60) {
|
||||
Start-Sleep -Seconds 2
|
||||
$candidate = Get-Container $tritonName
|
||||
if (-not $candidate.State.Running) {
|
||||
throw "M48S Triton stopped during startup"
|
||||
}
|
||||
if ($candidate.State.Health.Status -ceq "healthy") {
|
||||
$ready = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if (-not $ready) {
|
||||
throw "M48S Triton did not become healthy"
|
||||
}
|
||||
if (@((Get-Container $tritonName).HostConfig.PortBindings.PSObject.Properties).Count -ne 0) {
|
||||
throw "M48S Triton published a host port"
|
||||
}
|
||||
|
||||
$dockerRelease = Convert-ToDockerPath $release
|
||||
$dockerOutput = Convert-ToDockerPath $runOutput
|
||||
$maximumArguments = @()
|
||||
if ($MaximumFrames -gt 0) {
|
||||
$maximumArguments = @("--maximum-frames", ([string]$MaximumFrames))
|
||||
}
|
||||
$arguments = @(
|
||||
"run", "--name", $graphName,
|
||||
"--network", ("container:{0}" -f $tritonName),
|
||||
"--read-only",
|
||||
"--security-opt", "no-new-privileges:true",
|
||||
"--cap-drop", "ALL",
|
||||
"--pids-limit", "256",
|
||||
"--gpus", "all",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
|
||||
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
||||
"-e", "PYTHONPATH=/release/nodedc_mission_core-0.1.0-py3-none-any.whl:/opt/media:/opt/opencv:/opt/pillow",
|
||||
"-v", ("{0}:/release:ro" -f $dockerRelease),
|
||||
"-v", ("{0}:/output:rw" -f $dockerOutput),
|
||||
"-v", ("{0}:/opt/media:ro" -f (Convert-ToDockerPath $media)),
|
||||
"-v", ("{0}:/opt/opencv:ro" -f (Convert-ToDockerPath $opencv)),
|
||||
"-v", ("{0}:/opt/pillow:ro" -f (Convert-ToDockerPath $pillow)),
|
||||
"-v", ("{0}:/source/camera-index.jsonl:ro" -f (Convert-ToDockerPath $source.CameraIndex)),
|
||||
"-v", ("{0}:/source/source-pack.npz:ro" -f (Convert-ToDockerPath $source.SourcePack)),
|
||||
"-v", ("{0}:/source/local-surface.npz:ro" -f (Convert-ToDockerPath $source.LocalSurface)),
|
||||
"-v", ("{0}:/source/right.mp4:ro" -f (Convert-ToDockerPath $source.Video)),
|
||||
"-v", ("{0}:/source/mask.png:ro" -f (Convert-ToDockerPath $source.Mask)),
|
||||
"--entrypoint", "python3",
|
||||
$image,
|
||||
"/release/run_m48s_reference_graph_shadow_worker.py",
|
||||
"--graph-config", "/release/m48s-rf-detr-reference-graph-shadow-v0.json",
|
||||
"--baseline-profile", "/release/m4-recorded-realtime-baseline-v1.json",
|
||||
"--detector-profile", "/release/rf-detr-large-risk-shadow-v0.json",
|
||||
"--geometry-profile", "/release/m4-geometry-association-v1.json",
|
||||
"--temporal-motion-profile", "/release/m4-temporal-motion-v1.json",
|
||||
"--rolling-map-profile", "/release/m4-rolling-local-map-v1.json",
|
||||
"--threat-profile", "/release/m4-replay-threat-v3.json",
|
||||
"--camera-index", "/source/camera-index.jsonl",
|
||||
"--source-pack", "/source/source-pack.npz",
|
||||
"--local-surface", "/source/local-surface.npz",
|
||||
"--video", "/source/right.mp4",
|
||||
"--valid-fov-mask", "/source/mask.png",
|
||||
"--triton-origin", "http://127.0.0.1:8000",
|
||||
"--loops", ([string]$Loops),
|
||||
"--output", "/output/result.json",
|
||||
"--progress", "/output/progress.jsonl",
|
||||
"--frame-ledger", "/output/frames.jsonl"
|
||||
) + $maximumArguments
|
||||
& docker @arguments
|
||||
Assert-LastExitCode "M48S complete reference graph shadow"
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $runOutput "result.json") -PathType Leaf)) {
|
||||
throw "M48S graph result was not written"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $runOutput "frames.jsonl") -PathType Leaf)) {
|
||||
throw "M48S frame evidence ledger was not written"
|
||||
}
|
||||
} finally {
|
||||
foreach ($name in @($graphName, $tritonName)) {
|
||||
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
|
||||
& docker rm -f $name *> $null
|
||||
}
|
||||
}
|
||||
$historicalAfter = Get-Container "ndc-mission-core-triton"
|
||||
if (
|
||||
$historicalAfter.Id -cne $historicalTritonId -or
|
||||
-not $historicalAfter.State.Running -or
|
||||
$historicalAfter.State.Health.Status -cne "healthy"
|
||||
) {
|
||||
throw "Historical Triton changed during M48S shadow"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Output ("M48S_RESULT={0}" -f (Join-Path $runOutput "result.json"))
|
||||
Write-Output "HISTORICAL_TRITON_ACTION=none"
|
||||
Write-Output "DURABLE_WORKER_ACTION=none"
|
||||
Write-Output "PRODUCTION_ACCEPTED=false"
|
||||
Reference in New Issue
Block a user