620 lines
28 KiB
PowerShell
620 lines
28 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ReleaseRoot,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$CandidateRoot,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[a-f0-9]{64}$")]
|
|
[string]$ExpectedArtifactSha256,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[A-Za-z0-9._-]{1,96}$")]
|
|
[string]$RunId,
|
|
[ValidateRange(1.0, 120.0)]
|
|
[double]$SourceRateHz = 12.0,
|
|
[switch]$VegetationLoadGate,
|
|
[string]$VegetationAssetRoot = (
|
|
"D:\NDC_MISSIONCORE\datasets\vegetation-v1\observed-2026-08-27"
|
|
),
|
|
[string]$OutputRoot = (
|
|
"D:\NDC_MISSIONCORE\runtime\results\m49-tgs-integrated-graph-shadow"
|
|
)
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$ProgressPreference = "SilentlyContinue"
|
|
$TravelImageTag = "ndc/mission-core-m49-t3-travel:20260826"
|
|
$TravelImageId = "sha256:7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f"
|
|
$ParityImageTag = "ndc-mission-core-m48t-upstream-parity:1.9.4-cu130"
|
|
$ParityImageId = "sha256:ceb13548617e4bd3f619766bfdff00af3fa5160946b367828da6d2233dcdcba0"
|
|
$VegetationImageTag = "ndc/mission-core-lab-v1-goose:sg3.2.0-cu117-v1"
|
|
$VegetationImageId = "sha256:591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd"
|
|
$RuntimeImage = (
|
|
"nvcr.io/nvidia/tritonserver:26.06-py3@" +
|
|
"sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
|
|
)
|
|
|
|
function Assert-LastExitCode([string]$Operation) {
|
|
if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" }
|
|
}
|
|
|
|
function Get-Sha256([string]$Path) {
|
|
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
}
|
|
|
|
function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) {
|
|
if ($Create -and -not (Test-Path -LiteralPath $Path)) {
|
|
$null = New-Item -ItemType Directory -Path $Path
|
|
}
|
|
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
|
if (
|
|
-not $item.PSIsContainer -or
|
|
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
|
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
|
|
) { throw "$Label must be a real D: directory" }
|
|
return $item.FullName
|
|
}
|
|
|
|
function Resolve-DFile([string]$Path, [string]$Label) {
|
|
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
|
if (
|
|
$item.PSIsContainer -or
|
|
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
|
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
|
|
) { throw "$Label must be a real D: file" }
|
|
return $item.FullName
|
|
}
|
|
|
|
function Convert-ToDockerPath([string]$Path) { return ($Path -replace "\\", "/") }
|
|
|
|
function Get-Container([string]$Name) {
|
|
$rows = @(((& docker inspect $Name) | ConvertFrom-Json))
|
|
Assert-LastExitCode "Docker inspection for $Name"
|
|
if ($rows.Count -ne 1) { throw "Container identity for $Name is not unique" }
|
|
return $rows[0]
|
|
}
|
|
|
|
function Assert-Image([string]$Tag, [string]$ExpectedId) {
|
|
$rows = @(((& docker image inspect $Tag) | ConvertFrom-Json))
|
|
Assert-LastExitCode "Docker image inspection for $Tag"
|
|
if ($rows.Count -ne 1 -or [string]$rows[0].Id -cne $ExpectedId) {
|
|
throw "Pinned image identity changed for $Tag"
|
|
}
|
|
}
|
|
|
|
function Remove-ExactContainer([string]$Name) {
|
|
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$Name$") {
|
|
& docker rm --force $Name *> $null
|
|
}
|
|
}
|
|
|
|
function Wait-Healthy([string]$Name) {
|
|
foreach ($attempt in 1..60) {
|
|
Start-Sleep -Seconds 2
|
|
$container = Get-Container $Name
|
|
if (-not $container.State.Running) {
|
|
& docker logs $Name
|
|
throw "$Name stopped during startup"
|
|
}
|
|
if ($container.State.Health.Status -ceq "healthy") { return }
|
|
}
|
|
throw "$Name did not become healthy"
|
|
}
|
|
|
|
function Wait-SharedReady(
|
|
[string]$GraphReady,
|
|
[string]$TgsReady,
|
|
[string]$VegetationReady,
|
|
[string]$GraphName,
|
|
[string]$TgsName,
|
|
[string]$VegetationName
|
|
) {
|
|
$deadline = [DateTimeOffset]::UtcNow.AddMinutes(10)
|
|
$requiredFiles = @($GraphReady, $TgsReady)
|
|
$requiredContainers = @($GraphName, $TgsName)
|
|
if (-not [string]::IsNullOrWhiteSpace($VegetationReady)) {
|
|
$requiredFiles += $VegetationReady
|
|
$requiredContainers += $VegetationName
|
|
}
|
|
while ($requiredFiles.Where({ -not (Test-Path -LiteralPath $_) }).Count -gt 0) {
|
|
foreach ($name in $requiredContainers) {
|
|
$container = Get-Container $name
|
|
if (-not $container.State.Running) {
|
|
& docker logs $name
|
|
throw "$name stopped before shared-start readiness"
|
|
}
|
|
}
|
|
if ([DateTimeOffset]::UtcNow -ge $deadline) {
|
|
throw "M49 integrated shared-start readiness timed out"
|
|
}
|
|
Start-Sleep -Milliseconds 250
|
|
}
|
|
}
|
|
|
|
if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
|
|
throw "M49 integrated TGS graph shadow is pinned to Worker 006"
|
|
}
|
|
$release = Resolve-DDirectory $ReleaseRoot "M49 integrated release root" $false
|
|
$payload = Resolve-DDirectory (Join-Path $release "payload") "M49 integrated payload" $false
|
|
$candidate = Resolve-DDirectory $CandidateRoot "M49 native candidate root" $false
|
|
$output = Resolve-DDirectory $OutputRoot "M49 integrated output root" $true
|
|
$runCandidate = Join-Path $output $RunId
|
|
if (Test-Path -LiteralPath $runCandidate) { throw "M49 integrated output already exists" }
|
|
$null = New-Item -ItemType Directory -Path $runCandidate
|
|
$runOutput = Resolve-DDirectory $runCandidate "M49 integrated run output" $false
|
|
foreach ($directory in @("bin", "control", "graph", "tgs", "vegetation")) {
|
|
$null = New-Item -ItemType Directory -Path (Join-Path $runOutput $directory)
|
|
}
|
|
|
|
$releaseDocument = Get-Content -LiteralPath (Join-Path $payload "release.json") -Raw | ConvertFrom-Json
|
|
$expectedReleaseSchema = if ($VegetationLoadGate) {
|
|
"missioncore.lab-v1-vegetation-integrated-worker-release/v3"
|
|
} else {
|
|
"missioncore.m49-tgs-integrated-graph-worker-release/v1"
|
|
}
|
|
$expectedTransition = if ($VegetationLoadGate) {
|
|
"lab-v1-vegetation-m49-integrated-multirate-phased-shadow/v3"
|
|
} else {
|
|
"m49-tgs-native-risk-integrated-shadow/v1"
|
|
}
|
|
if (
|
|
$releaseDocument.schema_version -cne $expectedReleaseSchema -or
|
|
$releaseDocument.worker_id -cne "worker-006" -or
|
|
$releaseDocument.transition -cne $expectedTransition
|
|
) { throw "M49 integrated release contract changed" }
|
|
foreach ($property in $releaseDocument.files.PSObject.Properties) {
|
|
$path = Join-Path $payload $property.Name
|
|
if ((Get-Sha256 $path) -cne [string]$property.Value.sha256) {
|
|
throw "M49 integrated payload digest changed: $($property.Name)"
|
|
}
|
|
}
|
|
$wheelSha256 = [string]$releaseDocument.files."nodedc_mission_core-0.1.0-py3-none-any.whl".sha256
|
|
$runnerSha256 = [string]$releaseDocument.files."run_m48s_reference_graph_shadow_worker.py".sha256
|
|
$vegetationRunnerSha256 = if ($VegetationLoadGate) {
|
|
[string]$releaseDocument.files."run_vegetation_integrated_load.py".sha256
|
|
} else { "" }
|
|
|
|
$source = [ordered]@{
|
|
CameraIndex = (
|
|
"D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d" +
|
|
"\input\camera\sensor.camera.right\epoch-1\index.jsonl"
|
|
)
|
|
SourcePack = (
|
|
"D:\NDC_MISSIONCORE\runtime\derived" +
|
|
"\e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b" +
|
|
"\lidar-pack.npz"
|
|
)
|
|
LocalSurface = (
|
|
"D:\NDC_MISSIONCORE\runtime\derived" +
|
|
"\k1-local-surface-23762244c8bdb97de26fb721ac957d7a00bc9a63571ac4cfa4be19c4effc7d55" +
|
|
"\local-surface.npz"
|
|
)
|
|
Video = (
|
|
"D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs" +
|
|
"\right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8.mp4"
|
|
)
|
|
Mask = (
|
|
"D:\NDC_MISSIONCORE\runtime\inputs\e2" +
|
|
"\valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2" +
|
|
"\mask.png"
|
|
)
|
|
}
|
|
foreach ($entry in $source.GetEnumerator()) {
|
|
$null = Resolve-DFile $entry.Value "M49 source $($entry.Key)"
|
|
}
|
|
if ((Get-Sha256 $source.SourcePack) -cne [string]$releaseDocument.source_pack_sha256) {
|
|
throw "RAVNOVES00 source pack digest changed"
|
|
}
|
|
$videoSha256 = [string]$releaseDocument.video_sha256
|
|
if ((Get-Sha256 $source.Video) -cne $videoSha256) {
|
|
throw "RAVNOVES00 video digest changed"
|
|
}
|
|
|
|
$vegetation = $null
|
|
if ($VegetationLoadGate) {
|
|
$vegetationRoot = Resolve-DDirectory $VegetationAssetRoot "vegetation asset root" $false
|
|
$vegetation = [ordered]@{
|
|
Dataset = Resolve-DDirectory (
|
|
(Join-Path $vegetationRoot "goose-2d\validation")
|
|
) "GOOSE validation root" $false
|
|
Checkpoint = Resolve-DFile (
|
|
(Join-Path $vegetationRoot "models\goose\ddrnet_class_512.pth")
|
|
) "DDRNet checkpoint"
|
|
}
|
|
if (
|
|
(Get-Sha256 $vegetation.Checkpoint) -cne
|
|
"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"
|
|
) { throw "DDRNet checkpoint SHA-256 changed" }
|
|
}
|
|
|
|
$nativeConfig = Resolve-DFile (
|
|
(Join-Path $payload "rf_detr_large_native_kb4_config.pbtxt")
|
|
) "native RF-DETR config"
|
|
$nativeEngine = Resolve-DFile (
|
|
(Join-Path $candidate "rf-detr-native-uint8.plan")
|
|
) "native RF-DETR engine"
|
|
if ((Get-Sha256 $nativeEngine) -cne "b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695") {
|
|
throw "native RF-DETR engine SHA-256 changed"
|
|
}
|
|
$modelRoot = Join-Path $runOutput "triton-models"
|
|
$modelDirectory = Join-Path $modelRoot "rf_detr_large_native_kb4"
|
|
$modelVersionDirectory = Join-Path $modelDirectory "1"
|
|
$null = New-Item -ItemType Directory -Path $modelVersionDirectory
|
|
Copy-Item -LiteralPath $nativeConfig -Destination (Join-Path $modelDirectory "config.pbtxt")
|
|
Copy-Item -LiteralPath $nativeEngine -Destination (Join-Path $modelVersionDirectory "model.plan")
|
|
|
|
$media = Resolve-DDirectory (
|
|
"D:\NDC_MISSIONCORE\runtime\derived\perception-e15-media-pyav180-lz445-v1"
|
|
) "PyAV dependency" $false
|
|
$opencv = Resolve-DDirectory (
|
|
"D:\NDC_MISSIONCORE\runtime\derived\perception-e3-opencv413092-v1\packages"
|
|
) "OpenCV dependency" $false
|
|
$pillow = Resolve-DDirectory (
|
|
"D:\NDC_MISSIONCORE\runtime\derived\perception-p0-env-v1"
|
|
) "Pillow dependency" $false
|
|
|
|
Assert-Image $TravelImageTag $TravelImageId
|
|
Assert-Image $ParityImageTag $ParityImageId
|
|
if ($VegetationLoadGate) { Assert-Image $VegetationImageTag $VegetationImageId }
|
|
& docker image inspect $RuntimeImage *> $null
|
|
Assert-LastExitCode "pinned runtime image inspection"
|
|
$os = Get-CimInstance Win32_OperatingSystem
|
|
$freeMemoryGiB = [double]$os.FreePhysicalMemory / 1MB
|
|
$requiredMemoryGiB = if ($VegetationLoadGate) { 32.0 } else { 24.0 }
|
|
if ($freeMemoryGiB -lt $requiredMemoryGiB) {
|
|
throw (
|
|
"M49 integrated shadow requires {0:N0} GiB free memory; observed {1:N2} GiB" -f
|
|
$requiredMemoryGiB, $freeMemoryGiB
|
|
)
|
|
}
|
|
$canonicalBefore = Get-Container "ndc-mission-core-triton"
|
|
if (-not $canonicalBefore.State.Running -or $canonicalBefore.State.Health.Status -cne "healthy") {
|
|
throw "Canonical Mission Core Triton must remain healthy"
|
|
}
|
|
$canonicalId = [string]$canonicalBefore.Id
|
|
|
|
$prepareName = "ndc-mission-core-m49-integrated-prepare-$RunId"
|
|
$compileName = "ndc-mission-core-m49-integrated-compile-$RunId"
|
|
$tritonName = "ndc-mission-core-m49-integrated-triton-$RunId"
|
|
$graphName = "ndc-mission-core-m49-integrated-graph-$RunId"
|
|
$tgsName = "ndc-mission-core-m49-integrated-tgs-$RunId"
|
|
$vegetationName = "ndc-mission-core-m49-integrated-vegetation-$RunId"
|
|
$analyzeName = "ndc-mission-core-m49-integrated-analyze-$RunId"
|
|
$evidenceName = "ndc-mission-core-m49-integrated-evidence-$RunId"
|
|
$vegetationEvidenceName = "ndc-mission-core-m49-integrated-vegetation-evidence-$RunId"
|
|
$containers = @($prepareName, $compileName, $tritonName, $graphName, $tgsName, $analyzeName, $evidenceName)
|
|
if ($VegetationLoadGate) { $containers += @($vegetationName, $vegetationEvidenceName) }
|
|
foreach ($name in $containers) {
|
|
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
|
|
throw "M49 integrated container name already exists: $name"
|
|
}
|
|
}
|
|
|
|
$started = [DateTimeOffset]::UtcNow
|
|
try {
|
|
& docker run --rm --name $prepareName --network none --cpus 8 --memory 16g `
|
|
--entrypoint python3 `
|
|
--volume ((Convert-ToDockerPath $source.SourcePack) + ":/source/lidar-pack.npz:ro") `
|
|
--volume ((Convert-ToDockerPath $payload) + ":/release:ro") `
|
|
--volume ((Convert-ToDockerPath (Join-Path $runOutput "tgs")) + ":/tgs") `
|
|
$ParityImageTag /release/prepare_tgs_full_shadow_inputs.py `
|
|
--source-pack /source/lidar-pack.npz `
|
|
--config /release/m49-tgs-full-shadow-v1.json `
|
|
--output-root /tgs/inputs
|
|
Assert-LastExitCode "M49 integrated TGS input preparation"
|
|
|
|
& docker run --rm --name $compileName --network none --cpus 8 --memory 8g `
|
|
--entrypoint /bin/bash `
|
|
--volume ((Convert-ToDockerPath $payload) + ":/release:ro") `
|
|
--volume ((Convert-ToDockerPath (Join-Path $runOutput "bin")) + ":/out") `
|
|
$TravelImageTag /release/build_tgs_full_shadow_binary.sh /out/run_tgs_full_shadow
|
|
Assert-LastExitCode "M49 integrated TGS binary build"
|
|
|
|
& docker create --name $tritonName `
|
|
--read-only --security-opt "no-new-privileges:true" --cap-drop ALL `
|
|
--pids-limit 512 --shm-size 1g --gpus all `
|
|
--tmpfs "/tmp:rw,noexec,nosuid,size=2g" `
|
|
--health-cmd "curl --fail --silent http://127.0.0.1:8000/v2/health/ready" `
|
|
--health-interval 5s --health-timeout 3s --health-start-period 20s --health-retries 24 `
|
|
-v ((Convert-ToDockerPath $modelRoot) + ":/models:ro") `
|
|
$RuntimeImage tritonserver --model-repository=/models `
|
|
--model-control-mode=explicit --load-model=rf_detr_large_native_kb4 `
|
|
--disable-auto-complete-config --strict-readiness=true --exit-on-error=true `
|
|
--allow-http=true --allow-grpc=false --allow-metrics=false *> $null
|
|
Assert-LastExitCode "M49 integrated Triton creation"
|
|
& docker start $tritonName *> $null
|
|
Assert-LastExitCode "M49 integrated Triton start"
|
|
Wait-Healthy $tritonName
|
|
|
|
$dockerRelease = Convert-ToDockerPath $payload
|
|
$dockerRun = Convert-ToDockerPath $runOutput
|
|
$rate = [string]::Format([Globalization.CultureInfo]::InvariantCulture, "{0:R}", $SourceRateHz)
|
|
$graphArguments = @(
|
|
"create", "--name", $graphName,
|
|
"--network", ("container:{0}" -f $tritonName),
|
|
"--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
|
"--pids-limit", "256", "--gpus", "all",
|
|
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
|
|
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
|
"-e", "PYTHONPATH=/release/nodedc_mission_core-0.1.0-py3-none-any.whl:/opt/media:/opt/opencv:/opt/pillow",
|
|
"-v", ("{0}:/release:ro" -f $dockerRelease),
|
|
"-v", ("{0}:/shared:rw" -f $dockerRun),
|
|
"-v", ((Convert-ToDockerPath $media) + ":/opt/media:ro"),
|
|
"-v", ((Convert-ToDockerPath $opencv) + ":/opt/opencv:ro"),
|
|
"-v", ((Convert-ToDockerPath $pillow) + ":/opt/pillow:ro"),
|
|
"-v", ((Convert-ToDockerPath $source.CameraIndex) + ":/source/camera-index.jsonl:ro"),
|
|
"-v", ((Convert-ToDockerPath $source.SourcePack) + ":/source/source-pack.npz:ro"),
|
|
"-v", ((Convert-ToDockerPath $source.LocalSurface) + ":/source/local-surface.npz:ro"),
|
|
"-v", ((Convert-ToDockerPath $source.Video) + ":/source/right.mp4:ro"),
|
|
"-v", ((Convert-ToDockerPath $source.Mask) + ":/source/mask.png:ro"),
|
|
"--entrypoint", "python3", $RuntimeImage,
|
|
"/release/run_m48s_reference_graph_shadow_worker.py",
|
|
"--graph-config", "/release/m48n-rf-detr-native-reference-graph-shadow-v0.json",
|
|
"--baseline-profile", "/release/m4-recorded-realtime-baseline-v1.json",
|
|
"--detector-profile", "/release/rf-detr-large-native-kb4-risk-shadow-v0.json",
|
|
"--geometry-profile", "/release/m4-geometry-association-v1.json",
|
|
"--temporal-motion-profile", "/release/m4-temporal-motion-v1.json",
|
|
"--rolling-map-profile", "/release/m4-rolling-local-map-v1.json",
|
|
"--threat-profile", "/release/m4-replay-threat-v3.json",
|
|
"--camera-index", "/source/camera-index.jsonl",
|
|
"--source-pack", "/source/source-pack.npz",
|
|
"--local-surface", "/source/local-surface.npz",
|
|
"--video", "/source/right.mp4",
|
|
"--valid-fov-mask", "/source/mask.png",
|
|
"--triton-origin", "http://127.0.0.1:8000",
|
|
"--loops", "1", "--maximum-frames", "4489", "--source-rate-hz", $rate,
|
|
"--minimum-delivery-ratio", "1.0",
|
|
"--minimum-effective-world-state-fps", "11.209069",
|
|
"--maximum-world-state-completion-p95-ms", "125.0",
|
|
"--load-purpose", "reserve-gate",
|
|
"--runtime-artifact-sha256", $wheelSha256,
|
|
"--runner-sha256", $runnerSha256,
|
|
"--shared-start-ready-file", "/shared/control/graph.ready",
|
|
"--shared-start-file", "/shared/control/start.signal",
|
|
"--output", "/shared/graph/result.json",
|
|
"--progress", "/shared/graph/progress.jsonl",
|
|
"--frame-ledger", "/shared/graph/frames.jsonl"
|
|
)
|
|
& docker @graphArguments *> $null
|
|
Assert-LastExitCode "M49 integrated graph creation"
|
|
|
|
& docker create --name $tgsName --network none --cpus 16 --memory 24g `
|
|
--read-only --security-opt "no-new-privileges:true" --cap-drop ALL `
|
|
--pids-limit 256 --tmpfs "/tmp:rw,noexec,nosuid,size=2g" `
|
|
-e ("M49_SOURCE_RATE_HZ={0}" -f $rate) `
|
|
--entrypoint /bin/bash `
|
|
--volume ($dockerRelease + ":/release:ro") `
|
|
--volume ($dockerRun + ":/shared:rw") `
|
|
$TravelImageTag /release/run_tgs_integrated_shadow.sh *> $null
|
|
Assert-LastExitCode "M49 integrated TGS creation"
|
|
|
|
if ($VegetationLoadGate) {
|
|
$dockerVegetationDataset = Convert-ToDockerPath $vegetation.Dataset
|
|
$dockerVegetationCheckpoint = Convert-ToDockerPath $vegetation.Checkpoint
|
|
& docker create --name $vegetationName --network none --cpus 8 --memory 10g `
|
|
--gpus all --read-only --security-opt "no-new-privileges:true" --cap-drop ALL `
|
|
--pids-limit 512 --tmpfs "/tmp:rw,noexec,nosuid,size=2g" `
|
|
-e "HOME=/tmp" `
|
|
--entrypoint conda `
|
|
--volume ($dockerRelease + ":/release:ro") `
|
|
--volume ($dockerRun + ":/shared:rw") `
|
|
--volume ($dockerVegetationDataset + ":/data/goose:ro") `
|
|
--volume ($dockerVegetationCheckpoint + ":/models/candidate.pth:ro") `
|
|
--volume ((Convert-ToDockerPath $source.Video) + ":/source/right.mp4:ro") `
|
|
$VegetationImageTag run --no-capture-output --name goose python `
|
|
/release/run_vegetation_integrated_load.py `
|
|
--config /release/lab-v1-goose-vegetation-benchmark-v1.json `
|
|
--policy /release/lab-v1-vegetation-mission-policy-v1.json `
|
|
--provider-map /release/lab-v1-vegetation-provider-label-map-v1.json `
|
|
--checkpoint /models/candidate.pth `
|
|
--dataset-root /data/goose `
|
|
--video /source/right.mp4 `
|
|
--video-sha256 $videoSha256 `
|
|
--runtime-video-cache /tmp/vegetation-right.mp4 `
|
|
--source-rate-hz $rate `
|
|
--inference-stride 2 `
|
|
--inference-phase-offset-ms 40.0 `
|
|
--minimum-effective-timeline-fps 11.209069 `
|
|
--minimum-effective-inference-fps 5.604534 `
|
|
--maximum-inference-completion-p95-ms 125.0 `
|
|
--maximum-evidence-source-age-ms 125.0 `
|
|
--shared-start-ready-file /shared/control/vegetation.ready `
|
|
--shared-start-file /shared/control/start.signal `
|
|
--frame-ledger /shared/vegetation/frames.jsonl `
|
|
--output /shared/vegetation/result.json `
|
|
--release-sha256 $ExpectedArtifactSha256 *> $null
|
|
Assert-LastExitCode "M49 integrated vegetation creation"
|
|
}
|
|
|
|
& docker start $graphName *> $null
|
|
Assert-LastExitCode "M49 integrated graph start"
|
|
& docker start $tgsName *> $null
|
|
Assert-LastExitCode "M49 integrated TGS start"
|
|
if ($VegetationLoadGate) {
|
|
& docker start $vegetationName *> $null
|
|
Assert-LastExitCode "M49 integrated vegetation start"
|
|
}
|
|
$graphReady = Join-Path $runOutput "control\graph.ready"
|
|
$tgsReady = Join-Path $runOutput "control\tgs.ready"
|
|
$vegetationReady = if ($VegetationLoadGate) {
|
|
Join-Path $runOutput "control\vegetation.ready"
|
|
} else { "" }
|
|
Wait-SharedReady $graphReady $tgsReady $vegetationReady $graphName $tgsName $vegetationName
|
|
[DateTimeOffset]::UtcNow.ToString("o") | Set-Content -LiteralPath (
|
|
Join-Path $runOutput "control\start.signal"
|
|
) -Encoding utf8
|
|
|
|
$telemetryPath = Join-Path $runOutput "container-telemetry.jsonl"
|
|
$m49TelemetryPath = if ($VegetationLoadGate) {
|
|
Join-Path $runOutput "m49-container-telemetry.jsonl"
|
|
} else { $telemetryPath }
|
|
while ($true) {
|
|
$graphState = Get-Container $graphName
|
|
$tgsState = Get-Container $tgsName
|
|
$vegetationState = if ($VegetationLoadGate) {
|
|
Get-Container $vegetationName
|
|
} else { $null }
|
|
$running = @()
|
|
if ($graphState.State.Running) { $running += $graphName }
|
|
if ($tgsState.State.Running) { $running += $tgsName }
|
|
if ($VegetationLoadGate -and $vegetationState.State.Running) {
|
|
$running += $vegetationName
|
|
}
|
|
if ((Get-Container $tritonName).State.Running) { $running += $tritonName }
|
|
if ($running.Count -gt 0) {
|
|
$stats = @((& docker stats --no-stream --format "{{json .}}" @running))
|
|
Assert-LastExitCode "M49 integrated container telemetry"
|
|
foreach ($line in $stats) {
|
|
$value = $line | ConvertFrom-Json
|
|
$role = if ($value.Name -ceq $graphName) {
|
|
"graph"
|
|
} elseif ($value.Name -ceq $tgsName) {
|
|
"tgs"
|
|
} elseif ($value.Name -ceq $tritonName) {
|
|
"triton"
|
|
} elseif ($VegetationLoadGate -and $value.Name -ceq $vegetationName) {
|
|
"vegetation"
|
|
} else {
|
|
throw "Unknown M49 telemetry container"
|
|
}
|
|
$telemetryRow = [ordered]@{
|
|
observed_utc = [DateTimeOffset]::UtcNow.ToString("o")
|
|
role = $role
|
|
name = [string]$value.Name
|
|
cpu_percent = [string]$value.CPUPerc
|
|
memory_usage = [string]$value.MemUsage
|
|
memory_percent = [string]$value.MemPerc
|
|
pids = [string]$value.PIDs
|
|
} | ConvertTo-Json -Compress
|
|
$telemetryRow | Out-File -LiteralPath $telemetryPath -Encoding utf8 -Append
|
|
if ($VegetationLoadGate -and $role -cne "vegetation") {
|
|
$telemetryRow | Out-File -LiteralPath $m49TelemetryPath -Encoding utf8 -Append
|
|
}
|
|
}
|
|
}
|
|
$vegetationStopped = -not $VegetationLoadGate -or -not $vegetationState.State.Running
|
|
if (
|
|
-not $graphState.State.Running -and
|
|
-not $tgsState.State.Running -and
|
|
$vegetationStopped
|
|
) { break }
|
|
Start-Sleep -Seconds 1
|
|
}
|
|
$graphExit = [int](Get-Container $graphName).State.ExitCode
|
|
$tgsExit = [int](Get-Container $tgsName).State.ExitCode
|
|
$vegetationExit = if ($VegetationLoadGate) {
|
|
[int](Get-Container $vegetationName).State.ExitCode
|
|
} else { 0 }
|
|
$previousErrorAction = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
$graphLogs = & docker logs $graphName 2>&1
|
|
$tgsLogs = & docker logs $tgsName 2>&1
|
|
$vegetationLogs = if ($VegetationLoadGate) {
|
|
& docker logs $vegetationName 2>&1
|
|
} else { @() }
|
|
$ErrorActionPreference = $previousErrorAction
|
|
$graphLogs | Set-Content -LiteralPath (Join-Path $runOutput "graph.log") -Encoding utf8
|
|
$tgsLogs | Set-Content -LiteralPath (Join-Path $runOutput "tgs.log") -Encoding utf8
|
|
if ($VegetationLoadGate) {
|
|
$vegetationLogs | Set-Content -LiteralPath (
|
|
Join-Path $runOutput "vegetation.log"
|
|
) -Encoding utf8
|
|
}
|
|
if ($graphExit -ne 0) { throw "M49 integrated graph failed with exit code $graphExit" }
|
|
if ($tgsExit -ne 0) { throw "M49 integrated TGS failed with exit code $tgsExit" }
|
|
if ($vegetationExit -ne 0) {
|
|
throw "M49 integrated vegetation failed with exit code $vegetationExit"
|
|
}
|
|
|
|
& docker run --rm --name $analyzeName --network none --cpus 8 --memory 16g `
|
|
--entrypoint python3 `
|
|
--volume ($dockerRelease + ":/release:ro") `
|
|
--volume ($dockerRun + ":/shared:rw") `
|
|
$ParityImageTag /release/build_tgs_full_shadow_evidence.py `
|
|
--run-root /shared/tgs `
|
|
--config /release/m49-tgs-full-shadow-v1.json `
|
|
--output-root /shared/tgs/evidence
|
|
Assert-LastExitCode "M49 integrated TGS evidence analysis"
|
|
|
|
$m49ResultPath = if ($VegetationLoadGate) {
|
|
"/shared/m49-result.json"
|
|
} else { "/shared/result.json" }
|
|
$dockerM49TelemetryPath = if ($VegetationLoadGate) {
|
|
"/shared/m49-container-telemetry.jsonl"
|
|
} else { "/shared/container-telemetry.jsonl" }
|
|
& docker run --rm --name $evidenceName --network none --cpus 4 --memory 8g `
|
|
--entrypoint python3 `
|
|
--volume ($dockerRelease + ":/release:ro") `
|
|
--volume ($dockerRun + ":/shared:rw") `
|
|
$ParityImageTag /release/build_tgs_integrated_graph_evidence.py `
|
|
--profile /release/m49-tgs-integrated-graph-shadow-v1.json `
|
|
--graph-result /shared/graph/result.json `
|
|
--graph-frames /shared/graph/frames.jsonl `
|
|
--tgs-result /shared/tgs/evidence/result.json `
|
|
--tgs-timing /shared/tgs/tgs-full-timing.tsv `
|
|
--telemetry $dockerM49TelemetryPath `
|
|
--output $m49ResultPath `
|
|
--release-sha256 $ExpectedArtifactSha256
|
|
Assert-LastExitCode "M49 integrated evidence gate"
|
|
|
|
if ($VegetationLoadGate) {
|
|
& docker run --rm --name $vegetationEvidenceName --network none --cpus 4 --memory 8g `
|
|
--entrypoint python3 `
|
|
--volume ($dockerRelease + ":/release:ro") `
|
|
--volume ($dockerRun + ":/shared:rw") `
|
|
$ParityImageTag /release/build_vegetation_integrated_graph_evidence.py `
|
|
--profile /release/lab-v1-vegetation-integrated-multirate-phased-shadow-v3.json `
|
|
--m49-result /shared/m49-result.json `
|
|
--graph-frames /shared/graph/frames.jsonl `
|
|
--tgs-timing /shared/tgs/tgs-full-timing.tsv `
|
|
--vegetation-result /shared/vegetation/result.json `
|
|
--vegetation-frames /shared/vegetation/frames.jsonl `
|
|
--telemetry /shared/container-telemetry.jsonl `
|
|
--output /shared/result.json `
|
|
--release-sha256 $ExpectedArtifactSha256
|
|
Assert-LastExitCode "M49 integrated vegetation evidence gate"
|
|
}
|
|
} finally {
|
|
foreach ($name in $containers) { Remove-ExactContainer $name }
|
|
$canonicalAfter = Get-Container "ndc-mission-core-triton"
|
|
if (
|
|
[string]$canonicalAfter.Id -cne $canonicalId -or
|
|
-not $canonicalAfter.State.Running -or
|
|
$canonicalAfter.State.Health.Status -cne "healthy"
|
|
) { throw "Canonical Mission Core Triton changed during M49 integrated shadow" }
|
|
}
|
|
|
|
$completed = [DateTimeOffset]::UtcNow
|
|
$resultPath = Join-Path $runOutput "result.json"
|
|
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
|
|
throw "M49 integrated result is missing"
|
|
}
|
|
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
|
|
$summary = [ordered]@{
|
|
schema_version = if ($VegetationLoadGate) {
|
|
"missioncore.lab-v1-vegetation-integrated-worker-summary/v3"
|
|
} else {
|
|
"missioncore.m49-tgs-integrated-graph-worker-summary/v1"
|
|
}
|
|
worker_id = "worker-006"
|
|
run_id = $RunId
|
|
code_revision = [string]$releaseDocument.code_revision
|
|
source_rate_hz = $SourceRateHz
|
|
started_utc = $started.ToString("o")
|
|
completed_utc = $completed.ToString("o")
|
|
wall_seconds = [math]::Round(($completed - $started).TotalSeconds, 6)
|
|
free_memory_gib_before = [math]::Round($freeMemoryGiB, 6)
|
|
result_id = [string]$result.result_id
|
|
result_status = [string]$result.status
|
|
vegetation_load_gate = [bool]$VegetationLoadGate
|
|
canonical_triton_id = $canonicalId
|
|
canonical_triton_health = "healthy"
|
|
gauss_or_playcanvas_action = "none"
|
|
durable_worker_action = "none"
|
|
navigation_or_actuation_allowed = $false
|
|
}
|
|
$summary | ConvertTo-Json -Depth 3 | Set-Content -LiteralPath (
|
|
Join-Path $runOutput "worker-summary.json"
|
|
) -Encoding utf8
|
|
$summary | ConvertTo-Json -Depth 3
|