372 lines
17 KiB
PowerShell
372 lines
17 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[ValidateSet("Build", "Probe", "Validate", "Ravnoves", "RavnovesVideo", "Status")]
|
|
[string]$Mode = "Status",
|
|
|
|
[ValidateSet("Ddrnet", "Ppliteseg")]
|
|
[string]$Candidate = "Ddrnet",
|
|
|
|
[string]$AssetRoot = "D:\NDC_MISSIONCORE\datasets\vegetation-v1\observed-2026-08-27",
|
|
|
|
[string]$ToolRoot = "D:\NDC_MISSIONCORE\datasets\tooling\lab-v1-vegetation-goose",
|
|
|
|
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\experiments\lab-v1-vegetation",
|
|
|
|
[string]$RavnovesVideo = "D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs\right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8.mp4",
|
|
|
|
[string]$RavnovesSourceId = "RAVNOVES00/right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8",
|
|
|
|
[string]$RavnovesSha256 = "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8",
|
|
|
|
[ValidateRange(1, 1000000)]
|
|
[int]$RavnovesExpectedFrameCount = 4489,
|
|
|
|
[string]$RavnovesBaseM4ResultId = "m4-threat-replay-2a953c5f27f2a5b1dddc5c658c1de2c323d7796084a099c024987a1da03aa324",
|
|
|
|
[string]$RavnovesSourceProfile = ""
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$datasetPrefix = "D:\NDC_MISSIONCORE\datasets\"
|
|
$runtimePrefix = "D:\NDC_MISSIONCORE\runtime\experiments\"
|
|
if (-not $AssetRoot.StartsWith($datasetPrefix, [StringComparison]::OrdinalIgnoreCase)) {
|
|
throw "AssetRoot must stay under $datasetPrefix"
|
|
}
|
|
if (-not $ToolRoot.StartsWith($datasetPrefix, [StringComparison]::OrdinalIgnoreCase)) {
|
|
throw "ToolRoot must stay under $datasetPrefix"
|
|
}
|
|
if (-not $OutputRoot.StartsWith($runtimePrefix, [StringComparison]::OrdinalIgnoreCase)) {
|
|
throw "OutputRoot must stay under $runtimePrefix"
|
|
}
|
|
|
|
$image = "ndc/mission-core-lab-v1-goose:sg3.2.0-cu117-v1"
|
|
$canonicalContainer = "ndc-mission-core-triton"
|
|
$candidateKey = $Candidate.ToLowerInvariant()
|
|
$contextRoot = Join-Path $ToolRoot "context"
|
|
$configRoot = Join-Path $ToolRoot "config"
|
|
$benchmarkConfig = Join-Path $configRoot "lab-v1-goose-vegetation-benchmark-v1.json"
|
|
$policyConfig = Join-Path $configRoot "lab-v1-vegetation-mission-policy-v1.json"
|
|
$providerMapConfig = Join-Path $configRoot "lab-v1-vegetation-provider-label-map-v1.json"
|
|
$ravnovesProfileDocument = $null
|
|
if (-not [string]::IsNullOrWhiteSpace($RavnovesSourceProfile)) {
|
|
$resolvedProfile = (Resolve-Path -LiteralPath $RavnovesSourceProfile).Path
|
|
if (-not $resolvedProfile.StartsWith($ToolRoot, [StringComparison]::OrdinalIgnoreCase)) {
|
|
throw "RAVNOVES source profile must stay under ToolRoot"
|
|
}
|
|
$ravnovesProfileDocument = Get-Content -LiteralPath $resolvedProfile -Raw | ConvertFrom-Json
|
|
$source = $ravnovesProfileDocument.source
|
|
if (
|
|
$ravnovesProfileDocument.schema_version -ne "missioncore.lab-v1-ravnoves-source/v1" -or
|
|
$null -eq $source -or
|
|
[string]::IsNullOrWhiteSpace([string]$source.source_id) -or
|
|
[string]$source.source_sha256 -notmatch "^[a-f0-9]{64}$" -or
|
|
[int]$source.expected_width -ne 800 -or
|
|
[int]$source.expected_height -ne 600 -or
|
|
[int]$source.expected_frame_count -lt 1 -or
|
|
[string]$source.crop_contract -ne "center-600-square-to-512; outside-crop-is-undefined"
|
|
) {
|
|
throw "RAVNOVES source profile is incompatible"
|
|
}
|
|
$RavnovesSourceId = [string]$source.source_id
|
|
$RavnovesSha256 = [string]$source.source_sha256
|
|
$RavnovesExpectedFrameCount = [int]$source.expected_frame_count
|
|
$RavnovesBaseM4ResultId = [string]$source.base_m4_result_id
|
|
}
|
|
$datasetRoot = Join-Path $AssetRoot "goose-2d\validation"
|
|
$checkpointRelative = if ($candidateKey -eq "ddrnet") {
|
|
"models\goose\ddrnet_class_512.pth"
|
|
} else {
|
|
"models\goose\ppliteseg_class_512.pth"
|
|
}
|
|
$checkpoint = Join-Path $AssetRoot $checkpointRelative
|
|
$expectedCheckpointSha256 = if ($candidateKey -eq "ddrnet") {
|
|
"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"
|
|
} else {
|
|
"6dd412c0c99115e359896c4cab43a8e6bce9e09b843e7fa885fe597b0a6121cd"
|
|
}
|
|
$expectedCheckpointBytes = if ($candidateKey -eq "ddrnet") { 259419077 } else { 98208249 }
|
|
$frameIndices = @(0, 253, 512, 768, 1024, 1536, 2048, 2560, 3072, 3584, 4096, 4488)
|
|
$dockerConfig = "D:\NDC_MISSIONCORE\datasets\state\lab-v1-vegetation\docker-config"
|
|
|
|
function Get-CanonicalTritonIdentity {
|
|
$identity = & docker inspect $canonicalContainer --format "{{.Id}}|{{.Config.Image}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{end}}"
|
|
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($identity)) {
|
|
throw "Canonical Triton is unavailable"
|
|
}
|
|
$parts = $identity.Split("|")
|
|
if ($parts.Count -ne 4 -or $parts[2] -ne "running" -or $parts[3] -ne "healthy") {
|
|
throw "Canonical Triton is not running and healthy: $identity"
|
|
}
|
|
return $identity
|
|
}
|
|
|
|
function Assert-FileIdentity {
|
|
param([string]$Path, [long]$ExpectedBytes, [string]$ExpectedSha256)
|
|
$file = Get-Item -LiteralPath $Path -ErrorAction SilentlyContinue
|
|
if ($null -eq $file -or $file.Length -ne $ExpectedBytes) {
|
|
throw "File identity changed: $Path"
|
|
}
|
|
$actualSha256 = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($actualSha256 -ne $ExpectedSha256) {
|
|
throw "File digest changed: $Path"
|
|
}
|
|
}
|
|
|
|
function Assert-RunnerInputs {
|
|
foreach ($path in @($benchmarkConfig, $policyConfig, $providerMapConfig)) {
|
|
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
|
|
throw "Runner config is unavailable: $path"
|
|
}
|
|
}
|
|
if (-not (Test-Path -LiteralPath (Join-Path $contextRoot "Dockerfile") -PathType Leaf)) {
|
|
throw "Runner Dockerfile is unavailable"
|
|
}
|
|
if (-not (Test-Path -LiteralPath (Join-Path $contextRoot "run_goose_vegetation_benchmark.py") -PathType Leaf)) {
|
|
throw "Runner source is unavailable"
|
|
}
|
|
Assert-FileIdentity -Path $checkpoint -ExpectedBytes $expectedCheckpointBytes -ExpectedSha256 $expectedCheckpointSha256
|
|
}
|
|
|
|
function New-RunRoot {
|
|
param([string]$Kind)
|
|
$stamp = [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssfffZ")
|
|
$path = Join-Path $OutputRoot ("{0}-{1}-{2}" -f $Kind, $candidateKey, $stamp)
|
|
New-Item -ItemType Directory -Path $path | Out-Null
|
|
return $path
|
|
}
|
|
|
|
function Invoke-IsolatedRun {
|
|
param(
|
|
[ValidateSet("goose", "ravnoves", "ravnoves-video")][string]$RunMode,
|
|
[string]$RunRoot,
|
|
[int]$Limit,
|
|
[string]$FramesRoot = ""
|
|
)
|
|
$containerName = "ndc-lab-v1-goose-$candidateKey-$([Guid]::NewGuid().ToString('N').Substring(0, 10))"
|
|
$activeConfigRoot = $configRoot
|
|
if ($RunMode -eq "ravnoves-video" -and $null -ne $ravnovesProfileDocument) {
|
|
$activeConfigRoot = Join-Path $RunRoot "effective-config"
|
|
New-Item -ItemType Directory -Path $activeConfigRoot | Out-Null
|
|
Copy-Item -LiteralPath $policyConfig -Destination $activeConfigRoot
|
|
Copy-Item -LiteralPath $providerMapConfig -Destination $activeConfigRoot
|
|
$benchmark = Get-Content -LiteralPath $benchmarkConfig -Raw | ConvertFrom-Json
|
|
$benchmark.ravnoves = $ravnovesProfileDocument.source
|
|
$benchmark | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath (
|
|
Join-Path $activeConfigRoot "lab-v1-goose-vegetation-benchmark-v1.json"
|
|
) -Encoding utf8
|
|
}
|
|
$visualCount = if ($RunMode -eq "ravnoves-video") { 0 } else { 12 }
|
|
$arguments = @(
|
|
"run", "--rm", "--name", $containerName,
|
|
"--gpus", "all",
|
|
"--network", "none",
|
|
"--read-only",
|
|
"--cap-drop", "ALL",
|
|
"--security-opt", "no-new-privileges",
|
|
"--memory", "10g",
|
|
"--cpus", "8",
|
|
"--pids-limit", "512",
|
|
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
|
|
"--env", "HOME=/tmp",
|
|
"--mount", "type=bind,src=$datasetRoot,dst=/data/goose,readonly",
|
|
"--mount", "type=bind,src=$checkpoint,dst=/models/candidate.pth,readonly",
|
|
"--mount", "type=bind,src=$activeConfigRoot,dst=/config,readonly",
|
|
"--mount", "type=bind,src=$RunRoot,dst=/output",
|
|
$image,
|
|
"--mode", $RunMode,
|
|
"--candidate", $candidateKey,
|
|
"--config", "/config/lab-v1-goose-vegetation-benchmark-v1.json",
|
|
"--policy", "/config/lab-v1-vegetation-mission-policy-v1.json",
|
|
"--provider-map", "/config/lab-v1-vegetation-provider-label-map-v1.json",
|
|
"--checkpoint", "/models/candidate.pth",
|
|
"--dataset-root", "/data/goose",
|
|
"--output", "/output/result",
|
|
"--limit", $Limit.ToString(),
|
|
"--visual-count", $visualCount.ToString()
|
|
)
|
|
if ($RunMode -in @("ravnoves", "ravnoves-video")) {
|
|
$arguments = @($arguments[0..($arguments.Count - 1)])
|
|
$arguments += @("--frames-root", "/input")
|
|
$mountIndex = [Array]::IndexOf($arguments, $image)
|
|
$head = @($arguments[0..($mountIndex - 1)])
|
|
$tail = @($arguments[$mountIndex..($arguments.Count - 1)])
|
|
$arguments = $head + @("--mount", "type=bind,src=$FramesRoot,dst=/input,readonly") + $tail
|
|
}
|
|
$dockerExitCode = -1
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
try {
|
|
# Windows PowerShell exposes native stderr as ErrorRecord objects. Model
|
|
# libraries legitimately emit warnings there, so merge the stream and
|
|
# fail only on the native process exit code.
|
|
$ErrorActionPreference = "Continue"
|
|
& docker @arguments 2>&1 | ForEach-Object { Write-Output $_ }
|
|
$dockerExitCode = $LASTEXITCODE
|
|
}
|
|
finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
if ($dockerExitCode -ne 0) {
|
|
throw "LAB V1 container failed with exit code $dockerExitCode"
|
|
}
|
|
}
|
|
|
|
function Export-RavnovesFrames {
|
|
param([string]$Destination)
|
|
Assert-FileIdentity -Path $RavnovesVideo -ExpectedBytes (Get-Item -LiteralPath $RavnovesVideo).Length -ExpectedSha256 $RavnovesSha256
|
|
New-Item -ItemType Directory -Path $Destination | Out-Null
|
|
$expression = ($frameIndices | ForEach-Object { "eq(n\,$_ )" }) -join "+"
|
|
$temporaryPattern = Join-Path $Destination "selected-%03d.png"
|
|
& ffmpeg -hide_banner -loglevel error -i $RavnovesVideo -vf "select='$expression'" -fps_mode vfr $temporaryPattern
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "RAVNOVES exact frame extraction failed"
|
|
}
|
|
$selected = @(Get-ChildItem -LiteralPath $Destination -Filter "selected-*.png" | Sort-Object Name)
|
|
if ($selected.Count -ne $frameIndices.Count) {
|
|
throw "RAVNOVES frame island changed: expected $($frameIndices.Count), got $($selected.Count)"
|
|
}
|
|
for ($index = 0; $index -lt $selected.Count; $index++) {
|
|
$target = Join-Path $Destination ("frame-{0:D6}.png" -f $frameIndices[$index])
|
|
Move-Item -LiteralPath $selected[$index].FullName -Destination $target
|
|
}
|
|
}
|
|
|
|
function Export-RavnovesVideoFrames {
|
|
param([string]$Destination)
|
|
Assert-FileIdentity -Path $RavnovesVideo -ExpectedBytes (Get-Item -LiteralPath $RavnovesVideo).Length -ExpectedSha256 $RavnovesSha256
|
|
New-Item -ItemType Directory -Path $Destination | Out-Null
|
|
$decodedRoot = "{0}-decoded-by-pts" -f $Destination
|
|
$packetsPath = "{0}-packets.csv" -f $Destination
|
|
New-Item -ItemType Directory -Path $decodedRoot | Out-Null
|
|
& ffprobe -v error -select_streams v:0 -show_packets -show_entries packet=pts,flags -of csv=p=0 -o $packetsPath $RavnovesVideo
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "RAVNOVES full-video packet probe failed"
|
|
}
|
|
$packetRows = @(Get-Content -LiteralPath $packetsPath | Select-Object -First $RavnovesExpectedFrameCount)
|
|
if ($packetRows.Count -ne $RavnovesExpectedFrameCount) {
|
|
throw "RAVNOVES full-video packet sequence changed"
|
|
}
|
|
& ffmpeg -hide_banner -loglevel error `
|
|
-hwaccel cuda -hwaccel_output_format cuda -c:v h264_cuvid `
|
|
-err_detect ignore_err -flags +output_corrupt -copyts `
|
|
-i $RavnovesVideo -map 0:v:0 -vf "hwdownload,format=nv12" `
|
|
-fps_mode passthrough -enc_time_base demux -frames:v $RavnovesExpectedFrameCount `
|
|
-frame_pts 1 (Join-Path $decodedRoot "frame-%d.png")
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "RAVNOVES full-video frame extraction failed"
|
|
}
|
|
$decodedCount = @(Get-ChildItem -LiteralPath $decodedRoot -File -Filter "frame-*.png").Count
|
|
$repairs = @()
|
|
for ($index = 0; $index -lt $RavnovesExpectedFrameCount; $index++) {
|
|
$columns = ([string]$packetRows[$index]).Split(",")
|
|
if ($columns.Count -lt 2) {
|
|
throw "RAVNOVES full-video packet row is malformed"
|
|
}
|
|
$pts = [int64]::Parse($columns[0].Trim(), [Globalization.CultureInfo]::InvariantCulture)
|
|
$decodedPath = Join-Path $decodedRoot ("frame-{0}.png" -f $pts)
|
|
$canonicalPath = Join-Path $Destination ("frame-{0:D6}.png" -f ($index + 1))
|
|
if (Test-Path -LiteralPath $decodedPath -PathType Leaf) {
|
|
Move-Item -LiteralPath $decodedPath -Destination $canonicalPath
|
|
continue
|
|
}
|
|
if ($index -eq 0 -or $repairs.Count -ge 1) {
|
|
throw "RAVNOVES source contains more than one recoverable decoder gap"
|
|
}
|
|
$previousPath = Join-Path $Destination ("frame-{0:D6}.png" -f $index)
|
|
Copy-Item -LiteralPath $previousPath -Destination $canonicalPath
|
|
$repairs += [ordered]@{
|
|
sequence = $index + 1
|
|
packet_pts = $pts
|
|
method = "duplicate-previous-decoded-frame"
|
|
}
|
|
}
|
|
Remove-Item -LiteralPath $decodedRoot -Recurse -Force
|
|
Remove-Item -LiteralPath $packetsPath -Force
|
|
[ordered]@{
|
|
schema_version = "missioncore.recorded-video-decode-repair/v1"
|
|
decoder = "ffmpeg-h264_cuvid-output-corrupt"
|
|
packets_requested = $RavnovesExpectedFrameCount
|
|
frames_decoded = $decodedCount
|
|
repaired_frame_count = $repairs.Count
|
|
repairs = $repairs
|
|
} | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (
|
|
Join-Path (Split-Path $Destination -Parent) "decode-repair.json"
|
|
) -Encoding utf8
|
|
$frames = @(Get-ChildItem -LiteralPath $Destination -File -Filter "frame-*.png" | Sort-Object Name)
|
|
$lastFrameName = "frame-{0:D6}.png" -f $RavnovesExpectedFrameCount
|
|
if ($frames.Count -ne $RavnovesExpectedFrameCount -or $frames[0].Name -ne "frame-000001.png" -or $frames[-1].Name -ne $lastFrameName) {
|
|
throw "RAVNOVES full-video frame sequence changed"
|
|
}
|
|
}
|
|
|
|
if ($Mode -eq "Status") {
|
|
$imageIdentity = & docker image inspect $image --format "{{.Id}}" 2>$null
|
|
[ordered]@{
|
|
schema_version = "missioncore.lab-v1-goose-runner-status/v1"
|
|
observed_at_utc = [DateTime]::UtcNow.ToString("o")
|
|
worker_id = "worker-006"
|
|
image = $image
|
|
image_id = if ($LASTEXITCODE -eq 0) { $imageIdentity } else { $null }
|
|
canonical_triton = Get-CanonicalTritonIdentity
|
|
asset_root = $AssetRoot
|
|
output_root = $OutputRoot
|
|
candidate = $candidateKey
|
|
} | ConvertTo-Json -Depth 6
|
|
exit 0
|
|
}
|
|
|
|
Assert-RunnerInputs
|
|
$canonicalBefore = Get-CanonicalTritonIdentity
|
|
try {
|
|
if ($Mode -eq "Build") {
|
|
if (-not (Test-Path -LiteralPath (Join-Path $dockerConfig "config.json") -PathType Leaf)) {
|
|
throw "Isolated Docker client configuration is unavailable"
|
|
}
|
|
$previousDockerConfig = $env:DOCKER_CONFIG
|
|
try {
|
|
$env:DOCKER_CONFIG = $dockerConfig
|
|
& docker build --pull=false --label "com.nodedc.component=mission-core-lab-v1-goose" --label "com.nodedc.authority=shadow-only" --tag $image $contextRoot
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "LAB V1 image build failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
finally {
|
|
$env:DOCKER_CONFIG = $previousDockerConfig
|
|
}
|
|
}
|
|
elseif ($Mode -eq "Probe") {
|
|
$runRoot = New-RunRoot -Kind "probe"
|
|
Invoke-IsolatedRun -RunMode "goose" -RunRoot $runRoot -Limit 8
|
|
}
|
|
elseif ($Mode -eq "Validate") {
|
|
$runRoot = New-RunRoot -Kind "validation"
|
|
Invoke-IsolatedRun -RunMode "goose" -RunRoot $runRoot -Limit 0
|
|
}
|
|
elseif ($Mode -eq "Ravnoves") {
|
|
$runRoot = New-RunRoot -Kind "ravnoves"
|
|
$framesRoot = Join-Path $runRoot "input-frames"
|
|
Export-RavnovesFrames -Destination $framesRoot
|
|
Invoke-IsolatedRun -RunMode "ravnoves" -RunRoot $runRoot -Limit 0 -FramesRoot $framesRoot
|
|
}
|
|
elseif ($Mode -eq "RavnovesVideo") {
|
|
if ($candidateKey -ne "ddrnet") {
|
|
throw "Full-video shadow is admitted only for the selected DDRNet candidate"
|
|
}
|
|
$runRoot = New-RunRoot -Kind "ravnoves-video"
|
|
$framesRoot = Join-Path $runRoot "input-frames"
|
|
Export-RavnovesVideoFrames -Destination $framesRoot
|
|
Invoke-IsolatedRun -RunMode "ravnoves-video" -RunRoot $runRoot -Limit 0 -FramesRoot $framesRoot
|
|
Copy-Item -LiteralPath (Join-Path $runRoot "decode-repair.json") -Destination (
|
|
Join-Path $runRoot "result\decode-repair.json"
|
|
)
|
|
Remove-Item -LiteralPath $framesRoot -Recurse -Force
|
|
}
|
|
}
|
|
finally {
|
|
$canonicalAfter = Get-CanonicalTritonIdentity
|
|
if ($canonicalAfter -ne $canonicalBefore) {
|
|
throw "Canonical Triton identity changed during LAB V1 work"
|
|
}
|
|
}
|