NODEDC_MISSION_CORE/experiments/perception/worker/Invoke-E4FullSessionSegment...

422 lines
19 KiB
PowerShell

[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[string]$JobRoot,
[Parameter(Mandatory = $true)]
[string]$RunnerPath,
[Parameter(Mandatory = $true)]
[string]$ProfilePath,
[Parameter(Mandatory = $true)]
[string]$ValidFovRoot,
[ValidateRange(0, 256)]
[int]$PilotFrames = 0,
[ValidateRange(1, 1000)]
[int]$FreeGiBFloor = 360,
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3"
)
$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue"
function Assert-LastExitCode {
param([string]$Operation)
if ($LASTEXITCODE -ne 0) {
throw "$Operation failed with exit code $LASTEXITCODE"
}
}
function Assert-RegularFile {
param([string]$Path, [string]$Label)
$item = Get-Item -LiteralPath $Path -Force
if (-not $item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
return $item.FullName
}
throw "$Label is not a regular file"
}
function Assert-Directory {
param([string]$Path, [string]$Label)
$item = Get-Item -LiteralPath $Path -Force
if ($item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
return $item.FullName
}
throw "$Label is not a real directory"
}
function Assert-DDrivePath {
param([string]$Path, [string]$Label)
$fullPath = [IO.Path]::GetFullPath($Path)
if ([IO.Path]::GetPathRoot($fullPath).TrimEnd("\") -ine "D:") {
throw "$Label must be stored on D:"
}
return $fullPath
}
function Get-DFreeBytes {
return [int64](Get-PSDrive -Name D).Free
}
function Assert-FreeSpace {
param(
[string]$Phase,
[int64]$RequiredAdditionalBytes = 0
)
$freeBytes = Get-DFreeBytes
$floorBytes = [int64]$FreeGiBFloor * 1GB
$requiredBytes = $floorBytes + $RequiredAdditionalBytes
Write-Host (
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3} REQUIRED_ADDITIONAL_GIB={4}" -f
$Phase,
$freeBytes,
[math]::Round($freeBytes / 1GB, 3),
$FreeGiBFloor,
[math]::Round($RequiredAdditionalBytes / 1GB, 3)
)
if ($freeBytes -lt $requiredBytes) {
throw "D: does not have the guarded LAB E4 working-set reserve during $Phase"
}
return $freeBytes
}
function Convert-ToDockerPath {
param([string]$Path)
return $Path.Replace("\", "/")
}
$jobDirectory = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $JobRoot).Path "Job root") "Job root"
$runner = Assert-DDrivePath (Assert-RegularFile (Resolve-Path -LiteralPath $RunnerPath).Path "LAB E4 runner") "LAB E4 runner"
$profile = Assert-DDrivePath (Assert-RegularFile (Resolve-Path -LiteralPath $ProfilePath).Path "LAB E4 profile") "LAB E4 profile"
$validFov = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $ValidFovRoot).Path "Valid-FOV root") "Valid-FOV root"
$runtime = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $RuntimeRoot).Path "Runtime root") "Runtime root"
$runnerRoot = Split-Path $runner -Parent
if ((Split-Path $profile -Parent) -ne $runnerRoot) {
throw "LAB E4 runner and profile must share one read-only mount"
}
foreach ($dependency in @(
"run_recorded_perception_epoch.py",
"run_e3_rectified_segmentation.py",
"run_evaluation_prelabels.py"
)) {
$null = Assert-RegularFile (Join-Path $runnerRoot $dependency) "LAB E4 runner dependency"
}
$job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json
if ($job.schema_version -ne "missioncore.compute-job/v1" -or $job.job_id -ne (Split-Path $jobDirectory -Leaf)) {
throw "Compute job manifest is incompatible"
}
$sourceId = [string]$job.input.source_id
$epoch = [int]$job.input.codec_epoch
$fullFrameCount = [int]$job.input.segment_count
$activeFrameCount = if ($PilotFrames -gt 0) { $PilotFrames } else { $fullFrameCount }
$timelineStart = [double]$job.input.timeline.start_seconds
$timelineEnd = [double]$job.input.timeline.end_seconds
$timelineDuration = $timelineEnd - $timelineStart
if (
$sourceId -ne "sensor.camera.right" -or
$fullFrameCount -lt 1 -or
$activeFrameCount -gt $fullFrameCount -or
$timelineDuration -le 0
) {
throw "LAB E4 camera job contract is invalid"
}
Write-Output ("PHASE=job-manifest-validated JOB={0} FRAMES={1}/{2}" -f $job.job_id, $activeFrameCount, $fullFrameCount)
$epochRoot = Join-Path $jobDirectory ("input\camera\{0}\epoch-{1}" -f $sourceId, $epoch)
$epochRoot = Assert-DDrivePath (Assert-Directory $epochRoot "Camera epoch") "Camera epoch"
$initPath = Assert-RegularFile (Join-Path $epochRoot "init.mp4") "Camera init"
$segmentsRoot = Assert-Directory (Join-Path $epochRoot "segments") "Camera segments"
$derivedRoot = Join-Path $runtime "derived"
$tmpRoot = Join-Path $runtime "tmp"
$cacheRoot = Assert-DDrivePath (Assert-Directory (Join-Path $runtime "cache\perception-e3-models-v1") "EoMT cache") "EoMT cache"
$e3Environment = Assert-DDrivePath (Assert-Directory (Join-Path $derivedRoot "perception-e3-opencv413092-v1") "E3 dependency environment") "E3 dependency environment"
$torchEnvironment = Assert-DDrivePath (Assert-Directory (Join-Path $derivedRoot "perception-p0-env-v1") "Torch environment") "Torch environment"
$transformersEnvironment = Assert-DDrivePath (Assert-Directory (Join-Path $derivedRoot "perception-p0-transformers4576-v1") "Transformers environment") "Transformers environment"
$null = New-Item -ItemType Directory -Path $derivedRoot -Force
$null = New-Item -ItemType Directory -Path $tmpRoot -Force
# Conservative upper bound: decoded RGB + RGB overlay + one-byte semantic mask,
# plus 20% filesystem/encoding overhead and the reconstructed source stream.
$pixelWorkingSet = [int64]$activeFrameCount * 800 * 600 * 7
$workingSetReserve = [int64][math]::Ceiling(($pixelWorkingSet * 1.2) + [int64]$job.input.byte_length)
$freeBytesBefore = Assert-FreeSpace "preflight" $workingSetReserve
& docker image inspect $ContainerImage *> $null
Assert-LastExitCode "Existing container image inspection"
$runnerName = Split-Path $runner -Leaf
$profileName = Split-Path $profile -Leaf
$orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
$preflightArgs = @(
"run", "--rm", "--gpus", "all", "--network", "none", "--read-only",
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
"--pids-limit", "512", "--shm-size", "4g",
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
"-e", "TORCH_HOME=/cache/torch",
"-e", "HF_HOME=/cache/huggingface",
"-e", "HF_HUB_OFFLINE=1",
"-e", "TRANSFORMERS_OFFLINE=1",
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"),
"-v", ((Convert-ToDockerPath $e3Environment) + ":/environment:ro"),
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
"--entrypoint", "python3",
$ContainerImage,
("/runner/{0}" -f $runnerName), "preflight",
"--job", "/job",
"--profile", ("/runner/{0}" -f $profileName),
"--valid-fov-root", "/valid-fov",
"--cache", "/cache",
"--environment", "/environment"
)
Write-Output "PHASE=e4-preflight-start"
& docker @preflightArgs
Assert-LastExitCode "LAB E4 preflight"
Write-Output "PHASE=e4-preflight-complete"
$runToken = [Guid]::NewGuid().ToString("N")
$workRoot = Join-Path $tmpRoot ("{0}-e4-{1}" -f $job.job_id, $runToken)
$framesRoot = Join-Path $workRoot "frames"
$streamPath = Join-Path $workRoot "camera.mp4"
$ptsPath = Join-Path $workRoot "pts.json"
$timelinePath = Join-Path $workRoot "timeline.jsonl"
$publishRoot = Join-Path $derivedRoot (".{0}-e4-{1}.publish" -f $job.job_id, $runToken)
$stagingRoot = Join-Path $publishRoot "output"
$null = New-Item -ItemType Directory -Path $framesRoot
$null = New-Item -ItemType Directory -Path $publishRoot
$totalWatch = [Diagnostics.Stopwatch]::StartNew()
$completed = $false
Write-Output "PHASE=e4-private-staging-created"
try {
$stream = [IO.File]::Open($streamPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
try {
$input = [IO.File]::OpenRead($initPath)
try {
$input.CopyTo($stream)
}
finally {
$input.Dispose()
}
for ($sequence = 1; $sequence -le $activeFrameCount; $sequence++) {
$path = Assert-RegularFile (Join-Path $segmentsRoot ("{0}.m4s" -f $sequence)) "Camera segment"
$input = [IO.File]::OpenRead($path)
try {
$input.CopyTo($stream)
}
finally {
$input.Dispose()
}
if ($sequence % 500 -eq 0 -or $sequence -eq $activeFrameCount) {
Write-Output ("PHASE=e4-stream-reconstruction SEGMENTS={0}/{1}" -f $sequence, $activeFrameCount)
}
}
$stream.Flush($true)
}
finally {
$stream.Dispose()
}
$null = Assert-FreeSpace "post-stream-reconstruction"
$extractWatch = [Diagnostics.Stopwatch]::StartNew()
Write-Output "PHASE=e4-frame-extraction-start"
& ffmpeg -hide_banner -loglevel fatal -i $streamPath -map 0:v:0 -fps_mode passthrough -frames:v $activeFrameCount (Join-Path $framesRoot "frame-%06d.png")
Assert-LastExitCode "LAB E4 camera extraction"
& ffprobe -v error -select_streams v:0 -show_entries frame=best_effort_timestamp_time -of json $streamPath | Set-Content -LiteralPath $ptsPath -Encoding utf8
Assert-LastExitCode "LAB E4 camera timestamp probe"
$decodedFrames = @(Get-ChildItem -LiteralPath $framesRoot -File -Filter "frame-*.png")
$ptsDocument = Get-Content -LiteralPath $ptsPath -Raw | ConvertFrom-Json
$pts = @($ptsDocument.frames)
if ($decodedFrames.Count -ne $activeFrameCount -or $pts.Count -lt $activeFrameCount) {
throw "Decoded LAB E4 frame count differs from the requested camera epoch"
}
$firstEpochSeconds = [double]::Parse(
([string]$pts[0].best_effort_timestamp_time).Trim(),
[Globalization.CultureInfo]::InvariantCulture
)
$previousEpochSeconds = -1.0
$timelineWriter = [IO.StreamWriter]::new($timelinePath, $false, [Text.UTF8Encoding]::new($false))
try {
for ($index = 0; $index -lt $activeFrameCount; $index++) {
$epochSeconds = [double]::Parse(
([string]$pts[$index].best_effort_timestamp_time).Trim(),
[Globalization.CultureInfo]::InvariantCulture
) - $firstEpochSeconds
if ($epochSeconds -le $previousEpochSeconds -or $epochSeconds -gt ($timelineDuration + 0.001)) {
throw "Decoded LAB E4 timestamps are not strictly monotonic inside the camera timeline"
}
$row = [ordered]@{
frame_index = $index
epoch_seconds = $epochSeconds
session_seconds = $timelineStart + $epochSeconds
}
$timelineWriter.WriteLine(($row | ConvertTo-Json -Compress))
$previousEpochSeconds = $epochSeconds
}
$timelineWriter.Flush()
}
finally {
$timelineWriter.Dispose()
}
$extractWatch.Stop()
$freeBytesPostExtract = Assert-FreeSpace "post-frame-extraction"
Write-Output ("PHASE=e4-frame-extraction-complete FRAMES={0}" -f $activeFrameCount)
$runArgs = @(
"run", "--rm", "--gpus", "all", "--network", "none", "--read-only",
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
"--pids-limit", "512", "--shm-size", "4g",
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
"-e", "TORCH_HOME=/cache/torch",
"-e", "HF_HOME=/cache/huggingface",
"-e", "HF_HUB_OFFLINE=1",
"-e", "TRANSFORMERS_OFFLINE=1",
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
"-v", ((Convert-ToDockerPath $framesRoot) + ":/frames:ro"),
"-v", ((Convert-ToDockerPath $workRoot) + ":/work:ro"),
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"),
"-v", ((Convert-ToDockerPath $e3Environment) + ":/environment:ro"),
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
"--entrypoint", "python3",
$ContainerImage,
("/runner/{0}" -f $runnerName), "run",
"--job", "/job",
"--profile", ("/runner/{0}" -f $profileName),
"--valid-fov-root", "/valid-fov",
"--frames", "/frames",
"--timeline", "/work/timeline.jsonl",
"--cache", "/cache",
"--environment", "/environment",
"--output", "/publish/output",
"--frame-limit", [string]$activeFrameCount,
"--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB),
"--orchestrator-sha256", $orchestratorSha256,
"--container-image", $ContainerImage,
"--telemetry-interval-seconds", "1"
)
Write-Output ("PHASE=e4-inference-start FRAMES={0}" -f $activeFrameCount)
& docker @runArgs
Assert-LastExitCode "LAB E4 semantic inference"
$freeBytesPostInference = Assert-FreeSpace "post-inference"
Write-Output "PHASE=e4-inference-complete"
if ($PilotFrames -gt 0) {
$pilotParent = Join-Path $derivedRoot "e4-pilots"
$null = New-Item -ItemType Directory -Path $pilotParent -Force
$pilotRoot = Join-Path $pilotParent ("pilot-{0}-{1}" -f $activeFrameCount, $runToken)
Move-Item -LiteralPath $stagingRoot -Destination $pilotRoot
Remove-Item -LiteralPath $publishRoot -Force
$completed = $true
Write-Output ("PILOT_ROOT={0}" -f $pilotRoot)
Write-Output ("PILOT_FRAMES={0}" -f $activeFrameCount)
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
Write-Output ("DISK_FREE_BYTES_POST_EXTRACT={0}" -f $freeBytesPostExtract)
Write-Output ("DISK_FREE_BYTES_POST_INFERENCE={0}" -f $freeBytesPostInference)
return
}
$videoPath = Join-Path $stagingRoot "perception.mp4"
$encodeWatch = [Diagnostics.Stopwatch]::StartNew()
$averageFps = $fullFrameCount / $timelineDuration
$fpsText = $averageFps.ToString("0.#########", [Globalization.CultureInfo]::InvariantCulture)
$durationText = $timelineDuration.ToString("0.#########", [Globalization.CultureInfo]::InvariantCulture)
& ffmpeg -hide_banner -loglevel error -framerate $fpsText -i (Join-Path $stagingRoot "overlay-frames\frame-%06d.png") -t $durationText -c:v h264_nvenc -preset p4 -tune hq -rc vbr -cq 21 -b:v 0 -pix_fmt yuv420p -movflags +faststart $videoPath
Assert-LastExitCode "LAB E4 video encoding"
$encodeWatch.Stop()
$null = Assert-FreeSpace "post-video-encoding"
Write-Output "PHASE=e4-video-encoding-complete"
$masksPath = Join-Path $stagingRoot "masks.tar.gz"
$archiveWatch = [Diagnostics.Stopwatch]::StartNew()
& tar.exe -czf $masksPath -C $stagingRoot semantic-masks
Assert-LastExitCode "LAB E4 mask archive publication"
$archiveWatch.Stop()
$freeBytesPostArtifacts = Assert-FreeSpace "post-mask-archive"
Write-Output "PHASE=e4-mask-archive-complete"
$totalWatch.Stop()
$finalizeArgs = @(
"run", "--rm", "--network", "none", "--read-only",
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
"--pids-limit", "128", "--tmpfs", "/tmp:rw,noexec,nosuid,size=256m",
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
"-v", ((Convert-ToDockerPath $stagingRoot) + ":/output:rw"),
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
"--entrypoint", "python3",
$ContainerImage,
("/runner/{0}" -f $runnerName), "finalize",
"--output", "/output",
"--video", "/output/perception.mp4",
"--masks", "/output/masks.tar.gz",
"--extract-seconds", $extractWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
"--encode-seconds", $encodeWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
"--archive-seconds", $archiveWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
"--wall-seconds", $totalWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
"--encoder", "ffmpeg-h264_nvenc-p4-cq21-yuv420p-faststart",
"--disk-free-before-bytes", [string]$freeBytesBefore,
"--disk-free-post-extract-bytes", [string]$freeBytesPostExtract,
"--disk-free-post-inference-bytes", [string]$freeBytesPostInference,
"--disk-free-post-artifacts-bytes", [string]$freeBytesPostArtifacts,
"--disk-floor-bytes", [string]([int64]$FreeGiBFloor * 1GB),
"--working-set-reserve-bytes", [string]$workingSetReserve
)
& docker @finalizeArgs
Assert-LastExitCode "LAB E4 result finalization"
$result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json
if (
$result.schema_version -ne "missioncore.recorded-perception-result/v2" -or
$result.result_id -notmatch "^result-[a-f0-9]{64}$" -or
[int]$result.frames_processed -ne $fullFrameCount -or
$result.ground_truth -ne $false
) {
throw "Final LAB E4 result manifest is incompatible"
}
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
if (Test-Path -LiteralPath $finalRoot) {
throw "An immutable result with the same LAB E4 identity already exists: $finalRoot"
}
foreach ($temporaryChild in @("overlay-frames", "semantic-masks")) {
$temporaryPath = Join-Path $stagingRoot $temporaryChild
if (Test-Path -LiteralPath $temporaryPath) {
Remove-Item -LiteralPath $temporaryPath -Recurse -Force
}
}
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
Remove-Item -LiteralPath $publishRoot -Force
$completed = $true
$freeBytesFinal = Assert-FreeSpace "post-publication"
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
Write-Output ("RESULT_ID={0}" -f $result.result_id)
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
Write-Output ("DISK_FREE_BYTES_POST_EXTRACT={0}" -f $freeBytesPostExtract)
Write-Output ("DISK_FREE_BYTES_POST_INFERENCE={0}" -f $freeBytesPostInference)
Write-Output ("DISK_FREE_BYTES_POST_ARTIFACTS={0}" -f $freeBytesPostArtifacts)
Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeBytesFinal)
}
finally {
if (Test-Path -LiteralPath $workRoot) {
Remove-Item -LiteralPath $workRoot -Recurse -Force
}
if (-not $completed -and (Test-Path -LiteralPath $publishRoot)) {
Remove-Item -LiteralPath $publishRoot -Recurse -Force
}
}