[CmdletBinding()] param( [Parameter(Mandatory = $true)] [string]$JobRoot, [Parameter(Mandatory = $true)] [string]$RunnerPath, [Parameter(Mandatory = $true)] [string]$ProfilePath, [Parameter(Mandatory = $true)] [string]$ValidFovRoot, [ValidateRange(0, 256)] [int]$PilotFrames = 0, [ValidateRange(1, 1000)] [int]$FreeGiBFloor = 360, [string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime", [string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3" ) $ErrorActionPreference = "Stop" $ProgressPreference = "SilentlyContinue" function Assert-LastExitCode { param([string]$Operation) if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" } } function Assert-RegularFile { param([string]$Path, [string]$Label) $item = Get-Item -LiteralPath $Path -Force if (-not $item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) { return $item.FullName } throw "$Label is not a regular file" } function Assert-Directory { param([string]$Path, [string]$Label) $item = Get-Item -LiteralPath $Path -Force if ($item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) { return $item.FullName } throw "$Label is not a real directory" } function Assert-DDrivePath { param([string]$Path, [string]$Label) $fullPath = [IO.Path]::GetFullPath($Path) if ([IO.Path]::GetPathRoot($fullPath).TrimEnd("\") -ine "D:") { throw "$Label must be stored on D:" } return $fullPath } function Get-DFreeBytes { return [int64](Get-PSDrive -Name D).Free } function Assert-FreeSpace { param( [string]$Phase, [int64]$RequiredAdditionalBytes = 0 ) $freeBytes = Get-DFreeBytes $floorBytes = [int64]$FreeGiBFloor * 1GB $requiredBytes = $floorBytes + $RequiredAdditionalBytes Write-Host ( "DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3} REQUIRED_ADDITIONAL_GIB={4}" -f $Phase, $freeBytes, [math]::Round($freeBytes / 1GB, 3), $FreeGiBFloor, [math]::Round($RequiredAdditionalBytes / 1GB, 3) ) if ($freeBytes -lt $requiredBytes) { throw "D: does not have the guarded LAB E4 working-set reserve during $Phase" } return $freeBytes } function Convert-ToDockerPath { param([string]$Path) return $Path.Replace("\", "/") } $jobDirectory = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $JobRoot).Path "Job root") "Job root" $runner = Assert-DDrivePath (Assert-RegularFile (Resolve-Path -LiteralPath $RunnerPath).Path "LAB E4 runner") "LAB E4 runner" $profile = Assert-DDrivePath (Assert-RegularFile (Resolve-Path -LiteralPath $ProfilePath).Path "LAB E4 profile") "LAB E4 profile" $validFov = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $ValidFovRoot).Path "Valid-FOV root") "Valid-FOV root" $runtime = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $RuntimeRoot).Path "Runtime root") "Runtime root" $runnerRoot = Split-Path $runner -Parent if ((Split-Path $profile -Parent) -ne $runnerRoot) { throw "LAB E4 runner and profile must share one read-only mount" } foreach ($dependency in @( "run_recorded_perception_epoch.py", "run_e3_rectified_segmentation.py", "run_evaluation_prelabels.py" )) { $null = Assert-RegularFile (Join-Path $runnerRoot $dependency) "LAB E4 runner dependency" } $job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json if ($job.schema_version -ne "missioncore.compute-job/v1" -or $job.job_id -ne (Split-Path $jobDirectory -Leaf)) { throw "Compute job manifest is incompatible" } $sourceId = [string]$job.input.source_id $epoch = [int]$job.input.codec_epoch $fullFrameCount = [int]$job.input.segment_count $activeFrameCount = if ($PilotFrames -gt 0) { $PilotFrames } else { $fullFrameCount } $timelineStart = [double]$job.input.timeline.start_seconds $timelineEnd = [double]$job.input.timeline.end_seconds $timelineDuration = $timelineEnd - $timelineStart if ( $sourceId -ne "sensor.camera.right" -or $fullFrameCount -lt 1 -or $activeFrameCount -gt $fullFrameCount -or $timelineDuration -le 0 ) { throw "LAB E4 camera job contract is invalid" } Write-Output ("PHASE=job-manifest-validated JOB={0} FRAMES={1}/{2}" -f $job.job_id, $activeFrameCount, $fullFrameCount) $epochRoot = Join-Path $jobDirectory ("input\camera\{0}\epoch-{1}" -f $sourceId, $epoch) $epochRoot = Assert-DDrivePath (Assert-Directory $epochRoot "Camera epoch") "Camera epoch" $initPath = Assert-RegularFile (Join-Path $epochRoot "init.mp4") "Camera init" $segmentsRoot = Assert-Directory (Join-Path $epochRoot "segments") "Camera segments" $derivedRoot = Join-Path $runtime "derived" $tmpRoot = Join-Path $runtime "tmp" $cacheRoot = Assert-DDrivePath (Assert-Directory (Join-Path $runtime "cache\perception-e3-models-v1") "EoMT cache") "EoMT cache" $e3Environment = Assert-DDrivePath (Assert-Directory (Join-Path $derivedRoot "perception-e3-opencv413092-v1") "E3 dependency environment") "E3 dependency environment" $torchEnvironment = Assert-DDrivePath (Assert-Directory (Join-Path $derivedRoot "perception-p0-env-v1") "Torch environment") "Torch environment" $transformersEnvironment = Assert-DDrivePath (Assert-Directory (Join-Path $derivedRoot "perception-p0-transformers4576-v1") "Transformers environment") "Transformers environment" $null = New-Item -ItemType Directory -Path $derivedRoot -Force $null = New-Item -ItemType Directory -Path $tmpRoot -Force # Conservative upper bound: decoded RGB + RGB overlay + one-byte semantic mask, # plus 20% filesystem/encoding overhead and the reconstructed source stream. $pixelWorkingSet = [int64]$activeFrameCount * 800 * 600 * 7 $workingSetReserve = [int64][math]::Ceiling(($pixelWorkingSet * 1.2) + [int64]$job.input.byte_length) $freeBytesBefore = Assert-FreeSpace "preflight" $workingSetReserve & docker image inspect $ContainerImage *> $null Assert-LastExitCode "Existing container image inspection" $runnerName = Split-Path $runner -Leaf $profileName = Split-Path $profile -Leaf $orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant() $preflightArgs = @( "run", "--rm", "--gpus", "all", "--network", "none", "--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL", "--pids-limit", "512", "--shm-size", "4g", "--tmpfs", "/tmp:rw,noexec,nosuid,size=2g", "-e", "PYTHONPATH=/opt/transformers:/opt/env", "-e", "TORCH_HOME=/cache/torch", "-e", "HF_HOME=/cache/huggingface", "-e", "HF_HUB_OFFLINE=1", "-e", "TRANSFORMERS_OFFLINE=1", "-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"), "-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"), "-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"), "-v", ((Convert-ToDockerPath $e3Environment) + ":/environment:ro"), "-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"), "-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"), "-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"), "--entrypoint", "python3", $ContainerImage, ("/runner/{0}" -f $runnerName), "preflight", "--job", "/job", "--profile", ("/runner/{0}" -f $profileName), "--valid-fov-root", "/valid-fov", "--cache", "/cache", "--environment", "/environment" ) Write-Output "PHASE=e4-preflight-start" & docker @preflightArgs Assert-LastExitCode "LAB E4 preflight" Write-Output "PHASE=e4-preflight-complete" $runToken = [Guid]::NewGuid().ToString("N") $workRoot = Join-Path $tmpRoot ("{0}-e4-{1}" -f $job.job_id, $runToken) $framesRoot = Join-Path $workRoot "frames" $streamPath = Join-Path $workRoot "camera.mp4" $ptsPath = Join-Path $workRoot "pts.json" $timelinePath = Join-Path $workRoot "timeline.jsonl" $publishRoot = Join-Path $derivedRoot (".{0}-e4-{1}.publish" -f $job.job_id, $runToken) $stagingRoot = Join-Path $publishRoot "output" $null = New-Item -ItemType Directory -Path $framesRoot $null = New-Item -ItemType Directory -Path $publishRoot $totalWatch = [Diagnostics.Stopwatch]::StartNew() $completed = $false Write-Output "PHASE=e4-private-staging-created" try { $stream = [IO.File]::Open($streamPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None) try { $input = [IO.File]::OpenRead($initPath) try { $input.CopyTo($stream) } finally { $input.Dispose() } for ($sequence = 1; $sequence -le $activeFrameCount; $sequence++) { $path = Assert-RegularFile (Join-Path $segmentsRoot ("{0}.m4s" -f $sequence)) "Camera segment" $input = [IO.File]::OpenRead($path) try { $input.CopyTo($stream) } finally { $input.Dispose() } if ($sequence % 500 -eq 0 -or $sequence -eq $activeFrameCount) { Write-Output ("PHASE=e4-stream-reconstruction SEGMENTS={0}/{1}" -f $sequence, $activeFrameCount) } } $stream.Flush($true) } finally { $stream.Dispose() } $null = Assert-FreeSpace "post-stream-reconstruction" $extractWatch = [Diagnostics.Stopwatch]::StartNew() Write-Output "PHASE=e4-frame-extraction-start" & ffmpeg -hide_banner -loglevel fatal -i $streamPath -map 0:v:0 -fps_mode passthrough -frames:v $activeFrameCount (Join-Path $framesRoot "frame-%06d.png") Assert-LastExitCode "LAB E4 camera extraction" & ffprobe -v error -select_streams v:0 -show_entries frame=best_effort_timestamp_time -of json $streamPath | Set-Content -LiteralPath $ptsPath -Encoding utf8 Assert-LastExitCode "LAB E4 camera timestamp probe" $decodedFrames = @(Get-ChildItem -LiteralPath $framesRoot -File -Filter "frame-*.png") $ptsDocument = Get-Content -LiteralPath $ptsPath -Raw | ConvertFrom-Json $pts = @($ptsDocument.frames) if ($decodedFrames.Count -ne $activeFrameCount -or $pts.Count -lt $activeFrameCount) { throw "Decoded LAB E4 frame count differs from the requested camera epoch" } $firstEpochSeconds = [double]::Parse( ([string]$pts[0].best_effort_timestamp_time).Trim(), [Globalization.CultureInfo]::InvariantCulture ) $previousEpochSeconds = -1.0 $timelineWriter = [IO.StreamWriter]::new($timelinePath, $false, [Text.UTF8Encoding]::new($false)) try { for ($index = 0; $index -lt $activeFrameCount; $index++) { $epochSeconds = [double]::Parse( ([string]$pts[$index].best_effort_timestamp_time).Trim(), [Globalization.CultureInfo]::InvariantCulture ) - $firstEpochSeconds if ($epochSeconds -le $previousEpochSeconds -or $epochSeconds -gt ($timelineDuration + 0.001)) { throw "Decoded LAB E4 timestamps are not strictly monotonic inside the camera timeline" } $row = [ordered]@{ frame_index = $index epoch_seconds = $epochSeconds session_seconds = $timelineStart + $epochSeconds } $timelineWriter.WriteLine(($row | ConvertTo-Json -Compress)) $previousEpochSeconds = $epochSeconds } $timelineWriter.Flush() } finally { $timelineWriter.Dispose() } $extractWatch.Stop() $freeBytesPostExtract = Assert-FreeSpace "post-frame-extraction" Write-Output ("PHASE=e4-frame-extraction-complete FRAMES={0}" -f $activeFrameCount) $runArgs = @( "run", "--rm", "--gpus", "all", "--network", "none", "--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL", "--pids-limit", "512", "--shm-size", "4g", "--tmpfs", "/tmp:rw,noexec,nosuid,size=2g", "-e", "PYTHONPATH=/opt/transformers:/opt/env", "-e", "TORCH_HOME=/cache/torch", "-e", "HF_HOME=/cache/huggingface", "-e", "HF_HUB_OFFLINE=1", "-e", "TRANSFORMERS_OFFLINE=1", "-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"), "-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"), "-v", ((Convert-ToDockerPath $framesRoot) + ":/frames:ro"), "-v", ((Convert-ToDockerPath $workRoot) + ":/work:ro"), "-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"), "-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"), "-v", ((Convert-ToDockerPath $e3Environment) + ":/environment:ro"), "-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"), "-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"), "-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"), "--entrypoint", "python3", $ContainerImage, ("/runner/{0}" -f $runnerName), "run", "--job", "/job", "--profile", ("/runner/{0}" -f $profileName), "--valid-fov-root", "/valid-fov", "--frames", "/frames", "--timeline", "/work/timeline.jsonl", "--cache", "/cache", "--environment", "/environment", "--output", "/publish/output", "--frame-limit", [string]$activeFrameCount, "--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB), "--orchestrator-sha256", $orchestratorSha256, "--container-image", $ContainerImage, "--telemetry-interval-seconds", "1" ) Write-Output ("PHASE=e4-inference-start FRAMES={0}" -f $activeFrameCount) & docker @runArgs Assert-LastExitCode "LAB E4 semantic inference" $freeBytesPostInference = Assert-FreeSpace "post-inference" Write-Output "PHASE=e4-inference-complete" if ($PilotFrames -gt 0) { $pilotParent = Join-Path $derivedRoot "e4-pilots" $null = New-Item -ItemType Directory -Path $pilotParent -Force $pilotRoot = Join-Path $pilotParent ("pilot-{0}-{1}" -f $activeFrameCount, $runToken) Move-Item -LiteralPath $stagingRoot -Destination $pilotRoot Remove-Item -LiteralPath $publishRoot -Force $completed = $true Write-Output ("PILOT_ROOT={0}" -f $pilotRoot) Write-Output ("PILOT_FRAMES={0}" -f $activeFrameCount) Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore) Write-Output ("DISK_FREE_BYTES_POST_EXTRACT={0}" -f $freeBytesPostExtract) Write-Output ("DISK_FREE_BYTES_POST_INFERENCE={0}" -f $freeBytesPostInference) return } $videoPath = Join-Path $stagingRoot "perception.mp4" $encodeWatch = [Diagnostics.Stopwatch]::StartNew() $averageFps = $fullFrameCount / $timelineDuration $fpsText = $averageFps.ToString("0.#########", [Globalization.CultureInfo]::InvariantCulture) $durationText = $timelineDuration.ToString("0.#########", [Globalization.CultureInfo]::InvariantCulture) & ffmpeg -hide_banner -loglevel error -framerate $fpsText -i (Join-Path $stagingRoot "overlay-frames\frame-%06d.png") -t $durationText -c:v h264_nvenc -preset p4 -tune hq -rc vbr -cq 21 -b:v 0 -pix_fmt yuv420p -movflags +faststart $videoPath Assert-LastExitCode "LAB E4 video encoding" $encodeWatch.Stop() $null = Assert-FreeSpace "post-video-encoding" Write-Output "PHASE=e4-video-encoding-complete" $masksPath = Join-Path $stagingRoot "masks.tar.gz" $archiveWatch = [Diagnostics.Stopwatch]::StartNew() & tar.exe -czf $masksPath -C $stagingRoot semantic-masks Assert-LastExitCode "LAB E4 mask archive publication" $archiveWatch.Stop() $freeBytesPostArtifacts = Assert-FreeSpace "post-mask-archive" Write-Output "PHASE=e4-mask-archive-complete" $totalWatch.Stop() $finalizeArgs = @( "run", "--rm", "--network", "none", "--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL", "--pids-limit", "128", "--tmpfs", "/tmp:rw,noexec,nosuid,size=256m", "-e", "PYTHONPATH=/opt/transformers:/opt/env", "-v", ((Convert-ToDockerPath $stagingRoot) + ":/output:rw"), "-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"), "--entrypoint", "python3", $ContainerImage, ("/runner/{0}" -f $runnerName), "finalize", "--output", "/output", "--video", "/output/perception.mp4", "--masks", "/output/masks.tar.gz", "--extract-seconds", $extractWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture), "--encode-seconds", $encodeWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture), "--archive-seconds", $archiveWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture), "--wall-seconds", $totalWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture), "--encoder", "ffmpeg-h264_nvenc-p4-cq21-yuv420p-faststart", "--disk-free-before-bytes", [string]$freeBytesBefore, "--disk-free-post-extract-bytes", [string]$freeBytesPostExtract, "--disk-free-post-inference-bytes", [string]$freeBytesPostInference, "--disk-free-post-artifacts-bytes", [string]$freeBytesPostArtifacts, "--disk-floor-bytes", [string]([int64]$FreeGiBFloor * 1GB), "--working-set-reserve-bytes", [string]$workingSetReserve ) & docker @finalizeArgs Assert-LastExitCode "LAB E4 result finalization" $result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json if ( $result.schema_version -ne "missioncore.recorded-perception-result/v2" -or $result.result_id -notmatch "^result-[a-f0-9]{64}$" -or [int]$result.frames_processed -ne $fullFrameCount -or $result.ground_truth -ne $false ) { throw "Final LAB E4 result manifest is incompatible" } $finalRoot = Join-Path $derivedRoot ([string]$result.result_id) if (Test-Path -LiteralPath $finalRoot) { throw "An immutable result with the same LAB E4 identity already exists: $finalRoot" } foreach ($temporaryChild in @("overlay-frames", "semantic-masks")) { $temporaryPath = Join-Path $stagingRoot $temporaryChild if (Test-Path -LiteralPath $temporaryPath) { Remove-Item -LiteralPath $temporaryPath -Recurse -Force } } Move-Item -LiteralPath $stagingRoot -Destination $finalRoot Remove-Item -LiteralPath $publishRoot -Force $completed = $true $freeBytesFinal = Assert-FreeSpace "post-publication" Write-Output ("RESULT_ROOT={0}" -f $finalRoot) Write-Output ("RESULT_ID={0}" -f $result.result_id) Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore) Write-Output ("DISK_FREE_BYTES_POST_EXTRACT={0}" -f $freeBytesPostExtract) Write-Output ("DISK_FREE_BYTES_POST_INFERENCE={0}" -f $freeBytesPostInference) Write-Output ("DISK_FREE_BYTES_POST_ARTIFACTS={0}" -f $freeBytesPostArtifacts) Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeBytesFinal) } finally { if (Test-Path -LiteralPath $workRoot) { Remove-Item -LiteralPath $workRoot -Recurse -Force } if (-not $completed -and (Test-Path -LiteralPath $publishRoot)) { Remove-Item -LiteralPath $publishRoot -Recurse -Force } }