[CmdletBinding()] param( [Parameter(Mandatory = $true)] [string]$JobRoot, [Parameter(Mandatory = $true)] [string]$RunnerPath, [Parameter(Mandatory = $true)] [string]$ProfilePath, [Parameter(Mandatory = $true)] [string]$DetectorProfilePath, [Parameter(Mandatory = $true)] [string]$SemanticProfilePath, [Parameter(Mandatory = $true)] [string]$ValidFovRoot, [Parameter(Mandatory = $true)] [string]$LidarPackRoot, [ValidateRange(0, 1000000)] [int]$StartFrame = 1000, [ValidateRange(0, 1000000)] [int]$EndFrame = 1600, [ValidateRange(1, 1000)] [int]$FreeGiBFloor = 360, [string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime", [string]$ModelRoot = "D:\NDC_MISSIONCORE\runtime\models\yolox_s", [string]$TritonContainer = "mission-core-triton", [string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3" ) $ErrorActionPreference = "Stop" $ProgressPreference = "SilentlyContinue" function Assert-LastExitCode([string]$Operation) { if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" } } function Resolve-DDirectory([string]$Path, [string]$Label) { $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force $root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") if (-not $item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") { throw "$Label must be a real D: directory" } return $item.FullName } function Resolve-DFile([string]$Path, [string]$Label) { $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force $root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") { throw "$Label must be a regular D: file" } return $item.FullName } function Convert-ToDockerPath([string]$Path) { return $Path.Replace("\", "/") } function Get-DFreeBytes { return [int64](Get-PSDrive -Name D).Free } function Assert-FreeSpace([string]$Phase, [int64]$RequiredAdditionalBytes = 0) { $free = Get-DFreeBytes $floor = [int64]$FreeGiBFloor * 1GB Write-Host ( "DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3} REQUIRED_ADDITIONAL_GIB={4}" -f $Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor, [math]::Round($RequiredAdditionalBytes / 1GB, 3) ) if ($free -lt ($floor + $RequiredAdditionalBytes)) { throw "D: lacks the guarded LAB E10 reserve during $Phase" } return $free } function Test-TritonModelReady { try { $response = Invoke-WebRequest -UseBasicParsing -TimeoutSec 10 ` -Uri "http://127.0.0.1:8000/v2/models/yolox_s/ready" return $response.StatusCode -eq 200 } catch { return $false } } $jobDirectory = Resolve-DDirectory $JobRoot "Job root" $runner = Resolve-DFile $RunnerPath "LAB E10 runner" $profile = Resolve-DFile $ProfilePath "LAB E10 profile" $detectorProfile = Resolve-DFile $DetectorProfilePath "Detector profile" $semanticProfile = Resolve-DFile $SemanticProfilePath "Semantic profile" $validFov = Resolve-DDirectory $ValidFovRoot "Valid-FOV root" $lidarPack = Resolve-DDirectory $LidarPackRoot "LiDAR replay pack" $runtime = Resolve-DDirectory $RuntimeRoot "Runtime root" $model = Resolve-DDirectory $ModelRoot "YOLOX model root" $runnerRoot = Split-Path $runner -Parent foreach ($path in @($profile, $detectorProfile, $semanticProfile)) { if ((Split-Path $path -Parent) -ne $runnerRoot) { throw "Runner and profiles must share one mount" } } foreach ($dependency in @( "e10_fusion_runtime.py", "run_e9_multirate_perception.py", "run_e8_realtime_tracking.py", "run_e5_instance_tracking.py", "run_e4_full_session_segmentation.py", "run_recorded_perception_epoch.py", "run_e3_rectified_segmentation.py", "run_evaluation_prelabels.py" )) { $null = Resolve-DFile (Join-Path $runnerRoot $dependency) "LAB E10 dependency" } $job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json if ($job.schema_version -ne "missioncore.compute-job/v1" -or $job.job_id -ne (Split-Path $jobDirectory -Leaf)) { throw "Compute job manifest is incompatible" } $sourceId = [string]$job.input.source_id $epoch = [int]$job.input.codec_epoch $fullFrameCount = [int]$job.input.segment_count $clipFrameCount = $EndFrame - $StartFrame + 1 $timelineStart = [double]$job.input.timeline.start_seconds if ($sourceId -ne "sensor.camera.right" -or $StartFrame -lt 0 -or $EndFrame -lt $StartFrame -or $EndFrame -ge $fullFrameCount -or $clipFrameCount -lt 2) { throw "LAB E10 clip escapes the camera job" } $lidarManifest = Get-Content -LiteralPath (Join-Path $lidarPack "manifest.json") -Raw | ConvertFrom-Json if ( $lidarManifest.schema_version -ne "missioncore.e10-lidar-replay-pack/v1" -or $lidarManifest.identity.job_id -ne $job.job_id -or [int]$lidarManifest.identity.source_start_frame_index -ne $StartFrame -or [int]$lidarManifest.identity.source_end_frame_index -ne $EndFrame -or [int]$lidarManifest.identity.frame_count -ne $clipFrameCount ) { throw "LAB E10 LiDAR pack differs from the selected camera clip" } Write-Output ("PHASE=inputs-validated JOB={0} CLIP={1}-{2} FRAMES={3} LIDAR_PACK={4}" -f $job.job_id, $StartFrame, $EndFrame, $clipFrameCount, $lidarManifest.pack_id) $epochRoot = Resolve-DDirectory (Join-Path $jobDirectory ("input\camera\{0}\epoch-{1}" -f $sourceId, $epoch)) "Camera epoch" $initPath = Resolve-DFile (Join-Path $epochRoot "init.mp4") "Camera init" $segmentsRoot = Resolve-DDirectory (Join-Path $epochRoot "segments") "Camera segments" $derivedRoot = Join-Path $runtime "derived" $tmpRoot = Join-Path $runtime "tmp" $cacheRoot = Resolve-DDirectory (Join-Path $runtime "cache\perception-e3-models-v1") "EoMT cache" $e3Environment = Resolve-DDirectory (Join-Path $derivedRoot "perception-e3-opencv413092-v1") "E3 environment" $torchEnvironment = Resolve-DDirectory (Join-Path $derivedRoot "perception-p0-env-v1") "Torch environment" $transformersEnvironment = Resolve-DDirectory (Join-Path $derivedRoot "perception-p0-transformers4576-v1") "Transformers environment" $pixelWorkingSet = [int64]$clipFrameCount * 800 * 600 * 3 $partialStreamReserve = [int64][math]::Ceiling(([int64]$job.input.byte_length * ($EndFrame + 1) / $fullFrameCount) * 1.2) $workingSetReserve = [int64][math]::Ceiling($pixelWorkingSet * 1.4) + $partialStreamReserve + 3GB $freeBytesBefore = Assert-FreeSpace "preflight" $workingSetReserve & docker image inspect $ContainerImage *> $null Assert-LastExitCode "Existing container image inspection" $tritonState = docker inspect --format "{{.State.Running}}" $TritonContainer Assert-LastExitCode "Triton container inspection" if ($tritonState.Trim().ToLowerInvariant() -ne "true") { throw "LAB E10 requires the existing Triton container" } $runnerName = Split-Path $runner -Leaf $profileName = Split-Path $profile -Leaf $detectorProfileName = Split-Path $detectorProfile -Leaf $semanticProfileName = Split-Path $semanticProfile -Leaf $lidarMount = "/" + (Split-Path $lidarPack -Leaf) $orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant() $commonMounts = @( "-e", "PYTHONPATH=/runner:/opt/transformers:/opt/env", "-e", "TORCH_HOME=/cache/torch", "-e", "HF_HOME=/cache/huggingface", "-e", "HF_HUB_OFFLINE=1", "-e", "TRANSFORMERS_OFFLINE=1", "-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"), "-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"), "-v", ((Convert-ToDockerPath $lidarPack) + (":{0}:ro" -f $lidarMount)), "-v", ((Convert-ToDockerPath $model) + ":/model:ro"), "-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"), "-v", ((Convert-ToDockerPath $e3Environment) + ":/environment:ro"), "-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"), "-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"), "-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro") ) $commonRunnerArgs = @( "--job", "/job", "--profile", ("/runner/{0}" -f $profileName), "--detector-profile", ("/runner/{0}" -f $detectorProfileName), "--semantic-profile", ("/runner/{0}" -f $semanticProfileName), "--valid-fov-root", "/valid-fov", "--model-root", "/model", "--cache", "/cache", "--environment", "/environment", "--lidar-pack", $lidarMount ) $preflightArgs = @( "run", "--rm", "--gpus", "all", "--network", "none", "--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL", "--pids-limit", "512", "--shm-size", "4g", "--tmpfs", "/tmp:rw,noexec,nosuid,size=2g" ) + $commonMounts + @( "--entrypoint", "python3", $ContainerImage, ("/runner/{0}" -f $runnerName), "preflight" ) + $commonRunnerArgs Write-Output "PHASE=e10-preflight-start" & docker @preflightArgs Assert-LastExitCode "LAB E10 preflight" Write-Output "PHASE=e10-preflight-complete" $modelWasReady = Test-TritonModelReady $loadedByRun = $false if (-not $modelWasReady) { Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 -Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/load" -Method Post -ContentType "application/json" -Body "{}" *> $null $loadedByRun = $true $deadline = [DateTime]::UtcNow.AddSeconds(60) while (-not (Test-TritonModelReady)) { if ([DateTime]::UtcNow -ge $deadline) { throw "YOLOX-S did not become ready" } Start-Sleep -Milliseconds 500 } } Write-Output ("PHASE=e10-model-ready LOADED_BY_RUN={0}" -f $loadedByRun) $token = [Guid]::NewGuid().ToString("N") $workRoot = Join-Path $tmpRoot ("{0}-e10-{1}" -f $job.job_id, $token) $framesRoot = Join-Path $workRoot "frames" $streamPath = Join-Path $workRoot "camera.mp4" $ptsPath = Join-Path $workRoot "pts.json" $timelinePath = Join-Path $workRoot "timeline.jsonl" $publishRoot = Join-Path $derivedRoot (".{0}-e10-{1}.publish" -f $job.job_id, $token) $stagingRoot = Join-Path $publishRoot "output" $null = New-Item -ItemType Directory -Path $framesRoot $null = New-Item -ItemType Directory -Path $publishRoot $completed = $false $totalWatch = [Diagnostics.Stopwatch]::StartNew() try { $stream = [IO.File]::Open($streamPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None) try { $input = [IO.File]::OpenRead($initPath) try { $input.CopyTo($stream) } finally { $input.Dispose() } for ($sequence = 1; $sequence -le ($EndFrame + 1); $sequence++) { $path = Resolve-DFile (Join-Path $segmentsRoot ("{0}.m4s" -f $sequence)) "Camera segment" $input = [IO.File]::OpenRead($path) try { $input.CopyTo($stream) } finally { $input.Dispose() } } $stream.Flush($true) } finally { $stream.Dispose() } $null = Assert-FreeSpace "post-stream-reconstruction" $extractWatch = [Diagnostics.Stopwatch]::StartNew() $selectFilter = "select='between(n,{0},{1})'" -f $StartFrame, $EndFrame & ffmpeg -hide_banner -loglevel fatal -i $streamPath -map 0:v:0 -vf $selectFilter -fps_mode passthrough (Join-Path $framesRoot "frame-%06d.png") Assert-LastExitCode "LAB E10 camera extraction" & ffprobe -v error -select_streams v:0 -show_entries frame=best_effort_timestamp_time -of json $streamPath | Set-Content -LiteralPath $ptsPath -Encoding utf8 Assert-LastExitCode "LAB E10 timestamp probe" $decoded = @(Get-ChildItem -LiteralPath $framesRoot -File -Filter "frame-*.png") $pts = @((Get-Content -LiteralPath $ptsPath -Raw | ConvertFrom-Json).frames) if ($decoded.Count -ne $clipFrameCount -or $pts.Count -ne ($EndFrame + 1)) { throw "LAB E10 decoded frame count changed" } $firstEpochSeconds = [double]::Parse(([string]$pts[0].best_effort_timestamp_time).Trim(), [Globalization.CultureInfo]::InvariantCulture) $writer = [IO.StreamWriter]::new($timelinePath, $false, [Text.UTF8Encoding]::new($false)) try { for ($local = 0; $local -lt $clipFrameCount; $local++) { $source = $StartFrame + $local $epochSeconds = [double]::Parse(([string]$pts[$source].best_effort_timestamp_time).Trim(), [Globalization.CultureInfo]::InvariantCulture) - $firstEpochSeconds $row = [ordered]@{ frame_index = $local sequence = $local + 1 source_frame_index = $source source_sequence = $source + 1 epoch_seconds = $epochSeconds session_seconds = $timelineStart + $epochSeconds } $writer.WriteLine(($row | ConvertTo-Json -Compress)) } $writer.Flush() } finally { $writer.Dispose() } $extractWatch.Stop() $freePostExtract = Assert-FreeSpace "post-frame-extraction" $runArgs = @( "run", "--rm", "--gpus", "all", "--network", ("container:{0}" -f $TritonContainer), "--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL", "--pids-limit", "512", "--shm-size", "4g", "--tmpfs", "/tmp:rw,noexec,nosuid,size=2g" ) + $commonMounts + @( "-v", ((Convert-ToDockerPath $framesRoot) + ":/frames:ro"), "-v", ((Convert-ToDockerPath $workRoot) + ":/work:ro"), "-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"), "--entrypoint", "python3", $ContainerImage, ("/runner/{0}" -f $runnerName), "run" ) + $commonRunnerArgs + @( "--frames", "/frames", "--timeline", "/work/timeline.jsonl", "--output", "/publish/output", "--triton-url", "http://127.0.0.1:8000", "--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB), "--orchestrator-sha256", $orchestratorSha256, "--container-image", $ContainerImage ) Write-Output ("PHASE=e10-integrated-replay-start FRAMES={0}" -f $clipFrameCount) & docker @runArgs Assert-LastExitCode "LAB E10 integrated replay" $freePostReplay = Assert-FreeSpace "post-replay" $result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json if ($result.schema_version -ne "missioncore.e10-integrated-perception-result/v1" -or $result.result_id -notmatch "^e10-integrated-perception-[a-f0-9]{64}$") { throw "LAB E10 result manifest is incompatible" } $finalRoot = Join-Path $derivedRoot ([string]$result.result_id) if (Test-Path -LiteralPath $finalRoot) { throw "Immutable LAB E10 result already exists" } Move-Item -LiteralPath $stagingRoot -Destination $finalRoot Remove-Item -LiteralPath $publishRoot -Force $completed = $true $totalWatch.Stop() $freeFinal = Assert-FreeSpace "post-publication" Write-Output ("RESULT_ROOT={0}" -f $finalRoot) Write-Output ("RESULT_ID={0}" -f $result.result_id) Write-Output ("ACCEPTANCE_STATE={0}" -f $result.acceptance_state) Write-Output ("PREPARATION_SECONDS={0}" -f $extractWatch.Elapsed.TotalSeconds) Write-Output ("ORCHESTRATOR_WALL_SECONDS={0}" -f $totalWatch.Elapsed.TotalSeconds) Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore) Write-Output ("DISK_FREE_BYTES_POST_EXTRACT={0}" -f $freePostExtract) Write-Output ("DISK_FREE_BYTES_POST_REPLAY={0}" -f $freePostReplay) Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeFinal) } finally { if (Test-Path -LiteralPath $workRoot) { Remove-Item -LiteralPath $workRoot -Recurse -Force } if (-not $completed -and (Test-Path -LiteralPath $publishRoot)) { Remove-Item -LiteralPath $publishRoot -Recurse -Force } if ($loadedByRun) { try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 -Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/unload" -Method Post -ContentType "application/json" -Body "{}" *> $null Write-Output "PHASE=e10-model-state-restored" } catch { Write-Warning "LAB E10 could not restore the prior YOLOX-S state" } } }