[CmdletBinding()] param( [Parameter(Mandatory = $true)] [string]$JobRoot, [Parameter(Mandatory = $true)] [string]$RunnerPath, [Parameter(Mandatory = $true)] [string]$LiveProfilePath, [Parameter(Mandatory = $true)] [string]$E14ProfilePath, [Parameter(Mandatory = $true)] [string]$DetectorProfilePath, [Parameter(Mandatory = $true)] [string]$SemanticProfilePath, [Parameter(Mandatory = $true)] [string]$ValidFovRoot, [Parameter(Mandatory = $true)] [string]$ProjectionPackRoot, [Parameter(Mandatory = $true)] [string]$PackageRoot, [switch]$PreflightOnly, [switch]$PersistentService, [switch]$TokenStdin, [ValidateRange(5, 3600)] [int]$MaximumDurationSeconds = 20, [ValidateRange(1, 1000)] [int]$FreeGiBFloor = 360, [ValidateRange(1024, 65535)] [int]$SourcePort = 18012, [string]$SourceHost = "host.docker.internal", [string]$SourcePath = "/api/v1/device-plugins/nodedc.device.xgrids-lixelkity-k1/live-perception-shadow", [string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime", [string]$ModelRoot = "D:\NDC_MISSIONCORE\runtime\models\yolox_s", [string]$MediaRuntimeRoot = "D:\NDC_MISSIONCORE\runtime\derived\perception-e15-media-pyav180-lz445-v1", [string]$TritonContainer = "mission-core-triton", [string]$PersistentContainer = "mission-core-perception-worker", [string]$PersistentOutputRoot = "D:\NDC_MISSIONCORE\runtime\derived\.perception-persistent-publish", [ValidateRange(1024, 65535)] [int]$PersistentPort = 18020, [string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794" ) $ErrorActionPreference = "Stop" $ProgressPreference = "SilentlyContinue" function Assert-LastExitCode([string]$Operation) { if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" } } function Resolve-DDirectory([string]$Path, [string]$Label) { $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force $root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") if (-not $item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") { throw "$Label must be a real D: directory" } return $item.FullName } function Resolve-DFile([string]$Path, [string]$Label) { $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force $root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") { throw "$Label must be a regular D: file" } return $item.FullName } function Convert-ToDockerPath([string]$Path) { return $Path.Replace("\", "/") } function Get-DFreeBytes { return [int64](Get-PSDrive -Name D).Free } function Assert-FreeSpace([string]$Phase, [int64]$RequiredAdditionalBytes = 0) { $free = Get-DFreeBytes $floor = [int64]$FreeGiBFloor * 1GB Write-Host ( "DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3} REQUIRED_ADDITIONAL_GIB={4}" -f $Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor, [math]::Round($RequiredAdditionalBytes / 1GB, 3) ) if ($free -lt ($floor + $RequiredAdditionalBytes)) { throw "D: lacks the guarded LAB E15 reserve during $Phase" } return $free } function Test-TritonModelReady { try { $response = Invoke-WebRequest -UseBasicParsing -TimeoutSec 10 ` -Uri "http://127.0.0.1:8000/v2/models/yolox_s/ready" return $response.StatusCode -eq 200 } catch { return $false } } $jobDirectory = Resolve-DDirectory $JobRoot "Job root" $runner = Resolve-DFile $RunnerPath "LAB E15 runner" $liveProfile = Resolve-DFile $LiveProfilePath "LAB E15 live profile" $e14Profile = Resolve-DFile $E14ProfilePath "Accepted E14 profile" $detectorProfile = Resolve-DFile $DetectorProfilePath "Detector profile" $semanticProfile = Resolve-DFile $SemanticProfilePath "Semantic profile" $validFov = Resolve-DDirectory $ValidFovRoot "Valid-FOV root" $projectionPack = Resolve-DDirectory $ProjectionPackRoot "E15 projection pack" $package = Resolve-DDirectory $PackageRoot "Mission Core package root" $runtime = Resolve-DDirectory $RuntimeRoot "Runtime root" $model = Resolve-DDirectory $ModelRoot "YOLOX model root" $mediaRuntime = Resolve-DDirectory $MediaRuntimeRoot "E15 media runtime" $runnerRoot = Split-Path $runner -Parent foreach ($path in @($liveProfile, $e14Profile, $detectorProfile, $semanticProfile)) { if ((Split-Path $path -Parent) -ne $runnerRoot) { throw "Runner and profiles must share one immutable mount" } } foreach ($dependency in @( "e10_fusion_runtime.py", "e15_shadow_runtime.py", "run_e10_integrated_perception.py", "run_e12_shadow_transport_probe.py", "run_e9_multirate_perception.py", "run_e8_realtime_tracking.py", "run_e5_instance_tracking.py", "run_e4_full_session_segmentation.py", "run_recorded_perception_epoch.py", "run_e3_rectified_segmentation.py", "run_evaluation_prelabels.py" )) { $null = Resolve-DFile (Join-Path $runnerRoot $dependency) "LAB E15 dependency" } if (-not (Test-Path -LiteralPath (Join-Path $package "k1link\compute\live_perception.py") -PathType Leaf)) { throw "Mission Core package mount lacks live perception synchronization" } $job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json if ( $job.schema_version -ne "missioncore.compute-job/v1" -or $job.job_id -ne (Split-Path $jobDirectory -Leaf) -or $job.input.source_id -ne "sensor.camera.right" ) { throw "Compute bootstrap job manifest is incompatible" } $live = Get-Content -LiteralPath $liveProfile -Raw | ConvertFrom-Json if ( $live.schema_version -ne "missioncore.e15-shadow-inference-profile/v1" -or $live.mode -ne "replay-shadow-gate" -or [bool]$live.authority.commands_enabled -or [bool]$live.authority.navigation_or_safety_accepted -or $live.transport.pyav_version -ne "18.0.0" ) { throw "LAB E15 replay-shadow authority contract changed" } $projection = Get-Content -LiteralPath (Join-Path $projectionPack "manifest.json") -Raw | ConvertFrom-Json if ( $projection.schema_version -ne "missioncore.e15-live-projection-pack/v1" -or $projection.identity.source_id -ne "sensor.camera.right" -or $projection.identity.calibration_slot -ne "camera_1" -or $projection.identity.calibration_sha256 -ne $live.source.calibration_sha256 ) { throw "LAB E15 projection pack binding changed" } $mediaManifest = Get-Content -LiteralPath (Join-Path $mediaRuntime "manifest.json") -Raw | ConvertFrom-Json if ( $mediaManifest.schema_version -ne "missioncore.e15-media-runtime/v1" -or $mediaManifest.container_image -ne $ContainerImage -or $mediaManifest.packages.av -ne "18.0.0" -or $mediaManifest.packages.lz4 -ne "4.4.5" ) { throw "LAB E15 media runtime identity changed" } $derivedRoot = Resolve-DDirectory (Join-Path $runtime "derived") "Runtime derived root" $cacheRoot = Resolve-DDirectory (Join-Path $runtime "cache\perception-e3-models-v1") "EoMT cache" $e3Environment = Resolve-DDirectory (Join-Path $derivedRoot "perception-e3-opencv413092-v1") "E3 environment" $torchEnvironment = Resolve-DDirectory (Join-Path $derivedRoot "perception-p0-env-v1") "Torch environment" $transformersEnvironment = Resolve-DDirectory (Join-Path $derivedRoot "perception-p0-transformers4576-v1") "Transformers environment" $freeBytesBefore = Assert-FreeSpace "preflight" 512MB & docker image inspect $ContainerImage *> $null Assert-LastExitCode "Pinned container image inspection" $tritonState = docker inspect --format "{{.State.Running}}" $TritonContainer Assert-LastExitCode "Triton container inspection" if ($tritonState.Trim().ToLowerInvariant() -ne "true") { throw "LAB E15 requires the existing Triton container" } $runnerName = Split-Path $runner -Leaf $liveProfileName = Split-Path $liveProfile -Leaf $e14ProfileName = Split-Path $e14Profile -Leaf $detectorProfileName = Split-Path $detectorProfile -Leaf $semanticProfileName = Split-Path $semanticProfile -Leaf $projectionMount = "/" + (Split-Path $projectionPack -Leaf) $packageMount = "/" + (Split-Path $package -Leaf) $orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant() $commonMounts = @( "-e", ("PYTHONPATH=/runner:{0}:/opt/media:/opt/transformers:/opt/env" -f $packageMount), "-e", "PYTHONDONTWRITEBYTECODE=1", "-e", "TORCH_HOME=/cache/torch", "-e", "HF_HOME=/cache/huggingface", "-e", "HF_HUB_OFFLINE=1", "-e", "TRANSFORMERS_OFFLINE=1", "-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"), "-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"), "-v", ((Convert-ToDockerPath $projectionPack) + (":{0}:ro" -f $projectionMount)), "-v", ((Convert-ToDockerPath $model) + ":/model:ro"), "-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"), "-v", ((Convert-ToDockerPath $e3Environment) + ":/environment:ro"), "-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"), "-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"), "-v", ((Convert-ToDockerPath $mediaRuntime) + ":/opt/media:ro"), "-v", ((Convert-ToDockerPath $package) + (":{0}:ro" -f $packageMount)), "-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro") ) $commonRunnerArgs = @( "--job", "/job", "--live-profile", ("/runner/{0}" -f $liveProfileName), "--e14-profile", ("/runner/{0}" -f $e14ProfileName), "--detector-profile", ("/runner/{0}" -f $detectorProfileName), "--semantic-profile", ("/runner/{0}" -f $semanticProfileName), "--valid-fov-root", "/valid-fov", "--projection-pack", $projectionMount, "--model-root", "/model", "--cache", "/cache", "--environment", "/environment", "--worker-package", $packageMount ) if ($PersistentService) { if ($PreflightOnly -or $TokenStdin) { throw "Persistent service cannot be combined with one-shot switches" } $persistentParent = Resolve-DDirectory (Split-Path $PersistentOutputRoot -Parent) "Persistent output parent" if (-not (Test-Path -LiteralPath $PersistentOutputRoot)) { $null = New-Item -ItemType Directory -Path $PersistentOutputRoot } $persistentOutput = Resolve-DDirectory $PersistentOutputRoot "Persistent output root" if ((Split-Path $persistentOutput -Parent) -ne $persistentParent) { throw "Persistent output root must be a direct child of its guarded D: parent" } $runnerSha256 = (Get-FileHash -LiteralPath $runner -Algorithm SHA256).Hash.ToLowerInvariant() $existingContainerId = docker ps -a --filter ("name=^{0}$" -f $PersistentContainer) --format "{{.ID}}" Assert-LastExitCode "Persistent worker container lookup" if ($existingContainerId) { $labels = (docker inspect --format "{{json .Config.Labels}}" $PersistentContainer) | ConvertFrom-Json Assert-LastExitCode "Persistent worker label inspection" if ( $labels.'missioncore.role' -ne "perception-persistent-worker" -or $labels.'missioncore.runner.sha256' -ne $runnerSha256 ) { throw "Existing persistent worker has a different immutable identity" } $running = docker inspect --format "{{.State.Running}}" $PersistentContainer Assert-LastExitCode "Persistent worker state inspection" if ($running.Trim().ToLowerInvariant() -ne "true") { throw "Matching persistent worker exists but is not running" } $healthProbe = ( "import json,urllib.request;d=json.load(urllib.request.urlopen(" + "'http://127.0.0.1:{0}/health',timeout=5));" -f $PersistentPort ) + "raise SystemExit(0 if d.get('ok') and d.get('models_loaded') else 2)" $previousErrorPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" & docker exec $PersistentContainer python3 -c $healthProbe 2>$null $healthExitCode = $LASTEXITCODE $ErrorActionPreference = $previousErrorPreference if ($healthExitCode -ne 0) { throw "Existing persistent worker health probe failed" } Write-Output "STATE=persistent-worker-reused" Write-Output ("CONTAINER={0}" -f $PersistentContainer) return } if (-not (Test-TritonModelReady)) { Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 ` -Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/load" ` -Method Post -ContentType "application/json" -Body "{}" *> $null $deadline = [DateTime]::UtcNow.AddSeconds(60) while (-not (Test-TritonModelReady)) { if ([DateTime]::UtcNow -ge $deadline) { throw "YOLOX-S did not become ready" } Start-Sleep -Milliseconds 500 } } $serveArgs = @( "run", "--detach", "--name", $PersistentContainer, "--label", "missioncore.role=perception-persistent-worker", "--label", ("missioncore.runner.sha256={0}" -f $runnerSha256), "--gpus", "all", "--network", ("container:{0}" -f $TritonContainer), "--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL", "--pids-limit", "512", "--shm-size", "4g", "--tmpfs", "/tmp:rw,noexec,nosuid,size=2g" ) + $commonMounts + @( "-v", ((Convert-ToDockerPath $persistentOutput) + ":/publish:rw"), "--entrypoint", "python3", $ContainerImage, ("/runner/{0}" -f $runnerName), "serve" ) + $commonRunnerArgs + @( "--host", $SourceHost, "--port", [string]$SourcePort, "--path", $SourcePath, "--triton-url", "http://127.0.0.1:8000", "--output-root", "/publish", "--listen-host", "127.0.0.1", "--listen-port", [string]$PersistentPort, "--max-duration-seconds", [string]$MaximumDurationSeconds, "--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB), "--orchestrator-sha256", $orchestratorSha256, "--container-image", $ContainerImage ) Write-Output "PHASE=persistent-worker-model-load-start" & docker @serveArgs Assert-LastExitCode "Persistent worker container start" $healthProbe = ( "import json,urllib.request;d=json.load(urllib.request.urlopen(" + "'http://127.0.0.1:{0}/health',timeout=5));" -f $PersistentPort ) + "print(json.dumps(d,sort_keys=True));raise SystemExit(0 if d.get('ok') and d.get('models_loaded') else 2)" $deadline = [DateTime]::UtcNow.AddSeconds(240) do { Start-Sleep -Seconds 2 $running = docker inspect --format "{{.State.Running}}" $PersistentContainer if ($LASTEXITCODE -ne 0 -or $running.Trim().ToLowerInvariant() -ne "true") { & docker logs --tail 80 $PersistentContainer throw "Persistent worker exited during model load" } $previousErrorPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" $health = & docker exec $PersistentContainer python3 -c $healthProbe 2>$null $healthExitCode = $LASTEXITCODE $ErrorActionPreference = $previousErrorPreference $healthy = $healthExitCode -eq 0 } while (-not $healthy -and [DateTime]::UtcNow -lt $deadline) if (-not $healthy) { & docker logs --tail 80 $PersistentContainer throw "Persistent worker model load did not become ready" } Write-Output "STATE=persistent-worker-ready" Write-Output ("CONTAINER={0}" -f $PersistentContainer) Write-Output ("HEALTH={0}" -f $health) Write-Output ("DISK_FREE_BYTES={0}" -f (Get-DFreeBytes)) return } $preflightArgs = @( "run", "--rm", "--gpus", "all", "--network", "none", "--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL", "--pids-limit", "512", "--shm-size", "4g", "--tmpfs", "/tmp:rw,noexec,nosuid,size=2g" ) + $commonMounts + @( "--entrypoint", "python3", $ContainerImage, ("/runner/{0}" -f $runnerName), "preflight" ) + $commonRunnerArgs Write-Output "PHASE=e15-preflight-start" & docker @preflightArgs Assert-LastExitCode "LAB E15 preflight" Write-Output "PHASE=e15-preflight-complete" if ($PreflightOnly) { Write-Output "STATE=preflight-ready" Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore) Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f (Get-DFreeBytes)) return } if (-not $TokenStdin) { throw "LAB E15 requires the shadow token through stdin" } $shadowToken = [Console]::In.ReadLine() if (-not $shadowToken -or $shadowToken.Length -lt 40 -or $shadowToken.Length -gt 512) { throw "LAB E15 shadow token is missing or malformed" } $modelWasReady = Test-TritonModelReady $loadedByRun = $false if (-not $modelWasReady) { Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 ` -Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/load" ` -Method Post -ContentType "application/json" -Body "{}" *> $null $loadedByRun = $true $deadline = [DateTime]::UtcNow.AddSeconds(60) while (-not (Test-TritonModelReady)) { if ([DateTime]::UtcNow -ge $deadline) { throw "YOLOX-S did not become ready" } Start-Sleep -Milliseconds 500 } } Write-Output ("PHASE=e15-model-ready LOADED_BY_RUN={0}" -f $loadedByRun) $token = [Guid]::NewGuid().ToString("N") $publishRoot = Join-Path $derivedRoot (".{0}-e15-{1}.publish" -f $job.job_id, $token) $stagingRoot = Join-Path $publishRoot "output" $null = New-Item -ItemType Directory -Path $publishRoot $completed = $false $totalWatch = [Diagnostics.Stopwatch]::StartNew() try { $runArgs = @( "run", "--rm", "--interactive", "--gpus", "all", "--network", ("container:{0}" -f $TritonContainer), "--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL", "--pids-limit", "512", "--shm-size", "4g", "--tmpfs", "/tmp:rw,noexec,nosuid,size=2g" ) + $commonMounts + @( "-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"), "--entrypoint", "python3", $ContainerImage, ("/runner/{0}" -f $runnerName), "run" ) + $commonRunnerArgs + @( "--host", $SourceHost, "--port", [string]$SourcePort, "--path", $SourcePath, "--token-stdin", "--triton-url", "http://127.0.0.1:8000", "--output", "/publish/output", "--max-duration-seconds", [string]$MaximumDurationSeconds, "--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB), "--orchestrator-sha256", $orchestratorSha256, "--container-image", $ContainerImage ) Write-Output ("PHASE=e15-shadow-inference-start MAX_DURATION_SECONDS={0}" -f $MaximumDurationSeconds) $shadowToken | & docker @runArgs $runnerExitCode = $LASTEXITCODE $shadowToken = $null if ($runnerExitCode -ne 0 -and $runnerExitCode -ne 2) { throw "LAB E15 shadow inference failed with exit code $runnerExitCode" } $null = Assert-FreeSpace "post-shadow-inference" $resultPath = Join-Path $stagingRoot "result.json" if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) { throw "LAB E15 result manifest is missing" } $result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json if ( $result.schema_version -ne "missioncore.e15-shadow-inference-result/v1" -or $result.result_id -notmatch "^e15-shadow-inference-[a-f0-9]{64}$" -or $result.publication_scope -ne "live-shadow-diagnostic-only" ) { throw "LAB E15 result manifest is incompatible" } $finalRoot = Join-Path $derivedRoot ([string]$result.result_id) if (Test-Path -LiteralPath $finalRoot) { throw "Immutable LAB E15 result already exists" } Move-Item -LiteralPath $stagingRoot -Destination $finalRoot Remove-Item -LiteralPath $publishRoot -Force $completed = $true $totalWatch.Stop() $freeFinal = Assert-FreeSpace "post-publication" Write-Output ("RESULT_ROOT={0}" -f $finalRoot) Write-Output ("RESULT_ID={0}" -f $result.result_id) Write-Output ("ACCEPTANCE_STATE={0}" -f $result.acceptance_state) Write-Output ("RUNNER_EXIT_CODE={0}" -f $runnerExitCode) Write-Output ("ORCHESTRATOR_WALL_SECONDS={0}" -f $totalWatch.Elapsed.TotalSeconds) Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore) Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeFinal) } finally { $shadowToken = $null if (-not $completed -and (Test-Path -LiteralPath $publishRoot)) { Remove-Item -LiteralPath $publishRoot -Recurse -Force } if ($loadedByRun) { try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 ` -Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/unload" ` -Method Post -ContentType "application/json" -Body "{}" *> $null Write-Output "PHASE=e15-model-state-restored" } catch { Write-Warning "LAB E15 could not restore the prior YOLOX-S state" } } }