feat(perception): integrate calibrated operator pipeline
Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,349 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)] [string]$JobRoot,
|
||||
[Parameter(Mandatory = $true)] [string]$RunnerPath,
|
||||
[Parameter(Mandatory = $true)] [string]$ProfilePath,
|
||||
[Parameter(Mandatory = $true)] [string]$DetectorProfilePath,
|
||||
[Parameter(Mandatory = $true)] [string]$SemanticProfilePath,
|
||||
[Parameter(Mandatory = $true)] [string]$ValidFovRoot,
|
||||
[ValidateRange(0, 1000000)] [int]$StartFrame = 1000,
|
||||
[ValidateRange(0, 1000000)] [int]$EndFrame = 1600,
|
||||
[ValidateRange(1, 1000)] [int]$FreeGiBFloor = 360,
|
||||
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
|
||||
[string]$ModelRoot = "D:\NDC_MISSIONCORE\runtime\models\yolox_s",
|
||||
[string]$TritonContainer = "mission-core-triton",
|
||||
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode {
|
||||
param([string]$Operation)
|
||||
if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" }
|
||||
}
|
||||
|
||||
function Resolve-DDirectory {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
||||
if (-not $item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") {
|
||||
throw "$Label must be a real D: directory"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Resolve-DFile {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
||||
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") {
|
||||
throw "$Label must be a regular D: file"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath {
|
||||
param([string]$Path)
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
function Get-DFreeBytes { return [int64](Get-PSDrive -Name D).Free }
|
||||
|
||||
function Assert-FreeSpace {
|
||||
param([string]$Phase, [int64]$RequiredAdditionalBytes = 0)
|
||||
$freeBytes = Get-DFreeBytes
|
||||
$floorBytes = [int64]$FreeGiBFloor * 1GB
|
||||
Write-Host (
|
||||
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3} REQUIRED_ADDITIONAL_GIB={4}" -f
|
||||
$Phase, $freeBytes, [math]::Round($freeBytes / 1GB, 3), $FreeGiBFloor,
|
||||
[math]::Round($RequiredAdditionalBytes / 1GB, 3)
|
||||
)
|
||||
if ($freeBytes -lt ($floorBytes + $RequiredAdditionalBytes)) {
|
||||
throw "D: does not have the guarded LAB E9 working-set reserve during $Phase"
|
||||
}
|
||||
return $freeBytes
|
||||
}
|
||||
|
||||
function Test-TritonModelReady {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -TimeoutSec 10 `
|
||||
-Uri "http://127.0.0.1:8000/v2/models/yolox_s/ready"
|
||||
return $response.StatusCode -eq 200
|
||||
}
|
||||
catch { return $false }
|
||||
}
|
||||
|
||||
$jobDirectory = Resolve-DDirectory $JobRoot "Job root"
|
||||
$runner = Resolve-DFile $RunnerPath "LAB E9 runner"
|
||||
$profile = Resolve-DFile $ProfilePath "LAB E9 profile"
|
||||
$detectorProfile = Resolve-DFile $DetectorProfilePath "LAB E9 detector profile"
|
||||
$semanticProfile = Resolve-DFile $SemanticProfilePath "LAB E9 semantic profile"
|
||||
$validFov = Resolve-DDirectory $ValidFovRoot "Valid-FOV root"
|
||||
$runtime = Resolve-DDirectory $RuntimeRoot "Runtime root"
|
||||
$model = Resolve-DDirectory $ModelRoot "YOLOX model root"
|
||||
$runnerRoot = Split-Path $runner -Parent
|
||||
foreach ($path in @($profile, $detectorProfile, $semanticProfile)) {
|
||||
if ((Split-Path $path -Parent) -ne $runnerRoot) {
|
||||
throw "LAB E9 runner and profiles must share one read-only mount"
|
||||
}
|
||||
}
|
||||
foreach ($dependency in @(
|
||||
"run_e8_realtime_tracking.py",
|
||||
"run_e5_instance_tracking.py",
|
||||
"run_e4_full_session_segmentation.py",
|
||||
"run_recorded_perception_epoch.py",
|
||||
"run_e3_rectified_segmentation.py",
|
||||
"run_evaluation_prelabels.py"
|
||||
)) {
|
||||
$null = Resolve-DFile (Join-Path $runnerRoot $dependency) "LAB E9 runner dependency"
|
||||
}
|
||||
|
||||
$job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json
|
||||
if ($job.schema_version -ne "missioncore.compute-job/v1" -or $job.job_id -ne (Split-Path $jobDirectory -Leaf)) {
|
||||
throw "Compute job manifest is incompatible"
|
||||
}
|
||||
$sourceId = [string]$job.input.source_id
|
||||
$epoch = [int]$job.input.codec_epoch
|
||||
$fullFrameCount = [int]$job.input.segment_count
|
||||
$clipFrameCount = $EndFrame - $StartFrame + 1
|
||||
$timelineStart = [double]$job.input.timeline.start_seconds
|
||||
if (
|
||||
$sourceId -ne "sensor.camera.right" -or $StartFrame -lt 0 -or
|
||||
$EndFrame -lt $StartFrame -or $EndFrame -ge $fullFrameCount -or $clipFrameCount -lt 2
|
||||
) { throw "LAB E9 clip escapes the camera job" }
|
||||
Write-Output (
|
||||
"PHASE=job-manifest-validated JOB={0} CLIP={1}-{2} FRAMES={3}" -f
|
||||
$job.job_id, $StartFrame, $EndFrame, $clipFrameCount
|
||||
)
|
||||
|
||||
$epochRoot = Resolve-DDirectory (
|
||||
Join-Path $jobDirectory ("input\camera\{0}\epoch-{1}" -f $sourceId, $epoch)
|
||||
) "Camera epoch"
|
||||
$initPath = Resolve-DFile (Join-Path $epochRoot "init.mp4") "Camera init"
|
||||
$segmentsRoot = Resolve-DDirectory (Join-Path $epochRoot "segments") "Camera segments"
|
||||
$derivedRoot = Join-Path $runtime "derived"
|
||||
$tmpRoot = Join-Path $runtime "tmp"
|
||||
$cacheRoot = Resolve-DDirectory (Join-Path $runtime "cache\perception-e3-models-v1") "EoMT cache"
|
||||
$e3Environment = Resolve-DDirectory (Join-Path $derivedRoot "perception-e3-opencv413092-v1") "E3 environment"
|
||||
$torchEnvironment = Resolve-DDirectory (Join-Path $derivedRoot "perception-p0-env-v1") "Torch environment"
|
||||
$transformersEnvironment = Resolve-DDirectory (Join-Path $derivedRoot "perception-p0-transformers4576-v1") "Transformers environment"
|
||||
$null = New-Item -ItemType Directory -Path $derivedRoot -Force
|
||||
$null = New-Item -ItemType Directory -Path $tmpRoot -Force
|
||||
|
||||
$pixelWorkingSet = [int64]$clipFrameCount * 800 * 600 * 3
|
||||
$partialStreamReserve = [int64][math]::Ceiling(
|
||||
([int64]$job.input.byte_length * ($EndFrame + 1) / $fullFrameCount) * 1.2
|
||||
)
|
||||
$workingSetReserve = [int64][math]::Ceiling($pixelWorkingSet * 1.4) + $partialStreamReserve + 3GB
|
||||
$freeBytesBefore = Assert-FreeSpace "preflight" $workingSetReserve
|
||||
|
||||
& docker image inspect $ContainerImage *> $null
|
||||
Assert-LastExitCode "Existing container image inspection"
|
||||
$tritonState = docker inspect --format "{{.State.Running}}" $TritonContainer
|
||||
Assert-LastExitCode "Triton container inspection"
|
||||
if ($tritonState.Trim().ToLowerInvariant() -ne "true") {
|
||||
throw "LAB E9 requires the existing Triton container"
|
||||
}
|
||||
|
||||
$runnerName = Split-Path $runner -Leaf
|
||||
$profileName = Split-Path $profile -Leaf
|
||||
$detectorProfileName = Split-Path $detectorProfile -Leaf
|
||||
$semanticProfileName = Split-Path $semanticProfile -Leaf
|
||||
$orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$commonMounts = @(
|
||||
"-e", "PYTHONPATH=/runner:/opt/transformers:/opt/env",
|
||||
"-e", "TORCH_HOME=/cache/torch",
|
||||
"-e", "HF_HOME=/cache/huggingface",
|
||||
"-e", "HF_HUB_OFFLINE=1",
|
||||
"-e", "TRANSFORMERS_OFFLINE=1",
|
||||
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
||||
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
||||
"-v", ((Convert-ToDockerPath $model) + ":/model:ro"),
|
||||
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"),
|
||||
"-v", ((Convert-ToDockerPath $e3Environment) + ":/environment:ro"),
|
||||
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro")
|
||||
)
|
||||
$commonRunnerArgs = @(
|
||||
"--job", "/job",
|
||||
"--profile", ("/runner/{0}" -f $profileName),
|
||||
"--detector-profile", ("/runner/{0}" -f $detectorProfileName),
|
||||
"--semantic-profile", ("/runner/{0}" -f $semanticProfileName),
|
||||
"--valid-fov-root", "/valid-fov",
|
||||
"--model-root", "/model",
|
||||
"--cache", "/cache",
|
||||
"--environment", "/environment"
|
||||
)
|
||||
$preflightArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", "none", "--read-only",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "512", "--shm-size", "4g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g"
|
||||
) + $commonMounts + @(
|
||||
"--entrypoint", "python3", $ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "preflight"
|
||||
) + $commonRunnerArgs
|
||||
Write-Output "PHASE=e9-preflight-start"
|
||||
& docker @preflightArgs
|
||||
Assert-LastExitCode "LAB E9 preflight"
|
||||
Write-Output "PHASE=e9-preflight-complete"
|
||||
|
||||
$modelWasReady = Test-TritonModelReady
|
||||
$loadedByRun = $false
|
||||
if (-not $modelWasReady) {
|
||||
Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 `
|
||||
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/load" `
|
||||
-Method Post -ContentType "application/json" -Body "{}" *> $null
|
||||
$loadedByRun = $true
|
||||
$deadline = [DateTime]::UtcNow.AddSeconds(60)
|
||||
while (-not (Test-TritonModelReady)) {
|
||||
if ([DateTime]::UtcNow -ge $deadline) { throw "YOLOX-S did not become ready" }
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
}
|
||||
Write-Output ("PHASE=e9-model-ready LOADED_BY_RUN={0}" -f $loadedByRun)
|
||||
|
||||
$runToken = [Guid]::NewGuid().ToString("N")
|
||||
$workRoot = Join-Path $tmpRoot ("{0}-e9-{1}" -f $job.job_id, $runToken)
|
||||
$framesRoot = Join-Path $workRoot "frames"
|
||||
$streamPath = Join-Path $workRoot "camera.mp4"
|
||||
$ptsPath = Join-Path $workRoot "pts.json"
|
||||
$timelinePath = Join-Path $workRoot "timeline.jsonl"
|
||||
$publishRoot = Join-Path $derivedRoot (".{0}-e9-{1}.publish" -f $job.job_id, $runToken)
|
||||
$stagingRoot = Join-Path $publishRoot "output"
|
||||
$null = New-Item -ItemType Directory -Path $framesRoot
|
||||
$null = New-Item -ItemType Directory -Path $publishRoot
|
||||
$completed = $false
|
||||
$totalWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
|
||||
try {
|
||||
$stream = [IO.File]::Open($streamPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
||||
try {
|
||||
$input = [IO.File]::OpenRead($initPath)
|
||||
try { $input.CopyTo($stream) } finally { $input.Dispose() }
|
||||
for ($sequence = 1; $sequence -le ($EndFrame + 1); $sequence++) {
|
||||
$path = Resolve-DFile (Join-Path $segmentsRoot ("{0}.m4s" -f $sequence)) "Camera segment"
|
||||
$input = [IO.File]::OpenRead($path)
|
||||
try { $input.CopyTo($stream) } finally { $input.Dispose() }
|
||||
}
|
||||
$stream.Flush($true)
|
||||
}
|
||||
finally { $stream.Dispose() }
|
||||
$null = Assert-FreeSpace "post-stream-reconstruction"
|
||||
|
||||
$extractWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
$selectFilter = "select='between(n,{0},{1})'" -f $StartFrame, $EndFrame
|
||||
& ffmpeg -hide_banner -loglevel fatal -i $streamPath -map 0:v:0 `
|
||||
-vf $selectFilter -fps_mode passthrough (Join-Path $framesRoot "frame-%06d.png")
|
||||
Assert-LastExitCode "LAB E9 camera clip extraction"
|
||||
& ffprobe -v error -select_streams v:0 -show_entries frame=best_effort_timestamp_time `
|
||||
-of json $streamPath | Set-Content -LiteralPath $ptsPath -Encoding utf8
|
||||
Assert-LastExitCode "LAB E9 camera timestamp probe"
|
||||
|
||||
$decodedFrames = @(Get-ChildItem -LiteralPath $framesRoot -File -Filter "frame-*.png")
|
||||
$pts = @((Get-Content -LiteralPath $ptsPath -Raw | ConvertFrom-Json).frames)
|
||||
if ($decodedFrames.Count -ne $clipFrameCount -or $pts.Count -ne ($EndFrame + 1)) {
|
||||
throw "Decoded LAB E9 frame/timestamp count changed"
|
||||
}
|
||||
$firstEpochSeconds = [double]::Parse(
|
||||
([string]$pts[0].best_effort_timestamp_time).Trim(),
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
)
|
||||
$previousSessionSeconds = -1.0
|
||||
$timelineWriter = [IO.StreamWriter]::new(
|
||||
$timelinePath, $false, [Text.UTF8Encoding]::new($false)
|
||||
)
|
||||
try {
|
||||
for ($localIndex = 0; $localIndex -lt $clipFrameCount; $localIndex++) {
|
||||
$sourceIndex = $StartFrame + $localIndex
|
||||
$epochSeconds = [double]::Parse(
|
||||
([string]$pts[$sourceIndex].best_effort_timestamp_time).Trim(),
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
) - $firstEpochSeconds
|
||||
$sessionSeconds = $timelineStart + $epochSeconds
|
||||
if ($sessionSeconds -le $previousSessionSeconds) { throw "LAB E9 timeline is not monotonic" }
|
||||
$row = [ordered]@{
|
||||
frame_index = $localIndex
|
||||
sequence = $localIndex + 1
|
||||
source_frame_index = $sourceIndex
|
||||
source_sequence = $sourceIndex + 1
|
||||
epoch_seconds = $epochSeconds
|
||||
session_seconds = $sessionSeconds
|
||||
}
|
||||
$timelineWriter.WriteLine(($row | ConvertTo-Json -Compress))
|
||||
$previousSessionSeconds = $sessionSeconds
|
||||
}
|
||||
$timelineWriter.Flush()
|
||||
}
|
||||
finally { $timelineWriter.Dispose() }
|
||||
$extractWatch.Stop()
|
||||
$freeBytesPostExtract = Assert-FreeSpace "post-frame-extraction"
|
||||
Write-Output ("PHASE=e9-frame-extraction-complete FRAMES={0}" -f $clipFrameCount)
|
||||
|
||||
$runArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", ("container:{0}" -f $TritonContainer),
|
||||
"--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "512", "--shm-size", "4g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g"
|
||||
) + $commonMounts + @(
|
||||
"-v", ((Convert-ToDockerPath $framesRoot) + ":/frames:ro"),
|
||||
"-v", ((Convert-ToDockerPath $workRoot) + ":/work:ro"),
|
||||
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
|
||||
"--entrypoint", "python3", $ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "run"
|
||||
) + $commonRunnerArgs + @(
|
||||
"--frames", "/frames",
|
||||
"--timeline", "/work/timeline.jsonl",
|
||||
"--output", "/publish/output",
|
||||
"--triton-url", "http://127.0.0.1:8000",
|
||||
"--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB),
|
||||
"--orchestrator-sha256", $orchestratorSha256,
|
||||
"--container-image", $ContainerImage,
|
||||
"--telemetry-interval-seconds", "1"
|
||||
)
|
||||
Write-Output ("PHASE=e9-multirate-replay-start FRAMES={0}" -f $clipFrameCount)
|
||||
& docker @runArgs
|
||||
Assert-LastExitCode "LAB E9 multirate replay"
|
||||
$freeBytesPostReplay = Assert-FreeSpace "post-replay"
|
||||
|
||||
$result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$result.schema_version -ne "missioncore.e9-multirate-perception-result/v1" -or
|
||||
$result.result_id -notmatch "^e9-multirate-perception-[a-f0-9]{64}$" -or
|
||||
$result.acceptance_state -notin @("accepted", "rejected") -or
|
||||
$result.ground_truth -ne $false
|
||||
) { throw "LAB E9 result manifest is incompatible" }
|
||||
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
||||
if (Test-Path -LiteralPath $finalRoot) { throw "Immutable LAB E9 result already exists" }
|
||||
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
|
||||
Remove-Item -LiteralPath $publishRoot -Force
|
||||
$completed = $true
|
||||
$totalWatch.Stop()
|
||||
$freeBytesFinal = Assert-FreeSpace "post-publication"
|
||||
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
||||
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
||||
Write-Output ("ACCEPTANCE_STATE={0}" -f $result.acceptance_state)
|
||||
Write-Output ("PREPARATION_SECONDS={0}" -f $extractWatch.Elapsed.TotalSeconds)
|
||||
Write-Output ("ORCHESTRATOR_WALL_SECONDS={0}" -f $totalWatch.Elapsed.TotalSeconds)
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_EXTRACT={0}" -f $freeBytesPostExtract)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_REPLAY={0}" -f $freeBytesPostReplay)
|
||||
Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeBytesFinal)
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $workRoot) { Remove-Item -LiteralPath $workRoot -Recurse -Force }
|
||||
if (-not $completed -and (Test-Path -LiteralPath $publishRoot)) {
|
||||
Remove-Item -LiteralPath $publishRoot -Recurse -Force
|
||||
}
|
||||
if ($loadedByRun) {
|
||||
try {
|
||||
Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 `
|
||||
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/unload" `
|
||||
-Method Post -ContentType "application/json" -Body "{}" *> $null
|
||||
Write-Output "PHASE=e9-model-state-restored"
|
||||
}
|
||||
catch { Write-Warning "LAB E9 could not restore the prior unloaded YOLOX-S state" }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user