feat(perception): integrate calibrated operator pipeline
Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,153 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateRange(1, 1000)] [int]$FreeGiBFloor = 360,
|
||||
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
|
||||
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
|
||||
[string]$RuntimeName = "perception-e15-media-pyav180-lz445-v1"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode([string]$Operation) {
|
||||
if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" }
|
||||
}
|
||||
|
||||
function Resolve-DDirectory([string]$Path, [string]$Label) {
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
||||
if (-not $item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") {
|
||||
throw "$Label must be a real D: directory"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath([string]$Path) { return $Path.Replace("\", "/") }
|
||||
|
||||
function Assert-FreeSpace([string]$Phase) {
|
||||
$free = [int64](Get-PSDrive -Name D).Free
|
||||
$floor = [int64]$FreeGiBFloor * 1GB
|
||||
Write-Host (
|
||||
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3}" -f
|
||||
$Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor
|
||||
)
|
||||
if ($free -lt ($floor + 2GB)) {
|
||||
throw "D: lacks the guarded LAB E15 media-runtime reserve during $Phase"
|
||||
}
|
||||
return $free
|
||||
}
|
||||
|
||||
function Get-PayloadDigest([string]$Root) {
|
||||
$resolved = Resolve-DDirectory $Root "E15 media runtime"
|
||||
$lines = @(
|
||||
Get-ChildItem -LiteralPath $resolved -Recurse -File -Force |
|
||||
Where-Object { $_.Name -ne "manifest.json" } |
|
||||
Sort-Object FullName |
|
||||
ForEach-Object {
|
||||
$relative = $_.FullName.Substring($resolved.Length).TrimStart("\").Replace("\", "/")
|
||||
$hash = (Get-FileHash -LiteralPath $_.FullName -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
"{0}`t{1}`t{2}" -f $relative, $_.Length, $hash
|
||||
}
|
||||
)
|
||||
if ($lines.Count -lt 4) { throw "E15 media runtime payload is incomplete" }
|
||||
$bytes = [Text.Encoding]::UTF8.GetBytes(($lines -join "`n") + "`n")
|
||||
$hasher = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
return ([BitConverter]::ToString($hasher.ComputeHash($bytes))).Replace("-", "").ToLowerInvariant()
|
||||
}
|
||||
finally { $hasher.Dispose() }
|
||||
}
|
||||
|
||||
function Assert-Runtime([string]$Path) {
|
||||
$root = Resolve-DDirectory $Path "E15 media runtime"
|
||||
$manifestPath = Join-Path $root "manifest.json"
|
||||
if (-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) {
|
||||
throw "E15 media runtime manifest is missing"
|
||||
}
|
||||
$manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$manifest.schema_version -ne "missioncore.e15-media-runtime/v1" -or
|
||||
$manifest.runtime_name -ne $RuntimeName -or
|
||||
$manifest.container_image -ne $ContainerImage -or
|
||||
$manifest.packages.av -ne "18.0.0" -or
|
||||
$manifest.packages.lz4 -ne "4.4.5" -or
|
||||
$manifest.payload_sha256 -ne (Get-PayloadDigest $root)
|
||||
) { throw "E15 media runtime identity changed" }
|
||||
|
||||
$dockerRoot = Convert-ToDockerPath $root
|
||||
$verifyCode = "import av,lz4.version;print(av.__version__);print(lz4.version.version)"
|
||||
$verifyOutput = @(& docker run --rm --network none --read-only `
|
||||
--security-opt "no-new-privileges:true" --cap-drop ALL --pids-limit 64 `
|
||||
--tmpfs "/tmp:rw,noexec,nosuid,size=64m" `
|
||||
-e "PYTHONPATH=/opt/media" -e "PYTHONDONTWRITEBYTECODE=1" `
|
||||
-v ("{0}:/opt/media:ro" -f $dockerRoot) `
|
||||
--entrypoint python3 $ContainerImage -c $verifyCode)
|
||||
Assert-LastExitCode "E15 media runtime verification"
|
||||
if ($verifyOutput.Count -ne 2 -or $verifyOutput[0] -ne "18.0.0" -or $verifyOutput[1] -ne "4.4.5") {
|
||||
throw "E15 media runtime package versions changed"
|
||||
}
|
||||
Write-Host "MEDIA_RUNTIME_OK pyav=18.0.0 lz4=4.4.5"
|
||||
return $root
|
||||
}
|
||||
|
||||
$runtime = Resolve-DDirectory $RuntimeRoot "Runtime root"
|
||||
$derived = Resolve-DDirectory (Join-Path $runtime "derived") "Runtime derived root"
|
||||
$destination = Join-Path $derived $RuntimeName
|
||||
$freeBefore = Assert-FreeSpace "media-runtime-preflight"
|
||||
|
||||
& docker image inspect $ContainerImage *> $null
|
||||
Assert-LastExitCode "Pinned container image inspection"
|
||||
|
||||
if (Test-Path -LiteralPath $destination) {
|
||||
$resolved = Assert-Runtime $destination
|
||||
Write-Output "STATE=existing-verified"
|
||||
Write-Output ("MEDIA_RUNTIME_ROOT={0}" -f $resolved)
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f ([int64](Get-PSDrive -Name D).Free))
|
||||
return
|
||||
}
|
||||
|
||||
$token = [Guid]::NewGuid().ToString("N")
|
||||
$staging = Join-Path $derived (".{0}-{1}.tmp" -f $RuntimeName, $token)
|
||||
$null = New-Item -ItemType Directory -Path $staging
|
||||
$completed = $false
|
||||
try {
|
||||
$dockerStaging = Convert-ToDockerPath $staging
|
||||
Write-Output "PHASE=media-runtime-install-start"
|
||||
& docker run --rm --network bridge --read-only `
|
||||
--security-opt "no-new-privileges:true" --cap-drop ALL --pids-limit 128 `
|
||||
--tmpfs "/tmp:rw,nosuid,size=1g" `
|
||||
-e "PIP_DISABLE_PIP_VERSION_CHECK=1" -e "PYTHONDONTWRITEBYTECODE=1" `
|
||||
-v ("{0}:/target:rw" -f $dockerStaging) `
|
||||
--entrypoint python3 $ContainerImage -m pip install `
|
||||
--no-cache-dir --only-binary ":all:" --target /target `
|
||||
"av==18.0.0" "lz4==4.4.5"
|
||||
Assert-LastExitCode "E15 media runtime installation"
|
||||
|
||||
$payloadSha256 = Get-PayloadDigest $staging
|
||||
$manifest = [ordered]@{
|
||||
schema_version = "missioncore.e15-media-runtime/v1"
|
||||
runtime_name = $RuntimeName
|
||||
created_at_utc = [DateTime]::UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ")
|
||||
container_image = $ContainerImage
|
||||
packages = [ordered]@{ av = "18.0.0"; lz4 = "4.4.5" }
|
||||
payload_sha256 = $payloadSha256
|
||||
storage_scope = "D-only-immutable-runtime"
|
||||
}
|
||||
$manifest | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $staging "manifest.json") -Encoding utf8
|
||||
$null = Assert-Runtime $staging
|
||||
Move-Item -LiteralPath $staging -Destination $destination
|
||||
$completed = $true
|
||||
$resolved = Assert-Runtime $destination
|
||||
$freeAfter = Assert-FreeSpace "media-runtime-published"
|
||||
Write-Output "STATE=created-verified"
|
||||
Write-Output ("MEDIA_RUNTIME_ROOT={0}" -f $resolved)
|
||||
Write-Output ("PAYLOAD_SHA256={0}" -f $payloadSha256)
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f $freeAfter)
|
||||
}
|
||||
finally {
|
||||
if (-not $completed -and (Test-Path -LiteralPath $staging)) {
|
||||
Remove-Item -LiteralPath $staging -Recurse -Force
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)] [string]$JobRoot,
|
||||
[Parameter(Mandatory = $true)] [string]$RunnerPath,
|
||||
[Parameter(Mandatory = $true)] [string]$ProfilePath,
|
||||
[Parameter(Mandatory = $true)] [string]$DetectorProfilePath,
|
||||
[Parameter(Mandatory = $true)] [string]$SemanticProfilePath,
|
||||
[Parameter(Mandatory = $true)] [string]$ValidFovRoot,
|
||||
[Parameter(Mandatory = $true)] [string]$LidarPackRoot,
|
||||
[ValidateRange(0, 1000000)] [int]$StartFrame = 1000,
|
||||
[ValidateRange(0, 1000000)] [int]$EndFrame = 1600,
|
||||
[ValidateRange(1, 1000)] [int]$FreeGiBFloor = 360,
|
||||
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
|
||||
[string]$ModelRoot = "D:\NDC_MISSIONCORE\runtime\models\yolox_s",
|
||||
[string]$TritonContainer = "mission-core-triton",
|
||||
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode([string]$Operation) {
|
||||
if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" }
|
||||
}
|
||||
|
||||
function Resolve-DDirectory([string]$Path, [string]$Label) {
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
||||
if (-not $item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") {
|
||||
throw "$Label must be a real D: directory"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Resolve-DFile([string]$Path, [string]$Label) {
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
||||
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") {
|
||||
throw "$Label must be a regular D: file"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath([string]$Path) { return $Path.Replace("\", "/") }
|
||||
function Get-DFreeBytes { return [int64](Get-PSDrive -Name D).Free }
|
||||
function Assert-FreeSpace([string]$Phase, [int64]$RequiredAdditionalBytes = 0) {
|
||||
$free = Get-DFreeBytes
|
||||
$floor = [int64]$FreeGiBFloor * 1GB
|
||||
Write-Host (
|
||||
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3} REQUIRED_ADDITIONAL_GIB={4}" -f
|
||||
$Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor,
|
||||
[math]::Round($RequiredAdditionalBytes / 1GB, 3)
|
||||
)
|
||||
if ($free -lt ($floor + $RequiredAdditionalBytes)) {
|
||||
throw "D: lacks the guarded LAB E10 reserve during $Phase"
|
||||
}
|
||||
return $free
|
||||
}
|
||||
|
||||
function Test-TritonModelReady {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -TimeoutSec 10 `
|
||||
-Uri "http://127.0.0.1:8000/v2/models/yolox_s/ready"
|
||||
return $response.StatusCode -eq 200
|
||||
}
|
||||
catch { return $false }
|
||||
}
|
||||
|
||||
$jobDirectory = Resolve-DDirectory $JobRoot "Job root"
|
||||
$runner = Resolve-DFile $RunnerPath "LAB E10 runner"
|
||||
$profile = Resolve-DFile $ProfilePath "LAB E10 profile"
|
||||
$detectorProfile = Resolve-DFile $DetectorProfilePath "Detector profile"
|
||||
$semanticProfile = Resolve-DFile $SemanticProfilePath "Semantic profile"
|
||||
$validFov = Resolve-DDirectory $ValidFovRoot "Valid-FOV root"
|
||||
$lidarPack = Resolve-DDirectory $LidarPackRoot "LiDAR replay pack"
|
||||
$runtime = Resolve-DDirectory $RuntimeRoot "Runtime root"
|
||||
$model = Resolve-DDirectory $ModelRoot "YOLOX model root"
|
||||
$runnerRoot = Split-Path $runner -Parent
|
||||
foreach ($path in @($profile, $detectorProfile, $semanticProfile)) {
|
||||
if ((Split-Path $path -Parent) -ne $runnerRoot) { throw "Runner and profiles must share one mount" }
|
||||
}
|
||||
foreach ($dependency in @(
|
||||
"e10_fusion_runtime.py",
|
||||
"run_e9_multirate_perception.py",
|
||||
"run_e8_realtime_tracking.py",
|
||||
"run_e5_instance_tracking.py",
|
||||
"run_e4_full_session_segmentation.py",
|
||||
"run_recorded_perception_epoch.py",
|
||||
"run_e3_rectified_segmentation.py",
|
||||
"run_evaluation_prelabels.py"
|
||||
)) {
|
||||
$null = Resolve-DFile (Join-Path $runnerRoot $dependency) "LAB E10 dependency"
|
||||
}
|
||||
|
||||
$job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json
|
||||
if ($job.schema_version -ne "missioncore.compute-job/v1" -or $job.job_id -ne (Split-Path $jobDirectory -Leaf)) {
|
||||
throw "Compute job manifest is incompatible"
|
||||
}
|
||||
$sourceId = [string]$job.input.source_id
|
||||
$epoch = [int]$job.input.codec_epoch
|
||||
$fullFrameCount = [int]$job.input.segment_count
|
||||
$clipFrameCount = $EndFrame - $StartFrame + 1
|
||||
$timelineStart = [double]$job.input.timeline.start_seconds
|
||||
if ($sourceId -ne "sensor.camera.right" -or $StartFrame -lt 0 -or $EndFrame -lt $StartFrame -or $EndFrame -ge $fullFrameCount -or $clipFrameCount -lt 2) {
|
||||
throw "LAB E10 clip escapes the camera job"
|
||||
}
|
||||
$lidarManifest = Get-Content -LiteralPath (Join-Path $lidarPack "manifest.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$lidarManifest.schema_version -ne "missioncore.e10-lidar-replay-pack/v1" -or
|
||||
$lidarManifest.identity.job_id -ne $job.job_id -or
|
||||
[int]$lidarManifest.identity.source_start_frame_index -ne $StartFrame -or
|
||||
[int]$lidarManifest.identity.source_end_frame_index -ne $EndFrame -or
|
||||
[int]$lidarManifest.identity.frame_count -ne $clipFrameCount
|
||||
) { throw "LAB E10 LiDAR pack differs from the selected camera clip" }
|
||||
Write-Output ("PHASE=inputs-validated JOB={0} CLIP={1}-{2} FRAMES={3} LIDAR_PACK={4}" -f $job.job_id, $StartFrame, $EndFrame, $clipFrameCount, $lidarManifest.pack_id)
|
||||
|
||||
$epochRoot = Resolve-DDirectory (Join-Path $jobDirectory ("input\camera\{0}\epoch-{1}" -f $sourceId, $epoch)) "Camera epoch"
|
||||
$initPath = Resolve-DFile (Join-Path $epochRoot "init.mp4") "Camera init"
|
||||
$segmentsRoot = Resolve-DDirectory (Join-Path $epochRoot "segments") "Camera segments"
|
||||
$derivedRoot = Join-Path $runtime "derived"
|
||||
$tmpRoot = Join-Path $runtime "tmp"
|
||||
$cacheRoot = Resolve-DDirectory (Join-Path $runtime "cache\perception-e3-models-v1") "EoMT cache"
|
||||
$e3Environment = Resolve-DDirectory (Join-Path $derivedRoot "perception-e3-opencv413092-v1") "E3 environment"
|
||||
$torchEnvironment = Resolve-DDirectory (Join-Path $derivedRoot "perception-p0-env-v1") "Torch environment"
|
||||
$transformersEnvironment = Resolve-DDirectory (Join-Path $derivedRoot "perception-p0-transformers4576-v1") "Transformers environment"
|
||||
|
||||
$pixelWorkingSet = [int64]$clipFrameCount * 800 * 600 * 3
|
||||
$partialStreamReserve = [int64][math]::Ceiling(([int64]$job.input.byte_length * ($EndFrame + 1) / $fullFrameCount) * 1.2)
|
||||
$workingSetReserve = [int64][math]::Ceiling($pixelWorkingSet * 1.4) + $partialStreamReserve + 3GB
|
||||
$freeBytesBefore = Assert-FreeSpace "preflight" $workingSetReserve
|
||||
|
||||
& docker image inspect $ContainerImage *> $null
|
||||
Assert-LastExitCode "Existing container image inspection"
|
||||
$tritonState = docker inspect --format "{{.State.Running}}" $TritonContainer
|
||||
Assert-LastExitCode "Triton container inspection"
|
||||
if ($tritonState.Trim().ToLowerInvariant() -ne "true") { throw "LAB E10 requires the existing Triton container" }
|
||||
|
||||
$runnerName = Split-Path $runner -Leaf
|
||||
$profileName = Split-Path $profile -Leaf
|
||||
$detectorProfileName = Split-Path $detectorProfile -Leaf
|
||||
$semanticProfileName = Split-Path $semanticProfile -Leaf
|
||||
$lidarMount = "/" + (Split-Path $lidarPack -Leaf)
|
||||
$orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$commonMounts = @(
|
||||
"-e", "PYTHONPATH=/runner:/opt/transformers:/opt/env",
|
||||
"-e", "TORCH_HOME=/cache/torch",
|
||||
"-e", "HF_HOME=/cache/huggingface",
|
||||
"-e", "HF_HUB_OFFLINE=1",
|
||||
"-e", "TRANSFORMERS_OFFLINE=1",
|
||||
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
||||
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
||||
"-v", ((Convert-ToDockerPath $lidarPack) + (":{0}:ro" -f $lidarMount)),
|
||||
"-v", ((Convert-ToDockerPath $model) + ":/model:ro"),
|
||||
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"),
|
||||
"-v", ((Convert-ToDockerPath $e3Environment) + ":/environment:ro"),
|
||||
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro")
|
||||
)
|
||||
$commonRunnerArgs = @(
|
||||
"--job", "/job",
|
||||
"--profile", ("/runner/{0}" -f $profileName),
|
||||
"--detector-profile", ("/runner/{0}" -f $detectorProfileName),
|
||||
"--semantic-profile", ("/runner/{0}" -f $semanticProfileName),
|
||||
"--valid-fov-root", "/valid-fov",
|
||||
"--model-root", "/model",
|
||||
"--cache", "/cache",
|
||||
"--environment", "/environment",
|
||||
"--lidar-pack", $lidarMount
|
||||
)
|
||||
$preflightArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", "none", "--read-only",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "512", "--shm-size", "4g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g"
|
||||
) + $commonMounts + @(
|
||||
"--entrypoint", "python3", $ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "preflight"
|
||||
) + $commonRunnerArgs
|
||||
Write-Output "PHASE=e10-preflight-start"
|
||||
& docker @preflightArgs
|
||||
Assert-LastExitCode "LAB E10 preflight"
|
||||
Write-Output "PHASE=e10-preflight-complete"
|
||||
|
||||
$modelWasReady = Test-TritonModelReady
|
||||
$loadedByRun = $false
|
||||
if (-not $modelWasReady) {
|
||||
Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 -Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/load" -Method Post -ContentType "application/json" -Body "{}" *> $null
|
||||
$loadedByRun = $true
|
||||
$deadline = [DateTime]::UtcNow.AddSeconds(60)
|
||||
while (-not (Test-TritonModelReady)) {
|
||||
if ([DateTime]::UtcNow -ge $deadline) { throw "YOLOX-S did not become ready" }
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
}
|
||||
Write-Output ("PHASE=e10-model-ready LOADED_BY_RUN={0}" -f $loadedByRun)
|
||||
|
||||
$token = [Guid]::NewGuid().ToString("N")
|
||||
$workRoot = Join-Path $tmpRoot ("{0}-e10-{1}" -f $job.job_id, $token)
|
||||
$framesRoot = Join-Path $workRoot "frames"
|
||||
$streamPath = Join-Path $workRoot "camera.mp4"
|
||||
$ptsPath = Join-Path $workRoot "pts.json"
|
||||
$timelinePath = Join-Path $workRoot "timeline.jsonl"
|
||||
$publishRoot = Join-Path $derivedRoot (".{0}-e10-{1}.publish" -f $job.job_id, $token)
|
||||
$stagingRoot = Join-Path $publishRoot "output"
|
||||
$null = New-Item -ItemType Directory -Path $framesRoot
|
||||
$null = New-Item -ItemType Directory -Path $publishRoot
|
||||
$completed = $false
|
||||
$totalWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
|
||||
try {
|
||||
$stream = [IO.File]::Open($streamPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
||||
try {
|
||||
$input = [IO.File]::OpenRead($initPath)
|
||||
try { $input.CopyTo($stream) } finally { $input.Dispose() }
|
||||
for ($sequence = 1; $sequence -le ($EndFrame + 1); $sequence++) {
|
||||
$path = Resolve-DFile (Join-Path $segmentsRoot ("{0}.m4s" -f $sequence)) "Camera segment"
|
||||
$input = [IO.File]::OpenRead($path)
|
||||
try { $input.CopyTo($stream) } finally { $input.Dispose() }
|
||||
}
|
||||
$stream.Flush($true)
|
||||
}
|
||||
finally { $stream.Dispose() }
|
||||
$null = Assert-FreeSpace "post-stream-reconstruction"
|
||||
|
||||
$extractWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
$selectFilter = "select='between(n,{0},{1})'" -f $StartFrame, $EndFrame
|
||||
& ffmpeg -hide_banner -loglevel fatal -i $streamPath -map 0:v:0 -vf $selectFilter -fps_mode passthrough (Join-Path $framesRoot "frame-%06d.png")
|
||||
Assert-LastExitCode "LAB E10 camera extraction"
|
||||
& ffprobe -v error -select_streams v:0 -show_entries frame=best_effort_timestamp_time -of json $streamPath | Set-Content -LiteralPath $ptsPath -Encoding utf8
|
||||
Assert-LastExitCode "LAB E10 timestamp probe"
|
||||
$decoded = @(Get-ChildItem -LiteralPath $framesRoot -File -Filter "frame-*.png")
|
||||
$pts = @((Get-Content -LiteralPath $ptsPath -Raw | ConvertFrom-Json).frames)
|
||||
if ($decoded.Count -ne $clipFrameCount -or $pts.Count -ne ($EndFrame + 1)) { throw "LAB E10 decoded frame count changed" }
|
||||
$firstEpochSeconds = [double]::Parse(([string]$pts[0].best_effort_timestamp_time).Trim(), [Globalization.CultureInfo]::InvariantCulture)
|
||||
$writer = [IO.StreamWriter]::new($timelinePath, $false, [Text.UTF8Encoding]::new($false))
|
||||
try {
|
||||
for ($local = 0; $local -lt $clipFrameCount; $local++) {
|
||||
$source = $StartFrame + $local
|
||||
$epochSeconds = [double]::Parse(([string]$pts[$source].best_effort_timestamp_time).Trim(), [Globalization.CultureInfo]::InvariantCulture) - $firstEpochSeconds
|
||||
$row = [ordered]@{
|
||||
frame_index = $local
|
||||
sequence = $local + 1
|
||||
source_frame_index = $source
|
||||
source_sequence = $source + 1
|
||||
epoch_seconds = $epochSeconds
|
||||
session_seconds = $timelineStart + $epochSeconds
|
||||
}
|
||||
$writer.WriteLine(($row | ConvertTo-Json -Compress))
|
||||
}
|
||||
$writer.Flush()
|
||||
}
|
||||
finally { $writer.Dispose() }
|
||||
$extractWatch.Stop()
|
||||
$freePostExtract = Assert-FreeSpace "post-frame-extraction"
|
||||
|
||||
$runArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", ("container:{0}" -f $TritonContainer),
|
||||
"--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "512", "--shm-size", "4g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g"
|
||||
) + $commonMounts + @(
|
||||
"-v", ((Convert-ToDockerPath $framesRoot) + ":/frames:ro"),
|
||||
"-v", ((Convert-ToDockerPath $workRoot) + ":/work:ro"),
|
||||
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
|
||||
"--entrypoint", "python3", $ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "run"
|
||||
) + $commonRunnerArgs + @(
|
||||
"--frames", "/frames",
|
||||
"--timeline", "/work/timeline.jsonl",
|
||||
"--output", "/publish/output",
|
||||
"--triton-url", "http://127.0.0.1:8000",
|
||||
"--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB),
|
||||
"--orchestrator-sha256", $orchestratorSha256,
|
||||
"--container-image", $ContainerImage
|
||||
)
|
||||
Write-Output ("PHASE=e10-integrated-replay-start FRAMES={0}" -f $clipFrameCount)
|
||||
& docker @runArgs
|
||||
Assert-LastExitCode "LAB E10 integrated replay"
|
||||
$freePostReplay = Assert-FreeSpace "post-replay"
|
||||
|
||||
$result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json
|
||||
if ($result.schema_version -ne "missioncore.e10-integrated-perception-result/v1" -or $result.result_id -notmatch "^e10-integrated-perception-[a-f0-9]{64}$") {
|
||||
throw "LAB E10 result manifest is incompatible"
|
||||
}
|
||||
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
||||
if (Test-Path -LiteralPath $finalRoot) { throw "Immutable LAB E10 result already exists" }
|
||||
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
|
||||
Remove-Item -LiteralPath $publishRoot -Force
|
||||
$completed = $true
|
||||
$totalWatch.Stop()
|
||||
$freeFinal = Assert-FreeSpace "post-publication"
|
||||
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
||||
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
||||
Write-Output ("ACCEPTANCE_STATE={0}" -f $result.acceptance_state)
|
||||
Write-Output ("PREPARATION_SECONDS={0}" -f $extractWatch.Elapsed.TotalSeconds)
|
||||
Write-Output ("ORCHESTRATOR_WALL_SECONDS={0}" -f $totalWatch.Elapsed.TotalSeconds)
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_EXTRACT={0}" -f $freePostExtract)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_REPLAY={0}" -f $freePostReplay)
|
||||
Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeFinal)
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $workRoot) { Remove-Item -LiteralPath $workRoot -Recurse -Force }
|
||||
if (-not $completed -and (Test-Path -LiteralPath $publishRoot)) { Remove-Item -LiteralPath $publishRoot -Recurse -Force }
|
||||
if ($loadedByRun) {
|
||||
try {
|
||||
Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 -Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/unload" -Method Post -ContentType "application/json" -Body "{}" *> $null
|
||||
Write-Output "PHASE=e10-model-state-restored"
|
||||
}
|
||||
catch { Write-Warning "LAB E10 could not restore the prior YOLOX-S state" }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[switch]$TokenStdin,
|
||||
[ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")]
|
||||
[string]$RequestId = ("physical-k1-shadow-{0}" -f [Guid]::NewGuid().ToString("N")),
|
||||
[string]$PersistentContainer = "mission-core-perception-worker",
|
||||
[string]$PersistentOutputRoot = "D:\NDC_MISSIONCORE\runtime\derived\.perception-persistent-publish",
|
||||
[ValidateRange(1024, 65535)] [int]$PersistentPort = 18020,
|
||||
[ValidateRange(1, 1000)] [int]$FreeGiBFloor = 360
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode([string]$Operation) {
|
||||
if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" }
|
||||
}
|
||||
|
||||
function Resolve-DDirectory([string]$Path, [string]$Label) {
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
||||
if (-not $item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") {
|
||||
throw "$Label must be a real D: directory"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Get-DFreeBytes { return [int64](Get-PSDrive -Name D).Free }
|
||||
|
||||
if (-not $TokenStdin) { throw "Persistent shadow run requires the token through stdin" }
|
||||
$token = [Console]::In.ReadLine()
|
||||
if (-not $token -or $token.Length -lt 40 -or $token.Length -gt 512) {
|
||||
throw "Persistent shadow token is missing or malformed"
|
||||
}
|
||||
|
||||
$outputRoot = Resolve-DDirectory $PersistentOutputRoot "Persistent output root"
|
||||
$freeBefore = Get-DFreeBytes
|
||||
if ($freeBefore -lt ([int64]$FreeGiBFloor * 1GB + 512MB)) {
|
||||
throw "D: lacks the guarded persistent-run reserve"
|
||||
}
|
||||
$containerRunning = docker inspect --format "{{.State.Running}}" $PersistentContainer
|
||||
Assert-LastExitCode "Persistent worker inspection"
|
||||
if ($containerRunning.Trim().ToLowerInvariant() -ne "true") {
|
||||
throw "Persistent worker is not running"
|
||||
}
|
||||
$outputName = $RequestId
|
||||
$request = @{
|
||||
request_id = $RequestId
|
||||
output_name = $outputName
|
||||
token = $token
|
||||
} | ConvertTo-Json -Compress
|
||||
$token = $null
|
||||
$client = (
|
||||
"import sys,urllib.request,urllib.error;data=sys.stdin.buffer.read();" +
|
||||
"request=urllib.request.Request('http://127.0.0.1:{0}/run',data=data," -f $PersistentPort
|
||||
) + "headers={'Content-Type':'application/json'},method='POST');" +
|
||||
"`ntry:`n response=urllib.request.urlopen(request,timeout=3600); body=response.read(); code=response.status" +
|
||||
"`nexcept urllib.error.HTTPError as exc:`n body=exc.read(); code=exc.code" +
|
||||
"`nsys.stdout.buffer.write(body);raise SystemExit(0 if code==200 else 22)"
|
||||
try {
|
||||
$responseJson = $request | & docker exec -i $PersistentContainer python3 -c $client
|
||||
$request = $null
|
||||
Assert-LastExitCode "Persistent worker request"
|
||||
}
|
||||
finally {
|
||||
$token = $null
|
||||
$request = $null
|
||||
}
|
||||
$response = $responseJson | ConvertFrom-Json
|
||||
if ($response.request_id -ne $RequestId -or $response.models_reused -ne $true) {
|
||||
throw "Persistent worker response contract changed"
|
||||
}
|
||||
$staging = Join-Path $outputRoot $outputName
|
||||
$resultPath = Join-Path $staging "result.json"
|
||||
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
|
||||
throw "Persistent worker result manifest is missing"
|
||||
}
|
||||
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$result.schema_version -ne "missioncore.e15-shadow-inference-result/v1" -or
|
||||
$result.result_id -notmatch "^e15-shadow-inference-[a-f0-9]{64}$" -or
|
||||
$result.publication_scope -ne "live-shadow-diagnostic-only"
|
||||
) { throw "Persistent worker result manifest is incompatible" }
|
||||
$derivedRoot = Resolve-DDirectory (Split-Path $outputRoot -Parent) "Runtime derived root"
|
||||
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
||||
if (Test-Path -LiteralPath $finalRoot) {
|
||||
throw "Immutable persistent-worker result already exists"
|
||||
}
|
||||
Move-Item -LiteralPath $staging -Destination $finalRoot
|
||||
$freeFinal = Get-DFreeBytes
|
||||
if ($freeFinal -lt ([int64]$FreeGiBFloor * 1GB)) {
|
||||
throw "D: crossed the guarded floor after persistent run"
|
||||
}
|
||||
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
||||
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
||||
Write-Output ("ACCEPTANCE_STATE={0}" -f $result.acceptance_state)
|
||||
Write-Output ("RUNNER_EXIT_CODE={0}" -f $response.exit_code)
|
||||
Write-Output "MODELS_REUSED=true"
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeFinal)
|
||||
@@ -0,0 +1,432 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)] [string]$JobRoot,
|
||||
[Parameter(Mandatory = $true)] [string]$RunnerPath,
|
||||
[Parameter(Mandatory = $true)] [string]$LiveProfilePath,
|
||||
[Parameter(Mandatory = $true)] [string]$E14ProfilePath,
|
||||
[Parameter(Mandatory = $true)] [string]$DetectorProfilePath,
|
||||
[Parameter(Mandatory = $true)] [string]$SemanticProfilePath,
|
||||
[Parameter(Mandatory = $true)] [string]$ValidFovRoot,
|
||||
[Parameter(Mandatory = $true)] [string]$ProjectionPackRoot,
|
||||
[Parameter(Mandatory = $true)] [string]$PackageRoot,
|
||||
[switch]$PreflightOnly,
|
||||
[switch]$PersistentService,
|
||||
[switch]$TokenStdin,
|
||||
[ValidateRange(5, 3600)] [int]$MaximumDurationSeconds = 20,
|
||||
[ValidateRange(1, 1000)] [int]$FreeGiBFloor = 360,
|
||||
[ValidateRange(1024, 65535)] [int]$SourcePort = 18012,
|
||||
[string]$SourceHost = "host.docker.internal",
|
||||
[string]$SourcePath = "/api/v1/device-plugins/nodedc.device.xgrids-lixelkity-k1/live-perception-shadow",
|
||||
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
|
||||
[string]$ModelRoot = "D:\NDC_MISSIONCORE\runtime\models\yolox_s",
|
||||
[string]$MediaRuntimeRoot = "D:\NDC_MISSIONCORE\runtime\derived\perception-e15-media-pyav180-lz445-v1",
|
||||
[string]$TritonContainer = "mission-core-triton",
|
||||
[string]$PersistentContainer = "mission-core-perception-worker",
|
||||
[string]$PersistentOutputRoot = "D:\NDC_MISSIONCORE\runtime\derived\.perception-persistent-publish",
|
||||
[ValidateRange(1024, 65535)] [int]$PersistentPort = 18020,
|
||||
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode([string]$Operation) {
|
||||
if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" }
|
||||
}
|
||||
|
||||
function Resolve-DDirectory([string]$Path, [string]$Label) {
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
||||
if (-not $item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") {
|
||||
throw "$Label must be a real D: directory"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Resolve-DFile([string]$Path, [string]$Label) {
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
||||
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") {
|
||||
throw "$Label must be a regular D: file"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath([string]$Path) { return $Path.Replace("\", "/") }
|
||||
function Get-DFreeBytes { return [int64](Get-PSDrive -Name D).Free }
|
||||
|
||||
function Assert-FreeSpace([string]$Phase, [int64]$RequiredAdditionalBytes = 0) {
|
||||
$free = Get-DFreeBytes
|
||||
$floor = [int64]$FreeGiBFloor * 1GB
|
||||
Write-Host (
|
||||
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3} REQUIRED_ADDITIONAL_GIB={4}" -f
|
||||
$Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor,
|
||||
[math]::Round($RequiredAdditionalBytes / 1GB, 3)
|
||||
)
|
||||
if ($free -lt ($floor + $RequiredAdditionalBytes)) {
|
||||
throw "D: lacks the guarded LAB E15 reserve during $Phase"
|
||||
}
|
||||
return $free
|
||||
}
|
||||
|
||||
function Test-TritonModelReady {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -TimeoutSec 10 `
|
||||
-Uri "http://127.0.0.1:8000/v2/models/yolox_s/ready"
|
||||
return $response.StatusCode -eq 200
|
||||
}
|
||||
catch { return $false }
|
||||
}
|
||||
|
||||
$jobDirectory = Resolve-DDirectory $JobRoot "Job root"
|
||||
$runner = Resolve-DFile $RunnerPath "LAB E15 runner"
|
||||
$liveProfile = Resolve-DFile $LiveProfilePath "LAB E15 live profile"
|
||||
$e14Profile = Resolve-DFile $E14ProfilePath "Accepted E14 profile"
|
||||
$detectorProfile = Resolve-DFile $DetectorProfilePath "Detector profile"
|
||||
$semanticProfile = Resolve-DFile $SemanticProfilePath "Semantic profile"
|
||||
$validFov = Resolve-DDirectory $ValidFovRoot "Valid-FOV root"
|
||||
$projectionPack = Resolve-DDirectory $ProjectionPackRoot "E15 projection pack"
|
||||
$package = Resolve-DDirectory $PackageRoot "Mission Core package root"
|
||||
$runtime = Resolve-DDirectory $RuntimeRoot "Runtime root"
|
||||
$model = Resolve-DDirectory $ModelRoot "YOLOX model root"
|
||||
$mediaRuntime = Resolve-DDirectory $MediaRuntimeRoot "E15 media runtime"
|
||||
$runnerRoot = Split-Path $runner -Parent
|
||||
|
||||
foreach ($path in @($liveProfile, $e14Profile, $detectorProfile, $semanticProfile)) {
|
||||
if ((Split-Path $path -Parent) -ne $runnerRoot) {
|
||||
throw "Runner and profiles must share one immutable mount"
|
||||
}
|
||||
}
|
||||
foreach ($dependency in @(
|
||||
"e10_fusion_runtime.py",
|
||||
"e15_shadow_runtime.py",
|
||||
"run_e10_integrated_perception.py",
|
||||
"run_e12_shadow_transport_probe.py",
|
||||
"run_e9_multirate_perception.py",
|
||||
"run_e8_realtime_tracking.py",
|
||||
"run_e5_instance_tracking.py",
|
||||
"run_e4_full_session_segmentation.py",
|
||||
"run_recorded_perception_epoch.py",
|
||||
"run_e3_rectified_segmentation.py",
|
||||
"run_evaluation_prelabels.py"
|
||||
)) {
|
||||
$null = Resolve-DFile (Join-Path $runnerRoot $dependency) "LAB E15 dependency"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $package "k1link\compute\live_perception.py") -PathType Leaf)) {
|
||||
throw "Mission Core package mount lacks live perception synchronization"
|
||||
}
|
||||
|
||||
$job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$job.schema_version -ne "missioncore.compute-job/v1" -or
|
||||
$job.job_id -ne (Split-Path $jobDirectory -Leaf) -or
|
||||
$job.input.source_id -ne "sensor.camera.right"
|
||||
) { throw "Compute bootstrap job manifest is incompatible" }
|
||||
$live = Get-Content -LiteralPath $liveProfile -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$live.schema_version -ne "missioncore.e15-shadow-inference-profile/v1" -or
|
||||
$live.mode -ne "replay-shadow-gate" -or
|
||||
[bool]$live.authority.commands_enabled -or
|
||||
[bool]$live.authority.navigation_or_safety_accepted -or
|
||||
$live.transport.pyav_version -ne "18.0.0"
|
||||
) { throw "LAB E15 replay-shadow authority contract changed" }
|
||||
$projection = Get-Content -LiteralPath (Join-Path $projectionPack "manifest.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$projection.schema_version -ne "missioncore.e15-live-projection-pack/v1" -or
|
||||
$projection.identity.source_id -ne "sensor.camera.right" -or
|
||||
$projection.identity.calibration_slot -ne "camera_1" -or
|
||||
$projection.identity.calibration_sha256 -ne $live.source.calibration_sha256
|
||||
) { throw "LAB E15 projection pack binding changed" }
|
||||
$mediaManifest = Get-Content -LiteralPath (Join-Path $mediaRuntime "manifest.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$mediaManifest.schema_version -ne "missioncore.e15-media-runtime/v1" -or
|
||||
$mediaManifest.container_image -ne $ContainerImage -or
|
||||
$mediaManifest.packages.av -ne "18.0.0" -or
|
||||
$mediaManifest.packages.lz4 -ne "4.4.5"
|
||||
) { throw "LAB E15 media runtime identity changed" }
|
||||
|
||||
$derivedRoot = Resolve-DDirectory (Join-Path $runtime "derived") "Runtime derived root"
|
||||
$cacheRoot = Resolve-DDirectory (Join-Path $runtime "cache\perception-e3-models-v1") "EoMT cache"
|
||||
$e3Environment = Resolve-DDirectory (Join-Path $derivedRoot "perception-e3-opencv413092-v1") "E3 environment"
|
||||
$torchEnvironment = Resolve-DDirectory (Join-Path $derivedRoot "perception-p0-env-v1") "Torch environment"
|
||||
$transformersEnvironment = Resolve-DDirectory (Join-Path $derivedRoot "perception-p0-transformers4576-v1") "Transformers environment"
|
||||
$freeBytesBefore = Assert-FreeSpace "preflight" 512MB
|
||||
|
||||
& docker image inspect $ContainerImage *> $null
|
||||
Assert-LastExitCode "Pinned container image inspection"
|
||||
$tritonState = docker inspect --format "{{.State.Running}}" $TritonContainer
|
||||
Assert-LastExitCode "Triton container inspection"
|
||||
if ($tritonState.Trim().ToLowerInvariant() -ne "true") {
|
||||
throw "LAB E15 requires the existing Triton container"
|
||||
}
|
||||
|
||||
$runnerName = Split-Path $runner -Leaf
|
||||
$liveProfileName = Split-Path $liveProfile -Leaf
|
||||
$e14ProfileName = Split-Path $e14Profile -Leaf
|
||||
$detectorProfileName = Split-Path $detectorProfile -Leaf
|
||||
$semanticProfileName = Split-Path $semanticProfile -Leaf
|
||||
$projectionMount = "/" + (Split-Path $projectionPack -Leaf)
|
||||
$packageMount = "/" + (Split-Path $package -Leaf)
|
||||
$orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$commonMounts = @(
|
||||
"-e", ("PYTHONPATH=/runner:{0}:/opt/media:/opt/transformers:/opt/env" -f $packageMount),
|
||||
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
||||
"-e", "TORCH_HOME=/cache/torch",
|
||||
"-e", "HF_HOME=/cache/huggingface",
|
||||
"-e", "HF_HUB_OFFLINE=1",
|
||||
"-e", "TRANSFORMERS_OFFLINE=1",
|
||||
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
||||
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
||||
"-v", ((Convert-ToDockerPath $projectionPack) + (":{0}:ro" -f $projectionMount)),
|
||||
"-v", ((Convert-ToDockerPath $model) + ":/model:ro"),
|
||||
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"),
|
||||
"-v", ((Convert-ToDockerPath $e3Environment) + ":/environment:ro"),
|
||||
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
||||
"-v", ((Convert-ToDockerPath $mediaRuntime) + ":/opt/media:ro"),
|
||||
"-v", ((Convert-ToDockerPath $package) + (":{0}:ro" -f $packageMount)),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro")
|
||||
)
|
||||
$commonRunnerArgs = @(
|
||||
"--job", "/job",
|
||||
"--live-profile", ("/runner/{0}" -f $liveProfileName),
|
||||
"--e14-profile", ("/runner/{0}" -f $e14ProfileName),
|
||||
"--detector-profile", ("/runner/{0}" -f $detectorProfileName),
|
||||
"--semantic-profile", ("/runner/{0}" -f $semanticProfileName),
|
||||
"--valid-fov-root", "/valid-fov",
|
||||
"--projection-pack", $projectionMount,
|
||||
"--model-root", "/model",
|
||||
"--cache", "/cache",
|
||||
"--environment", "/environment",
|
||||
"--worker-package", $packageMount
|
||||
)
|
||||
|
||||
if ($PersistentService) {
|
||||
if ($PreflightOnly -or $TokenStdin) {
|
||||
throw "Persistent service cannot be combined with one-shot switches"
|
||||
}
|
||||
$persistentParent = Resolve-DDirectory (Split-Path $PersistentOutputRoot -Parent) "Persistent output parent"
|
||||
if (-not (Test-Path -LiteralPath $PersistentOutputRoot)) {
|
||||
$null = New-Item -ItemType Directory -Path $PersistentOutputRoot
|
||||
}
|
||||
$persistentOutput = Resolve-DDirectory $PersistentOutputRoot "Persistent output root"
|
||||
if ((Split-Path $persistentOutput -Parent) -ne $persistentParent) {
|
||||
throw "Persistent output root must be a direct child of its guarded D: parent"
|
||||
}
|
||||
$runnerSha256 = (Get-FileHash -LiteralPath $runner -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$existingContainerId = docker ps -a --filter ("name=^{0}$" -f $PersistentContainer) --format "{{.ID}}"
|
||||
Assert-LastExitCode "Persistent worker container lookup"
|
||||
if ($existingContainerId) {
|
||||
$labels = (docker inspect --format "{{json .Config.Labels}}" $PersistentContainer) | ConvertFrom-Json
|
||||
Assert-LastExitCode "Persistent worker label inspection"
|
||||
if (
|
||||
$labels.'missioncore.role' -ne "perception-persistent-worker" -or
|
||||
$labels.'missioncore.runner.sha256' -ne $runnerSha256
|
||||
) { throw "Existing persistent worker has a different immutable identity" }
|
||||
$running = docker inspect --format "{{.State.Running}}" $PersistentContainer
|
||||
Assert-LastExitCode "Persistent worker state inspection"
|
||||
if ($running.Trim().ToLowerInvariant() -ne "true") {
|
||||
throw "Matching persistent worker exists but is not running"
|
||||
}
|
||||
$healthProbe = (
|
||||
"import json,urllib.request;d=json.load(urllib.request.urlopen(" +
|
||||
"'http://127.0.0.1:{0}/health',timeout=5));" -f $PersistentPort
|
||||
) + "raise SystemExit(0 if d.get('ok') and d.get('models_loaded') else 2)"
|
||||
$previousErrorPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
& docker exec $PersistentContainer python3 -c $healthProbe 2>$null
|
||||
$healthExitCode = $LASTEXITCODE
|
||||
$ErrorActionPreference = $previousErrorPreference
|
||||
if ($healthExitCode -ne 0) { throw "Existing persistent worker health probe failed" }
|
||||
Write-Output "STATE=persistent-worker-reused"
|
||||
Write-Output ("CONTAINER={0}" -f $PersistentContainer)
|
||||
return
|
||||
}
|
||||
|
||||
if (-not (Test-TritonModelReady)) {
|
||||
Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 `
|
||||
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/load" `
|
||||
-Method Post -ContentType "application/json" -Body "{}" *> $null
|
||||
$deadline = [DateTime]::UtcNow.AddSeconds(60)
|
||||
while (-not (Test-TritonModelReady)) {
|
||||
if ([DateTime]::UtcNow -ge $deadline) { throw "YOLOX-S did not become ready" }
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
}
|
||||
|
||||
$serveArgs = @(
|
||||
"run", "--detach", "--name", $PersistentContainer,
|
||||
"--label", "missioncore.role=perception-persistent-worker",
|
||||
"--label", ("missioncore.runner.sha256={0}" -f $runnerSha256),
|
||||
"--gpus", "all", "--network", ("container:{0}" -f $TritonContainer),
|
||||
"--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "512", "--shm-size", "4g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g"
|
||||
) + $commonMounts + @(
|
||||
"-v", ((Convert-ToDockerPath $persistentOutput) + ":/publish:rw"),
|
||||
"--entrypoint", "python3", $ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "serve"
|
||||
) + $commonRunnerArgs + @(
|
||||
"--host", $SourceHost,
|
||||
"--port", [string]$SourcePort,
|
||||
"--path", $SourcePath,
|
||||
"--triton-url", "http://127.0.0.1:8000",
|
||||
"--output-root", "/publish",
|
||||
"--listen-host", "127.0.0.1",
|
||||
"--listen-port", [string]$PersistentPort,
|
||||
"--max-duration-seconds", [string]$MaximumDurationSeconds,
|
||||
"--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB),
|
||||
"--orchestrator-sha256", $orchestratorSha256,
|
||||
"--container-image", $ContainerImage
|
||||
)
|
||||
Write-Output "PHASE=persistent-worker-model-load-start"
|
||||
& docker @serveArgs
|
||||
Assert-LastExitCode "Persistent worker container start"
|
||||
$healthProbe = (
|
||||
"import json,urllib.request;d=json.load(urllib.request.urlopen(" +
|
||||
"'http://127.0.0.1:{0}/health',timeout=5));" -f $PersistentPort
|
||||
) + "print(json.dumps(d,sort_keys=True));raise SystemExit(0 if d.get('ok') and d.get('models_loaded') else 2)"
|
||||
$deadline = [DateTime]::UtcNow.AddSeconds(240)
|
||||
do {
|
||||
Start-Sleep -Seconds 2
|
||||
$running = docker inspect --format "{{.State.Running}}" $PersistentContainer
|
||||
if ($LASTEXITCODE -ne 0 -or $running.Trim().ToLowerInvariant() -ne "true") {
|
||||
& docker logs --tail 80 $PersistentContainer
|
||||
throw "Persistent worker exited during model load"
|
||||
}
|
||||
$previousErrorPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
$health = & docker exec $PersistentContainer python3 -c $healthProbe 2>$null
|
||||
$healthExitCode = $LASTEXITCODE
|
||||
$ErrorActionPreference = $previousErrorPreference
|
||||
$healthy = $healthExitCode -eq 0
|
||||
} while (-not $healthy -and [DateTime]::UtcNow -lt $deadline)
|
||||
if (-not $healthy) {
|
||||
& docker logs --tail 80 $PersistentContainer
|
||||
throw "Persistent worker model load did not become ready"
|
||||
}
|
||||
Write-Output "STATE=persistent-worker-ready"
|
||||
Write-Output ("CONTAINER={0}" -f $PersistentContainer)
|
||||
Write-Output ("HEALTH={0}" -f $health)
|
||||
Write-Output ("DISK_FREE_BYTES={0}" -f (Get-DFreeBytes))
|
||||
return
|
||||
}
|
||||
|
||||
$preflightArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", "none", "--read-only",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "512", "--shm-size", "4g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g"
|
||||
) + $commonMounts + @(
|
||||
"--entrypoint", "python3", $ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "preflight"
|
||||
) + $commonRunnerArgs
|
||||
Write-Output "PHASE=e15-preflight-start"
|
||||
& docker @preflightArgs
|
||||
Assert-LastExitCode "LAB E15 preflight"
|
||||
Write-Output "PHASE=e15-preflight-complete"
|
||||
if ($PreflightOnly) {
|
||||
Write-Output "STATE=preflight-ready"
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f (Get-DFreeBytes))
|
||||
return
|
||||
}
|
||||
if (-not $TokenStdin) { throw "LAB E15 requires the shadow token through stdin" }
|
||||
$shadowToken = [Console]::In.ReadLine()
|
||||
if (-not $shadowToken -or $shadowToken.Length -lt 40 -or $shadowToken.Length -gt 512) {
|
||||
throw "LAB E15 shadow token is missing or malformed"
|
||||
}
|
||||
|
||||
$modelWasReady = Test-TritonModelReady
|
||||
$loadedByRun = $false
|
||||
if (-not $modelWasReady) {
|
||||
Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 `
|
||||
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/load" `
|
||||
-Method Post -ContentType "application/json" -Body "{}" *> $null
|
||||
$loadedByRun = $true
|
||||
$deadline = [DateTime]::UtcNow.AddSeconds(60)
|
||||
while (-not (Test-TritonModelReady)) {
|
||||
if ([DateTime]::UtcNow -ge $deadline) { throw "YOLOX-S did not become ready" }
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
}
|
||||
Write-Output ("PHASE=e15-model-ready LOADED_BY_RUN={0}" -f $loadedByRun)
|
||||
|
||||
$token = [Guid]::NewGuid().ToString("N")
|
||||
$publishRoot = Join-Path $derivedRoot (".{0}-e15-{1}.publish" -f $job.job_id, $token)
|
||||
$stagingRoot = Join-Path $publishRoot "output"
|
||||
$null = New-Item -ItemType Directory -Path $publishRoot
|
||||
$completed = $false
|
||||
$totalWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
try {
|
||||
$runArgs = @(
|
||||
"run", "--rm", "--interactive", "--gpus", "all",
|
||||
"--network", ("container:{0}" -f $TritonContainer),
|
||||
"--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "512", "--shm-size", "4g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g"
|
||||
) + $commonMounts + @(
|
||||
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
|
||||
"--entrypoint", "python3", $ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "run"
|
||||
) + $commonRunnerArgs + @(
|
||||
"--host", $SourceHost,
|
||||
"--port", [string]$SourcePort,
|
||||
"--path", $SourcePath,
|
||||
"--token-stdin",
|
||||
"--triton-url", "http://127.0.0.1:8000",
|
||||
"--output", "/publish/output",
|
||||
"--max-duration-seconds", [string]$MaximumDurationSeconds,
|
||||
"--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB),
|
||||
"--orchestrator-sha256", $orchestratorSha256,
|
||||
"--container-image", $ContainerImage
|
||||
)
|
||||
Write-Output ("PHASE=e15-shadow-inference-start MAX_DURATION_SECONDS={0}" -f $MaximumDurationSeconds)
|
||||
$shadowToken | & docker @runArgs
|
||||
$runnerExitCode = $LASTEXITCODE
|
||||
$shadowToken = $null
|
||||
if ($runnerExitCode -ne 0 -and $runnerExitCode -ne 2) {
|
||||
throw "LAB E15 shadow inference failed with exit code $runnerExitCode"
|
||||
}
|
||||
$null = Assert-FreeSpace "post-shadow-inference"
|
||||
|
||||
$resultPath = Join-Path $stagingRoot "result.json"
|
||||
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
|
||||
throw "LAB E15 result manifest is missing"
|
||||
}
|
||||
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$result.schema_version -ne "missioncore.e15-shadow-inference-result/v1" -or
|
||||
$result.result_id -notmatch "^e15-shadow-inference-[a-f0-9]{64}$" -or
|
||||
$result.publication_scope -ne "live-shadow-diagnostic-only"
|
||||
) { throw "LAB E15 result manifest is incompatible" }
|
||||
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
||||
if (Test-Path -LiteralPath $finalRoot) { throw "Immutable LAB E15 result already exists" }
|
||||
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
|
||||
Remove-Item -LiteralPath $publishRoot -Force
|
||||
$completed = $true
|
||||
$totalWatch.Stop()
|
||||
$freeFinal = Assert-FreeSpace "post-publication"
|
||||
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
||||
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
||||
Write-Output ("ACCEPTANCE_STATE={0}" -f $result.acceptance_state)
|
||||
Write-Output ("RUNNER_EXIT_CODE={0}" -f $runnerExitCode)
|
||||
Write-Output ("ORCHESTRATOR_WALL_SECONDS={0}" -f $totalWatch.Elapsed.TotalSeconds)
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeFinal)
|
||||
}
|
||||
finally {
|
||||
$shadowToken = $null
|
||||
if (-not $completed -and (Test-Path -LiteralPath $publishRoot)) {
|
||||
Remove-Item -LiteralPath $publishRoot -Recurse -Force
|
||||
}
|
||||
if ($loadedByRun) {
|
||||
try {
|
||||
Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 `
|
||||
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/unload" `
|
||||
-Method Post -ContentType "application/json" -Body "{}" *> $null
|
||||
Write-Output "PHASE=e15-model-state-restored"
|
||||
}
|
||||
catch { Write-Warning "LAB E15 could not restore the prior YOLOX-S state" }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$EvaluationPack,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$E2Prelabels,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ValidFovRoot,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RunnerPath,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$BaselineRunnerPath,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ProfilePath,
|
||||
|
||||
[ValidateRange(0, 64)]
|
||||
[int]$MaxFrames = 0,
|
||||
|
||||
[ValidateRange(1, 1000)]
|
||||
[int]$FreeGiBFloor = 360,
|
||||
|
||||
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
|
||||
|
||||
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode {
|
||||
param([string]$Operation)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-RegularFile {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if (-not $item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
return $item.FullName
|
||||
}
|
||||
throw "$Label is not a regular file"
|
||||
}
|
||||
|
||||
function Assert-Directory {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if ($item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
return $item.FullName
|
||||
}
|
||||
throw "$Label is not a real directory"
|
||||
}
|
||||
|
||||
function Assert-DDrivePath {
|
||||
param([string]$Path, [string]$Label)
|
||||
$fullPath = [IO.Path]::GetFullPath($Path)
|
||||
if ([IO.Path]::GetPathRoot($fullPath).TrimEnd("\") -ine "D:") {
|
||||
throw "$Label must be stored on D:"
|
||||
}
|
||||
return $fullPath
|
||||
}
|
||||
|
||||
function Assert-FreeSpace {
|
||||
param([string]$Phase)
|
||||
$freeBytes = [int64](Get-PSDrive -Name D).Free
|
||||
$floorBytes = [int64]$FreeGiBFloor * 1GB
|
||||
$freeGiB = [math]::Round($freeBytes / 1GB, 3)
|
||||
Write-Output ("DISK_GUARD PHASE={0} DRIVE=D FREE_GIB={1} FLOOR_GIB={2}" -f $Phase, $freeGiB, $FreeGiBFloor)
|
||||
if ($freeBytes -lt $floorBytes) {
|
||||
throw "D: free-space floor was crossed during $Phase"
|
||||
}
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath {
|
||||
param([string]$Path)
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
$packRoot = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $EvaluationPack).Path "Evaluation pack") "Evaluation pack"
|
||||
$prelabelsRoot = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $E2Prelabels).Path "E2 prelabels") "E2 prelabels"
|
||||
$validFov = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $ValidFovRoot).Path "Valid-FOV root") "Valid-FOV root"
|
||||
$runner = Assert-DDrivePath (Assert-RegularFile (Resolve-Path -LiteralPath $RunnerPath).Path "LAB E3 runner") "LAB E3 runner"
|
||||
$baselineRunner = Assert-DDrivePath (Assert-RegularFile (Resolve-Path -LiteralPath $BaselineRunnerPath).Path "Baseline runner") "Baseline runner"
|
||||
$profile = Assert-DDrivePath (Assert-RegularFile (Resolve-Path -LiteralPath $ProfilePath).Path "LAB E3 profile") "LAB E3 profile"
|
||||
$runtime = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $RuntimeRoot).Path "Runtime root") "Runtime root"
|
||||
if (
|
||||
(Split-Path $runner -Parent) -ne (Split-Path $baselineRunner -Parent) -or
|
||||
(Split-Path $runner -Parent) -ne (Split-Path $profile -Parent)
|
||||
) {
|
||||
throw "LAB E3 runner, baseline runner, and profile must share one read-only mount"
|
||||
}
|
||||
|
||||
$pack = Get-Content -LiteralPath (Join-Path $packRoot "manifest.json") -Raw | ConvertFrom-Json
|
||||
$prelabels = Get-Content -LiteralPath (Join-Path $prelabelsRoot "result.json") -Raw | ConvertFrom-Json
|
||||
$profileDocument = Get-Content -LiteralPath $profile -Raw | ConvertFrom-Json
|
||||
$packFrameCount = @($pack.identity.frames).Count
|
||||
if (
|
||||
$pack.schema_version -ne "missioncore.perception-evaluation-pack/v1" -or
|
||||
$pack.generation_id -ne (Split-Path $packRoot -Leaf) -or
|
||||
$pack.identity.source_id -ne "sensor.camera.right" -or
|
||||
$pack.identity.calibration_slot -ne "camera_1" -or
|
||||
$packFrameCount -ne 64 -or
|
||||
$prelabels.schema_version -ne "missioncore.perception-evaluation-prelabels/v1" -or
|
||||
$prelabels.identity.evaluation_pack_id -ne $pack.generation_id -or
|
||||
$profileDocument.schema_version -ne "missioncore.k1-e3-rectified-segmentation-profile/v1" -or
|
||||
$profileDocument.source.source_id -ne $pack.identity.source_id -or
|
||||
$profileDocument.source.calibration_slot -ne $pack.identity.calibration_slot -or
|
||||
$profileDocument.source.calibration_sha256 -ne $pack.identity.calibration_sha256
|
||||
) {
|
||||
throw "LAB E3 inputs are incompatible"
|
||||
}
|
||||
$null = Assert-RegularFile (Join-Path $validFov "manifest.json") "Valid-FOV manifest"
|
||||
$null = Assert-RegularFile (Join-Path $validFov "mask.png") "Valid-FOV mask"
|
||||
|
||||
& docker image inspect $ContainerImage *> $null
|
||||
Assert-LastExitCode "Existing container image inspection"
|
||||
Assert-FreeSpace "preflight"
|
||||
|
||||
$cacheRoot = Join-Path $runtime "cache\perception-e3-models-v1"
|
||||
$derivedRoot = Join-Path $runtime "derived\e3-segmentation"
|
||||
$environmentRoot = Join-Path $runtime "derived\perception-e3-opencv413092-v1"
|
||||
$torchEnvironment = Assert-Directory (Join-Path $runtime "derived\perception-p0-env-v1") "Torch environment"
|
||||
$transformersEnvironment = Assert-Directory (Join-Path $runtime "derived\perception-p0-transformers4576-v1") "Transformers environment"
|
||||
$runnerRoot = Split-Path $runner -Parent
|
||||
$runnerName = Split-Path $runner -Leaf
|
||||
$profileName = Split-Path $profile -Leaf
|
||||
$null = New-Item -ItemType Directory -Path $cacheRoot -Force
|
||||
$null = New-Item -ItemType Directory -Path $derivedRoot -Force
|
||||
|
||||
if (-not (Test-Path -LiteralPath $environmentRoot)) {
|
||||
$environmentToken = [Guid]::NewGuid().ToString("N")
|
||||
$environmentStaging = Join-Path (Split-Path $environmentRoot -Parent) (".e3-environment-{0}.publish" -f $environmentToken)
|
||||
$null = New-Item -ItemType Directory -Path $environmentStaging
|
||||
$null = New-Item -ItemType Directory -Path (Join-Path $environmentStaging "tmp")
|
||||
try {
|
||||
$prepareArgs = @(
|
||||
"run", "--rm", "--network", "bridge", "--read-only",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "512", "--tmpfs", "/tmp:rw,noexec,nosuid,size=1g",
|
||||
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
|
||||
"-e", "HF_HOME=/cache/huggingface",
|
||||
"-e", "PIP_NO_CACHE_DIR=1",
|
||||
"-e", "HOME=/environment/tmp",
|
||||
"-e", "TMPDIR=/environment/tmp",
|
||||
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:rw"),
|
||||
"-v", ((Convert-ToDockerPath $environmentStaging) + ":/environment:rw"),
|
||||
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3",
|
||||
$ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "prepare",
|
||||
"--profile", ("/runner/{0}" -f $profileName),
|
||||
"--cache", "/cache",
|
||||
"--environment", "/environment"
|
||||
)
|
||||
Write-Output "PHASE=e3-dependency-prepare-start"
|
||||
& docker @prepareArgs
|
||||
Assert-LastExitCode "LAB E3 dependency preparation"
|
||||
$null = Assert-RegularFile (Join-Path $environmentStaging "manifest.json") "LAB E3 dependency manifest"
|
||||
Remove-Item -LiteralPath (Join-Path $environmentStaging "tmp") -Recurse -Force
|
||||
Move-Item -LiteralPath $environmentStaging -Destination $environmentRoot
|
||||
Write-Output "PHASE=e3-dependency-prepare-complete"
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $environmentStaging) {
|
||||
Remove-Item -LiteralPath $environmentStaging -Recurse -Force
|
||||
}
|
||||
}
|
||||
}
|
||||
$environment = Assert-Directory $environmentRoot "LAB E3 dependency environment"
|
||||
$null = Assert-RegularFile (Join-Path $environment "manifest.json") "LAB E3 dependency manifest"
|
||||
Assert-FreeSpace "post-dependency-prepare"
|
||||
|
||||
$runToken = [Guid]::NewGuid().ToString("N")
|
||||
$publishRoot = Join-Path $derivedRoot (".{0}-{1}.publish" -f $pack.generation_id, $runToken)
|
||||
$stagingRoot = Join-Path $publishRoot "output"
|
||||
$null = New-Item -ItemType Directory -Path $publishRoot
|
||||
|
||||
try {
|
||||
$runArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", "none", "--read-only",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "512", "--shm-size", "4g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
|
||||
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
|
||||
"-e", "TORCH_HOME=/cache/torch",
|
||||
"-e", "HF_HOME=/cache/huggingface",
|
||||
"-e", "HF_HUB_OFFLINE=1",
|
||||
"-e", "TRANSFORMERS_OFFLINE=1",
|
||||
"-e", "HOME=/tmp",
|
||||
"-v", ((Convert-ToDockerPath $packRoot) + ":/evaluation-pack:ro"),
|
||||
"-v", ((Convert-ToDockerPath $prelabelsRoot) + ":/e2-prelabels:ro"),
|
||||
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
||||
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
|
||||
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"),
|
||||
"-v", ((Convert-ToDockerPath $environment) + ":/environment:ro"),
|
||||
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3",
|
||||
$ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "run",
|
||||
"--profile", ("/runner/{0}" -f $profileName),
|
||||
"--evaluation-pack", "/evaluation-pack",
|
||||
"--e2-prelabels", "/e2-prelabels",
|
||||
"--valid-fov-root", "/valid-fov",
|
||||
"--cache", "/cache",
|
||||
"--environment", "/environment",
|
||||
"--output", "/publish/output",
|
||||
"--telemetry-interval-seconds", "1"
|
||||
)
|
||||
if ($MaxFrames -gt 0) {
|
||||
$runArgs += @("--max-frames", [string]$MaxFrames)
|
||||
}
|
||||
$expectedFrames = if ($MaxFrames -gt 0) { $MaxFrames } else { $packFrameCount }
|
||||
Write-Output ("PHASE=e3-run-start PACK={0} FRAMES={1}" -f $pack.generation_id, $expectedFrames)
|
||||
& docker @runArgs
|
||||
Assert-LastExitCode "LAB E3 segmentation"
|
||||
|
||||
$result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$result.schema_version -ne "missioncore.k1-e3-rectified-segmentation-result/v1" -or
|
||||
$result.result_id -notmatch "^e3-segmentation-[a-f0-9]{64}$" -or
|
||||
$result.identity.evaluation_pack_id -ne $pack.generation_id -or
|
||||
[int]$result.identity.frame_count -ne $expectedFrames -or
|
||||
$result.ground_truth -ne $false
|
||||
) {
|
||||
throw "LAB E3 result manifest is incompatible"
|
||||
}
|
||||
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
||||
if (Test-Path -LiteralPath $finalRoot) {
|
||||
throw "An immutable LAB E3 result with the same identity already exists: $finalRoot"
|
||||
}
|
||||
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
|
||||
Remove-Item -LiteralPath $publishRoot -Force
|
||||
Assert-FreeSpace "post-run"
|
||||
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
||||
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $publishRoot) {
|
||||
Remove-Item -LiteralPath $publishRoot -Recurse -Force
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,421 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$JobRoot,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RunnerPath,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ProfilePath,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ValidFovRoot,
|
||||
|
||||
[ValidateRange(0, 256)]
|
||||
[int]$PilotFrames = 0,
|
||||
|
||||
[ValidateRange(1, 1000)]
|
||||
[int]$FreeGiBFloor = 360,
|
||||
|
||||
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
|
||||
|
||||
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode {
|
||||
param([string]$Operation)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-RegularFile {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if (-not $item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
return $item.FullName
|
||||
}
|
||||
throw "$Label is not a regular file"
|
||||
}
|
||||
|
||||
function Assert-Directory {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if ($item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
return $item.FullName
|
||||
}
|
||||
throw "$Label is not a real directory"
|
||||
}
|
||||
|
||||
function Assert-DDrivePath {
|
||||
param([string]$Path, [string]$Label)
|
||||
$fullPath = [IO.Path]::GetFullPath($Path)
|
||||
if ([IO.Path]::GetPathRoot($fullPath).TrimEnd("\") -ine "D:") {
|
||||
throw "$Label must be stored on D:"
|
||||
}
|
||||
return $fullPath
|
||||
}
|
||||
|
||||
function Get-DFreeBytes {
|
||||
return [int64](Get-PSDrive -Name D).Free
|
||||
}
|
||||
|
||||
function Assert-FreeSpace {
|
||||
param(
|
||||
[string]$Phase,
|
||||
[int64]$RequiredAdditionalBytes = 0
|
||||
)
|
||||
$freeBytes = Get-DFreeBytes
|
||||
$floorBytes = [int64]$FreeGiBFloor * 1GB
|
||||
$requiredBytes = $floorBytes + $RequiredAdditionalBytes
|
||||
Write-Host (
|
||||
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3} REQUIRED_ADDITIONAL_GIB={4}" -f
|
||||
$Phase,
|
||||
$freeBytes,
|
||||
[math]::Round($freeBytes / 1GB, 3),
|
||||
$FreeGiBFloor,
|
||||
[math]::Round($RequiredAdditionalBytes / 1GB, 3)
|
||||
)
|
||||
if ($freeBytes -lt $requiredBytes) {
|
||||
throw "D: does not have the guarded LAB E4 working-set reserve during $Phase"
|
||||
}
|
||||
return $freeBytes
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath {
|
||||
param([string]$Path)
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
$jobDirectory = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $JobRoot).Path "Job root") "Job root"
|
||||
$runner = Assert-DDrivePath (Assert-RegularFile (Resolve-Path -LiteralPath $RunnerPath).Path "LAB E4 runner") "LAB E4 runner"
|
||||
$profile = Assert-DDrivePath (Assert-RegularFile (Resolve-Path -LiteralPath $ProfilePath).Path "LAB E4 profile") "LAB E4 profile"
|
||||
$validFov = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $ValidFovRoot).Path "Valid-FOV root") "Valid-FOV root"
|
||||
$runtime = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $RuntimeRoot).Path "Runtime root") "Runtime root"
|
||||
$runnerRoot = Split-Path $runner -Parent
|
||||
if ((Split-Path $profile -Parent) -ne $runnerRoot) {
|
||||
throw "LAB E4 runner and profile must share one read-only mount"
|
||||
}
|
||||
foreach ($dependency in @(
|
||||
"run_recorded_perception_epoch.py",
|
||||
"run_e3_rectified_segmentation.py",
|
||||
"run_evaluation_prelabels.py"
|
||||
)) {
|
||||
$null = Assert-RegularFile (Join-Path $runnerRoot $dependency) "LAB E4 runner dependency"
|
||||
}
|
||||
|
||||
$job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json
|
||||
if ($job.schema_version -ne "missioncore.compute-job/v1" -or $job.job_id -ne (Split-Path $jobDirectory -Leaf)) {
|
||||
throw "Compute job manifest is incompatible"
|
||||
}
|
||||
$sourceId = [string]$job.input.source_id
|
||||
$epoch = [int]$job.input.codec_epoch
|
||||
$fullFrameCount = [int]$job.input.segment_count
|
||||
$activeFrameCount = if ($PilotFrames -gt 0) { $PilotFrames } else { $fullFrameCount }
|
||||
$timelineStart = [double]$job.input.timeline.start_seconds
|
||||
$timelineEnd = [double]$job.input.timeline.end_seconds
|
||||
$timelineDuration = $timelineEnd - $timelineStart
|
||||
if (
|
||||
$sourceId -ne "sensor.camera.right" -or
|
||||
$fullFrameCount -lt 1 -or
|
||||
$activeFrameCount -gt $fullFrameCount -or
|
||||
$timelineDuration -le 0
|
||||
) {
|
||||
throw "LAB E4 camera job contract is invalid"
|
||||
}
|
||||
Write-Output ("PHASE=job-manifest-validated JOB={0} FRAMES={1}/{2}" -f $job.job_id, $activeFrameCount, $fullFrameCount)
|
||||
|
||||
$epochRoot = Join-Path $jobDirectory ("input\camera\{0}\epoch-{1}" -f $sourceId, $epoch)
|
||||
$epochRoot = Assert-DDrivePath (Assert-Directory $epochRoot "Camera epoch") "Camera epoch"
|
||||
$initPath = Assert-RegularFile (Join-Path $epochRoot "init.mp4") "Camera init"
|
||||
$segmentsRoot = Assert-Directory (Join-Path $epochRoot "segments") "Camera segments"
|
||||
$derivedRoot = Join-Path $runtime "derived"
|
||||
$tmpRoot = Join-Path $runtime "tmp"
|
||||
$cacheRoot = Assert-DDrivePath (Assert-Directory (Join-Path $runtime "cache\perception-e3-models-v1") "EoMT cache") "EoMT cache"
|
||||
$e3Environment = Assert-DDrivePath (Assert-Directory (Join-Path $derivedRoot "perception-e3-opencv413092-v1") "E3 dependency environment") "E3 dependency environment"
|
||||
$torchEnvironment = Assert-DDrivePath (Assert-Directory (Join-Path $derivedRoot "perception-p0-env-v1") "Torch environment") "Torch environment"
|
||||
$transformersEnvironment = Assert-DDrivePath (Assert-Directory (Join-Path $derivedRoot "perception-p0-transformers4576-v1") "Transformers environment") "Transformers environment"
|
||||
$null = New-Item -ItemType Directory -Path $derivedRoot -Force
|
||||
$null = New-Item -ItemType Directory -Path $tmpRoot -Force
|
||||
|
||||
# Conservative upper bound: decoded RGB + RGB overlay + one-byte semantic mask,
|
||||
# plus 20% filesystem/encoding overhead and the reconstructed source stream.
|
||||
$pixelWorkingSet = [int64]$activeFrameCount * 800 * 600 * 7
|
||||
$workingSetReserve = [int64][math]::Ceiling(($pixelWorkingSet * 1.2) + [int64]$job.input.byte_length)
|
||||
$freeBytesBefore = Assert-FreeSpace "preflight" $workingSetReserve
|
||||
|
||||
& docker image inspect $ContainerImage *> $null
|
||||
Assert-LastExitCode "Existing container image inspection"
|
||||
$runnerName = Split-Path $runner -Leaf
|
||||
$profileName = Split-Path $profile -Leaf
|
||||
$orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$preflightArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", "none", "--read-only",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "512", "--shm-size", "4g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
|
||||
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
|
||||
"-e", "TORCH_HOME=/cache/torch",
|
||||
"-e", "HF_HOME=/cache/huggingface",
|
||||
"-e", "HF_HUB_OFFLINE=1",
|
||||
"-e", "TRANSFORMERS_OFFLINE=1",
|
||||
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
||||
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
||||
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"),
|
||||
"-v", ((Convert-ToDockerPath $e3Environment) + ":/environment:ro"),
|
||||
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3",
|
||||
$ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "preflight",
|
||||
"--job", "/job",
|
||||
"--profile", ("/runner/{0}" -f $profileName),
|
||||
"--valid-fov-root", "/valid-fov",
|
||||
"--cache", "/cache",
|
||||
"--environment", "/environment"
|
||||
)
|
||||
Write-Output "PHASE=e4-preflight-start"
|
||||
& docker @preflightArgs
|
||||
Assert-LastExitCode "LAB E4 preflight"
|
||||
Write-Output "PHASE=e4-preflight-complete"
|
||||
|
||||
$runToken = [Guid]::NewGuid().ToString("N")
|
||||
$workRoot = Join-Path $tmpRoot ("{0}-e4-{1}" -f $job.job_id, $runToken)
|
||||
$framesRoot = Join-Path $workRoot "frames"
|
||||
$streamPath = Join-Path $workRoot "camera.mp4"
|
||||
$ptsPath = Join-Path $workRoot "pts.json"
|
||||
$timelinePath = Join-Path $workRoot "timeline.jsonl"
|
||||
$publishRoot = Join-Path $derivedRoot (".{0}-e4-{1}.publish" -f $job.job_id, $runToken)
|
||||
$stagingRoot = Join-Path $publishRoot "output"
|
||||
$null = New-Item -ItemType Directory -Path $framesRoot
|
||||
$null = New-Item -ItemType Directory -Path $publishRoot
|
||||
$totalWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
$completed = $false
|
||||
Write-Output "PHASE=e4-private-staging-created"
|
||||
|
||||
try {
|
||||
$stream = [IO.File]::Open($streamPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
||||
try {
|
||||
$input = [IO.File]::OpenRead($initPath)
|
||||
try {
|
||||
$input.CopyTo($stream)
|
||||
}
|
||||
finally {
|
||||
$input.Dispose()
|
||||
}
|
||||
for ($sequence = 1; $sequence -le $activeFrameCount; $sequence++) {
|
||||
$path = Assert-RegularFile (Join-Path $segmentsRoot ("{0}.m4s" -f $sequence)) "Camera segment"
|
||||
$input = [IO.File]::OpenRead($path)
|
||||
try {
|
||||
$input.CopyTo($stream)
|
||||
}
|
||||
finally {
|
||||
$input.Dispose()
|
||||
}
|
||||
if ($sequence % 500 -eq 0 -or $sequence -eq $activeFrameCount) {
|
||||
Write-Output ("PHASE=e4-stream-reconstruction SEGMENTS={0}/{1}" -f $sequence, $activeFrameCount)
|
||||
}
|
||||
}
|
||||
$stream.Flush($true)
|
||||
}
|
||||
finally {
|
||||
$stream.Dispose()
|
||||
}
|
||||
$null = Assert-FreeSpace "post-stream-reconstruction"
|
||||
|
||||
$extractWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
Write-Output "PHASE=e4-frame-extraction-start"
|
||||
& ffmpeg -hide_banner -loglevel fatal -i $streamPath -map 0:v:0 -fps_mode passthrough -frames:v $activeFrameCount (Join-Path $framesRoot "frame-%06d.png")
|
||||
Assert-LastExitCode "LAB E4 camera extraction"
|
||||
& ffprobe -v error -select_streams v:0 -show_entries frame=best_effort_timestamp_time -of json $streamPath | Set-Content -LiteralPath $ptsPath -Encoding utf8
|
||||
Assert-LastExitCode "LAB E4 camera timestamp probe"
|
||||
|
||||
$decodedFrames = @(Get-ChildItem -LiteralPath $framesRoot -File -Filter "frame-*.png")
|
||||
$ptsDocument = Get-Content -LiteralPath $ptsPath -Raw | ConvertFrom-Json
|
||||
$pts = @($ptsDocument.frames)
|
||||
if ($decodedFrames.Count -ne $activeFrameCount -or $pts.Count -lt $activeFrameCount) {
|
||||
throw "Decoded LAB E4 frame count differs from the requested camera epoch"
|
||||
}
|
||||
$firstEpochSeconds = [double]::Parse(
|
||||
([string]$pts[0].best_effort_timestamp_time).Trim(),
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
)
|
||||
$previousEpochSeconds = -1.0
|
||||
$timelineWriter = [IO.StreamWriter]::new($timelinePath, $false, [Text.UTF8Encoding]::new($false))
|
||||
try {
|
||||
for ($index = 0; $index -lt $activeFrameCount; $index++) {
|
||||
$epochSeconds = [double]::Parse(
|
||||
([string]$pts[$index].best_effort_timestamp_time).Trim(),
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
) - $firstEpochSeconds
|
||||
if ($epochSeconds -le $previousEpochSeconds -or $epochSeconds -gt ($timelineDuration + 0.001)) {
|
||||
throw "Decoded LAB E4 timestamps are not strictly monotonic inside the camera timeline"
|
||||
}
|
||||
$row = [ordered]@{
|
||||
frame_index = $index
|
||||
epoch_seconds = $epochSeconds
|
||||
session_seconds = $timelineStart + $epochSeconds
|
||||
}
|
||||
$timelineWriter.WriteLine(($row | ConvertTo-Json -Compress))
|
||||
$previousEpochSeconds = $epochSeconds
|
||||
}
|
||||
$timelineWriter.Flush()
|
||||
}
|
||||
finally {
|
||||
$timelineWriter.Dispose()
|
||||
}
|
||||
$extractWatch.Stop()
|
||||
$freeBytesPostExtract = Assert-FreeSpace "post-frame-extraction"
|
||||
Write-Output ("PHASE=e4-frame-extraction-complete FRAMES={0}" -f $activeFrameCount)
|
||||
|
||||
$runArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", "none", "--read-only",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "512", "--shm-size", "4g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
|
||||
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
|
||||
"-e", "TORCH_HOME=/cache/torch",
|
||||
"-e", "HF_HOME=/cache/huggingface",
|
||||
"-e", "HF_HUB_OFFLINE=1",
|
||||
"-e", "TRANSFORMERS_OFFLINE=1",
|
||||
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
||||
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
||||
"-v", ((Convert-ToDockerPath $framesRoot) + ":/frames:ro"),
|
||||
"-v", ((Convert-ToDockerPath $workRoot) + ":/work:ro"),
|
||||
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
|
||||
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"),
|
||||
"-v", ((Convert-ToDockerPath $e3Environment) + ":/environment:ro"),
|
||||
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3",
|
||||
$ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "run",
|
||||
"--job", "/job",
|
||||
"--profile", ("/runner/{0}" -f $profileName),
|
||||
"--valid-fov-root", "/valid-fov",
|
||||
"--frames", "/frames",
|
||||
"--timeline", "/work/timeline.jsonl",
|
||||
"--cache", "/cache",
|
||||
"--environment", "/environment",
|
||||
"--output", "/publish/output",
|
||||
"--frame-limit", [string]$activeFrameCount,
|
||||
"--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB),
|
||||
"--orchestrator-sha256", $orchestratorSha256,
|
||||
"--container-image", $ContainerImage,
|
||||
"--telemetry-interval-seconds", "1"
|
||||
)
|
||||
Write-Output ("PHASE=e4-inference-start FRAMES={0}" -f $activeFrameCount)
|
||||
& docker @runArgs
|
||||
Assert-LastExitCode "LAB E4 semantic inference"
|
||||
$freeBytesPostInference = Assert-FreeSpace "post-inference"
|
||||
Write-Output "PHASE=e4-inference-complete"
|
||||
|
||||
if ($PilotFrames -gt 0) {
|
||||
$pilotParent = Join-Path $derivedRoot "e4-pilots"
|
||||
$null = New-Item -ItemType Directory -Path $pilotParent -Force
|
||||
$pilotRoot = Join-Path $pilotParent ("pilot-{0}-{1}" -f $activeFrameCount, $runToken)
|
||||
Move-Item -LiteralPath $stagingRoot -Destination $pilotRoot
|
||||
Remove-Item -LiteralPath $publishRoot -Force
|
||||
$completed = $true
|
||||
Write-Output ("PILOT_ROOT={0}" -f $pilotRoot)
|
||||
Write-Output ("PILOT_FRAMES={0}" -f $activeFrameCount)
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_EXTRACT={0}" -f $freeBytesPostExtract)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_INFERENCE={0}" -f $freeBytesPostInference)
|
||||
return
|
||||
}
|
||||
|
||||
$videoPath = Join-Path $stagingRoot "perception.mp4"
|
||||
$encodeWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
$averageFps = $fullFrameCount / $timelineDuration
|
||||
$fpsText = $averageFps.ToString("0.#########", [Globalization.CultureInfo]::InvariantCulture)
|
||||
$durationText = $timelineDuration.ToString("0.#########", [Globalization.CultureInfo]::InvariantCulture)
|
||||
& ffmpeg -hide_banner -loglevel error -framerate $fpsText -i (Join-Path $stagingRoot "overlay-frames\frame-%06d.png") -t $durationText -c:v h264_nvenc -preset p4 -tune hq -rc vbr -cq 21 -b:v 0 -pix_fmt yuv420p -movflags +faststart $videoPath
|
||||
Assert-LastExitCode "LAB E4 video encoding"
|
||||
$encodeWatch.Stop()
|
||||
$null = Assert-FreeSpace "post-video-encoding"
|
||||
Write-Output "PHASE=e4-video-encoding-complete"
|
||||
|
||||
$masksPath = Join-Path $stagingRoot "masks.tar.gz"
|
||||
$archiveWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
& tar.exe -czf $masksPath -C $stagingRoot semantic-masks
|
||||
Assert-LastExitCode "LAB E4 mask archive publication"
|
||||
$archiveWatch.Stop()
|
||||
$freeBytesPostArtifacts = Assert-FreeSpace "post-mask-archive"
|
||||
Write-Output "PHASE=e4-mask-archive-complete"
|
||||
|
||||
$totalWatch.Stop()
|
||||
$finalizeArgs = @(
|
||||
"run", "--rm", "--network", "none", "--read-only",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "128", "--tmpfs", "/tmp:rw,noexec,nosuid,size=256m",
|
||||
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
|
||||
"-v", ((Convert-ToDockerPath $stagingRoot) + ":/output:rw"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3",
|
||||
$ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "finalize",
|
||||
"--output", "/output",
|
||||
"--video", "/output/perception.mp4",
|
||||
"--masks", "/output/masks.tar.gz",
|
||||
"--extract-seconds", $extractWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
|
||||
"--encode-seconds", $encodeWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
|
||||
"--archive-seconds", $archiveWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
|
||||
"--wall-seconds", $totalWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
|
||||
"--encoder", "ffmpeg-h264_nvenc-p4-cq21-yuv420p-faststart",
|
||||
"--disk-free-before-bytes", [string]$freeBytesBefore,
|
||||
"--disk-free-post-extract-bytes", [string]$freeBytesPostExtract,
|
||||
"--disk-free-post-inference-bytes", [string]$freeBytesPostInference,
|
||||
"--disk-free-post-artifacts-bytes", [string]$freeBytesPostArtifacts,
|
||||
"--disk-floor-bytes", [string]([int64]$FreeGiBFloor * 1GB),
|
||||
"--working-set-reserve-bytes", [string]$workingSetReserve
|
||||
)
|
||||
& docker @finalizeArgs
|
||||
Assert-LastExitCode "LAB E4 result finalization"
|
||||
|
||||
$result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$result.schema_version -ne "missioncore.recorded-perception-result/v2" -or
|
||||
$result.result_id -notmatch "^result-[a-f0-9]{64}$" -or
|
||||
[int]$result.frames_processed -ne $fullFrameCount -or
|
||||
$result.ground_truth -ne $false
|
||||
) {
|
||||
throw "Final LAB E4 result manifest is incompatible"
|
||||
}
|
||||
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
||||
if (Test-Path -LiteralPath $finalRoot) {
|
||||
throw "An immutable result with the same LAB E4 identity already exists: $finalRoot"
|
||||
}
|
||||
|
||||
foreach ($temporaryChild in @("overlay-frames", "semantic-masks")) {
|
||||
$temporaryPath = Join-Path $stagingRoot $temporaryChild
|
||||
if (Test-Path -LiteralPath $temporaryPath) {
|
||||
Remove-Item -LiteralPath $temporaryPath -Recurse -Force
|
||||
}
|
||||
}
|
||||
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
|
||||
Remove-Item -LiteralPath $publishRoot -Force
|
||||
$completed = $true
|
||||
$freeBytesFinal = Assert-FreeSpace "post-publication"
|
||||
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
||||
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_EXTRACT={0}" -f $freeBytesPostExtract)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_INFERENCE={0}" -f $freeBytesPostInference)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_ARTIFACTS={0}" -f $freeBytesPostArtifacts)
|
||||
Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeBytesFinal)
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $workRoot) {
|
||||
Remove-Item -LiteralPath $workRoot -Recurse -Force
|
||||
}
|
||||
if (-not $completed -and (Test-Path -LiteralPath $publishRoot)) {
|
||||
Remove-Item -LiteralPath $publishRoot -Recurse -Force
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,544 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$JobRoot,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RunnerPath,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ProfilePath,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ValidFovRoot,
|
||||
|
||||
[ValidateRange(0, 1000000)]
|
||||
[int]$StartFrame = 1000,
|
||||
|
||||
[ValidateRange(0, 1000000)]
|
||||
[int]$EndFrame = 1600,
|
||||
|
||||
[ValidateRange(1, 1000)]
|
||||
[int]$FreeGiBFloor = 360,
|
||||
|
||||
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
|
||||
|
||||
[string]$ModelRoot = "D:\NDC_MISSIONCORE\runtime\models\yolox_s",
|
||||
|
||||
[string]$TritonContainer = "mission-core-triton",
|
||||
|
||||
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode {
|
||||
param([string]$Operation)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-RegularFile {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if (-not $item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
return $item.FullName
|
||||
}
|
||||
throw "$Label is not a regular file"
|
||||
}
|
||||
|
||||
function Assert-Directory {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if ($item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
return $item.FullName
|
||||
}
|
||||
throw "$Label is not a real directory"
|
||||
}
|
||||
|
||||
function Assert-DDrivePath {
|
||||
param([string]$Path, [string]$Label)
|
||||
$fullPath = [IO.Path]::GetFullPath($Path)
|
||||
if ([IO.Path]::GetPathRoot($fullPath).TrimEnd("\") -ine "D:") {
|
||||
throw "$Label must be stored on D:"
|
||||
}
|
||||
return $fullPath
|
||||
}
|
||||
|
||||
function Get-DFreeBytes {
|
||||
return [int64](Get-PSDrive -Name D).Free
|
||||
}
|
||||
|
||||
function Assert-FreeSpace {
|
||||
param(
|
||||
[string]$Phase,
|
||||
[int64]$RequiredAdditionalBytes = 0
|
||||
)
|
||||
$freeBytes = Get-DFreeBytes
|
||||
$floorBytes = [int64]$FreeGiBFloor * 1GB
|
||||
Write-Host (
|
||||
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3} REQUIRED_ADDITIONAL_GIB={4}" -f
|
||||
$Phase,
|
||||
$freeBytes,
|
||||
[math]::Round($freeBytes / 1GB, 3),
|
||||
$FreeGiBFloor,
|
||||
[math]::Round($RequiredAdditionalBytes / 1GB, 3)
|
||||
)
|
||||
if ($freeBytes -lt ($floorBytes + $RequiredAdditionalBytes)) {
|
||||
throw "D: does not have the guarded LAB E5 working-set reserve during $Phase"
|
||||
}
|
||||
return $freeBytes
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath {
|
||||
param([string]$Path)
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
function Test-TritonModelReady {
|
||||
param([string]$ModelName)
|
||||
try {
|
||||
$response = Invoke-WebRequest `
|
||||
-Uri ("http://127.0.0.1:8000/v2/models/{0}/ready" -f $ModelName) `
|
||||
-Method Get `
|
||||
-UseBasicParsing `
|
||||
-TimeoutSec 10
|
||||
return $response.StatusCode -eq 200
|
||||
}
|
||||
catch {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
||||
$jobDirectory = Assert-DDrivePath (
|
||||
Assert-Directory (Resolve-Path -LiteralPath $JobRoot).Path "Job root"
|
||||
) "Job root"
|
||||
$runner = Assert-DDrivePath (
|
||||
Assert-RegularFile (Resolve-Path -LiteralPath $RunnerPath).Path "LAB E5 runner"
|
||||
) "LAB E5 runner"
|
||||
$profile = Assert-DDrivePath (
|
||||
Assert-RegularFile (Resolve-Path -LiteralPath $ProfilePath).Path "LAB E5 profile"
|
||||
) "LAB E5 profile"
|
||||
$validFov = Assert-DDrivePath (
|
||||
Assert-Directory (Resolve-Path -LiteralPath $ValidFovRoot).Path "Valid-FOV root"
|
||||
) "Valid-FOV root"
|
||||
$runtime = Assert-DDrivePath (
|
||||
Assert-Directory (Resolve-Path -LiteralPath $RuntimeRoot).Path "Runtime root"
|
||||
) "Runtime root"
|
||||
$model = Assert-DDrivePath (
|
||||
Assert-Directory (Resolve-Path -LiteralPath $ModelRoot).Path "YOLOX model root"
|
||||
) "YOLOX model root"
|
||||
$runnerRoot = Split-Path $runner -Parent
|
||||
if ((Split-Path $profile -Parent) -ne $runnerRoot) {
|
||||
throw "LAB E5 runner and profile must share one read-only mount"
|
||||
}
|
||||
foreach ($dependency in @(
|
||||
"run_recorded_perception_epoch.py",
|
||||
"run_e3_rectified_segmentation.py",
|
||||
"run_evaluation_prelabels.py"
|
||||
)) {
|
||||
$null = Assert-RegularFile (Join-Path $runnerRoot $dependency) "LAB E5 runner dependency"
|
||||
}
|
||||
|
||||
$job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json
|
||||
if ($job.schema_version -ne "missioncore.compute-job/v1" -or $job.job_id -ne (Split-Path $jobDirectory -Leaf)) {
|
||||
throw "Compute job manifest is incompatible"
|
||||
}
|
||||
$sourceId = [string]$job.input.source_id
|
||||
$epoch = [int]$job.input.codec_epoch
|
||||
$fullFrameCount = [int]$job.input.segment_count
|
||||
$clipFrameCount = $EndFrame - $StartFrame + 1
|
||||
$timelineStart = [double]$job.input.timeline.start_seconds
|
||||
if (
|
||||
$sourceId -ne "sensor.camera.right" -or
|
||||
$StartFrame -lt 0 -or
|
||||
$EndFrame -lt $StartFrame -or
|
||||
$EndFrame -ge $fullFrameCount -or
|
||||
$clipFrameCount -lt 2
|
||||
) {
|
||||
throw "LAB E5 clip escapes the camera job"
|
||||
}
|
||||
Write-Output (
|
||||
"PHASE=job-manifest-validated JOB={0} CLIP={1}-{2} FRAMES={3}" -f
|
||||
$job.job_id, $StartFrame, $EndFrame, $clipFrameCount
|
||||
)
|
||||
|
||||
$epochRoot = Join-Path $jobDirectory ("input\camera\{0}\epoch-{1}" -f $sourceId, $epoch)
|
||||
$epochRoot = Assert-DDrivePath (Assert-Directory $epochRoot "Camera epoch") "Camera epoch"
|
||||
$initPath = Assert-RegularFile (Join-Path $epochRoot "init.mp4") "Camera init"
|
||||
$segmentsRoot = Assert-Directory (Join-Path $epochRoot "segments") "Camera segments"
|
||||
$derivedRoot = Join-Path $runtime "derived"
|
||||
$tmpRoot = Join-Path $runtime "tmp"
|
||||
$pythonEnvironment = Assert-DDrivePath (
|
||||
Assert-Directory (Join-Path $derivedRoot "perception-p0-env-v1") "Python environment"
|
||||
) "Python environment"
|
||||
$null = New-Item -ItemType Directory -Path $derivedRoot -Force
|
||||
$null = New-Item -ItemType Directory -Path $tmpRoot -Force
|
||||
|
||||
# Conservative bound: partial reconstructed stream, decoded RGB PNG payload,
|
||||
# RGB overlays, encoded result and 2 GiB of filesystem/codec overhead.
|
||||
$pixelWorkingSet = [int64]$clipFrameCount * 800 * 600 * 6
|
||||
$partialStreamReserve = [int64][math]::Ceiling(
|
||||
([int64]$job.input.byte_length * ($EndFrame + 1) / $fullFrameCount) * 1.2
|
||||
)
|
||||
$workingSetReserve = [int64][math]::Ceiling($pixelWorkingSet * 1.4) + $partialStreamReserve + 2GB
|
||||
$freeBytesBefore = Assert-FreeSpace "preflight" $workingSetReserve
|
||||
|
||||
& docker image inspect $ContainerImage *> $null
|
||||
Assert-LastExitCode "Existing container image inspection"
|
||||
$tritonState = docker inspect --format "{{.State.Running}}" $TritonContainer
|
||||
Assert-LastExitCode "Triton container inspection"
|
||||
if ($tritonState.Trim().ToLowerInvariant() -ne "true") {
|
||||
throw "LAB E5 requires the existing Triton container"
|
||||
}
|
||||
|
||||
$runnerName = Split-Path $runner -Leaf
|
||||
$profileName = Split-Path $profile -Leaf
|
||||
$orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$preflightArgs = @(
|
||||
"run", "--rm", "--network", "none", "--read-only",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "128", "--tmpfs", "/tmp:rw,noexec,nosuid,size=256m",
|
||||
"-e", "PYTHONPATH=/runner:/opt/env",
|
||||
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
||||
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
||||
"-v", ((Convert-ToDockerPath $model) + ":/model:ro"),
|
||||
"-v", ((Convert-ToDockerPath $pythonEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3",
|
||||
$ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "preflight",
|
||||
"--job", "/job",
|
||||
"--profile", ("/runner/{0}" -f $profileName),
|
||||
"--valid-fov-root", "/valid-fov",
|
||||
"--model-root", "/model"
|
||||
)
|
||||
Write-Output "PHASE=e5-preflight-start"
|
||||
& docker @preflightArgs
|
||||
Assert-LastExitCode "LAB E5 preflight"
|
||||
Write-Output "PHASE=e5-preflight-complete"
|
||||
|
||||
$modelWasReady = Test-TritonModelReady "yolox_s"
|
||||
$loadedByRun = $false
|
||||
if (-not $modelWasReady) {
|
||||
Invoke-WebRequest `
|
||||
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/load" `
|
||||
-Method Post `
|
||||
-ContentType "application/json" `
|
||||
-Body "{}" `
|
||||
-UseBasicParsing `
|
||||
-TimeoutSec 60 *> $null
|
||||
$loadedByRun = $true
|
||||
$deadline = [DateTime]::UtcNow.AddSeconds(60)
|
||||
while (-not (Test-TritonModelReady "yolox_s")) {
|
||||
if ([DateTime]::UtcNow -ge $deadline) {
|
||||
throw "YOLOX-S did not become ready in Triton"
|
||||
}
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
}
|
||||
Write-Output ("PHASE=e5-model-ready LOADED_BY_RUN={0}" -f $loadedByRun)
|
||||
|
||||
$runToken = [Guid]::NewGuid().ToString("N")
|
||||
$workRoot = Join-Path $tmpRoot ("{0}-e5-{1}" -f $job.job_id, $runToken)
|
||||
$framesRoot = Join-Path $workRoot "frames"
|
||||
$streamPath = Join-Path $workRoot "camera.mp4"
|
||||
$ptsPath = Join-Path $workRoot "pts.json"
|
||||
$timelinePath = Join-Path $workRoot "timeline.jsonl"
|
||||
$publishRoot = Join-Path $derivedRoot (".{0}-e5-{1}.publish" -f $job.job_id, $runToken)
|
||||
$stagingRoot = Join-Path $publishRoot "output"
|
||||
$null = New-Item -ItemType Directory -Path $framesRoot
|
||||
$null = New-Item -ItemType Directory -Path $publishRoot
|
||||
$totalWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
$completed = $false
|
||||
Write-Output "PHASE=e5-private-staging-created"
|
||||
|
||||
try {
|
||||
$stream = [IO.File]::Open(
|
||||
$streamPath,
|
||||
[IO.FileMode]::CreateNew,
|
||||
[IO.FileAccess]::Write,
|
||||
[IO.FileShare]::None
|
||||
)
|
||||
try {
|
||||
$input = [IO.File]::OpenRead($initPath)
|
||||
try {
|
||||
$input.CopyTo($stream)
|
||||
}
|
||||
finally {
|
||||
$input.Dispose()
|
||||
}
|
||||
for ($sequence = 1; $sequence -le ($EndFrame + 1); $sequence++) {
|
||||
$path = Assert-RegularFile (
|
||||
Join-Path $segmentsRoot ("{0}.m4s" -f $sequence)
|
||||
) "Camera segment"
|
||||
$input = [IO.File]::OpenRead($path)
|
||||
try {
|
||||
$input.CopyTo($stream)
|
||||
}
|
||||
finally {
|
||||
$input.Dispose()
|
||||
}
|
||||
if ($sequence % 500 -eq 0 -or $sequence -eq ($EndFrame + 1)) {
|
||||
Write-Output (
|
||||
"PHASE=e5-stream-reconstruction SEGMENTS={0}/{1}" -f
|
||||
$sequence, ($EndFrame + 1)
|
||||
)
|
||||
}
|
||||
}
|
||||
$stream.Flush($true)
|
||||
}
|
||||
finally {
|
||||
$stream.Dispose()
|
||||
}
|
||||
$null = Assert-FreeSpace "post-stream-reconstruction"
|
||||
|
||||
$extractWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
Write-Output "PHASE=e5-frame-extraction-start"
|
||||
$selectFilter = "select='between(n,{0},{1})'" -f $StartFrame, $EndFrame
|
||||
& ffmpeg `
|
||||
-hide_banner -loglevel fatal `
|
||||
-i $streamPath `
|
||||
-map 0:v:0 `
|
||||
-vf $selectFilter `
|
||||
-fps_mode passthrough `
|
||||
(Join-Path $framesRoot "frame-%06d.png")
|
||||
Assert-LastExitCode "LAB E5 camera clip extraction"
|
||||
& ffprobe `
|
||||
-v error `
|
||||
-select_streams v:0 `
|
||||
-show_entries frame=best_effort_timestamp_time `
|
||||
-of json `
|
||||
$streamPath | Set-Content -LiteralPath $ptsPath -Encoding utf8
|
||||
Assert-LastExitCode "LAB E5 camera timestamp probe"
|
||||
|
||||
$decodedFrames = @(Get-ChildItem -LiteralPath $framesRoot -File -Filter "frame-*.png")
|
||||
$ptsDocument = Get-Content -LiteralPath $ptsPath -Raw | ConvertFrom-Json
|
||||
$pts = @($ptsDocument.frames)
|
||||
if ($decodedFrames.Count -ne $clipFrameCount -or $pts.Count -ne ($EndFrame + 1)) {
|
||||
throw "Decoded LAB E5 frame/timestamp count differs from the selected clip"
|
||||
}
|
||||
$firstEpochSeconds = [double]::Parse(
|
||||
([string]$pts[0].best_effort_timestamp_time).Trim(),
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
)
|
||||
$previousSessionSeconds = -1.0
|
||||
$timelineWriter = [IO.StreamWriter]::new(
|
||||
$timelinePath,
|
||||
$false,
|
||||
[Text.UTF8Encoding]::new($false)
|
||||
)
|
||||
try {
|
||||
for ($localIndex = 0; $localIndex -lt $clipFrameCount; $localIndex++) {
|
||||
$sourceIndex = $StartFrame + $localIndex
|
||||
$epochSeconds = [double]::Parse(
|
||||
([string]$pts[$sourceIndex].best_effort_timestamp_time).Trim(),
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
) - $firstEpochSeconds
|
||||
$sessionSeconds = $timelineStart + $epochSeconds
|
||||
if ($sessionSeconds -le $previousSessionSeconds) {
|
||||
throw "Decoded LAB E5 clip timestamps are not strictly monotonic"
|
||||
}
|
||||
$row = [ordered]@{
|
||||
frame_index = $localIndex
|
||||
sequence = $localIndex + 1
|
||||
source_frame_index = $sourceIndex
|
||||
source_sequence = $sourceIndex + 1
|
||||
epoch_seconds = $epochSeconds
|
||||
session_seconds = $sessionSeconds
|
||||
}
|
||||
$timelineWriter.WriteLine(($row | ConvertTo-Json -Compress))
|
||||
$previousSessionSeconds = $sessionSeconds
|
||||
}
|
||||
$timelineWriter.Flush()
|
||||
}
|
||||
finally {
|
||||
$timelineWriter.Dispose()
|
||||
}
|
||||
$extractWatch.Stop()
|
||||
$freeBytesPostExtract = Assert-FreeSpace "post-frame-extraction"
|
||||
Write-Output ("PHASE=e5-frame-extraction-complete FRAMES={0}" -f $clipFrameCount)
|
||||
|
||||
$runArgs = @(
|
||||
"run", "--rm", "--gpus", "all",
|
||||
"--network", ("container:{0}" -f $TritonContainer),
|
||||
"--read-only",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "256", "--shm-size", "1g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=512m",
|
||||
"-e", "PYTHONPATH=/runner:/opt/env",
|
||||
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
||||
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
||||
"-v", ((Convert-ToDockerPath $model) + ":/model:ro"),
|
||||
"-v", ((Convert-ToDockerPath $pythonEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $framesRoot) + ":/frames:ro"),
|
||||
"-v", ((Convert-ToDockerPath $workRoot) + ":/work:ro"),
|
||||
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3",
|
||||
$ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "run",
|
||||
"--job", "/job",
|
||||
"--profile", ("/runner/{0}" -f $profileName),
|
||||
"--valid-fov-root", "/valid-fov",
|
||||
"--model-root", "/model",
|
||||
"--frames", "/frames",
|
||||
"--timeline", "/work/timeline.jsonl",
|
||||
"--output", "/publish/output",
|
||||
"--triton-url", "http://127.0.0.1:8000",
|
||||
"--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB),
|
||||
"--orchestrator-sha256", $orchestratorSha256,
|
||||
"--container-image", $ContainerImage,
|
||||
"--telemetry-interval-seconds", "1"
|
||||
)
|
||||
Write-Output ("PHASE=e5-inference-start FRAMES={0}" -f $clipFrameCount)
|
||||
& docker @runArgs
|
||||
Assert-LastExitCode "LAB E5 detector/tracker inference"
|
||||
$freeBytesPostInference = Assert-FreeSpace "post-inference"
|
||||
Write-Output "PHASE=e5-inference-complete"
|
||||
|
||||
$timelineRows = @(Get-Content -LiteralPath $timelinePath | ForEach-Object {
|
||||
$_ | ConvertFrom-Json
|
||||
})
|
||||
$clipSpanSeconds = [double]$timelineRows[-1].session_seconds - [double]$timelineRows[0].session_seconds
|
||||
if ($clipSpanSeconds -le 0) {
|
||||
throw "LAB E5 clip duration is invalid"
|
||||
}
|
||||
$averageFps = ($clipFrameCount - 1) / $clipSpanSeconds
|
||||
$fpsText = $averageFps.ToString("0.#########", [Globalization.CultureInfo]::InvariantCulture)
|
||||
$videoPath = Join-Path $stagingRoot "tracking.mp4"
|
||||
$encodeWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
& ffmpeg `
|
||||
-hide_banner -loglevel error `
|
||||
-framerate $fpsText `
|
||||
-i (Join-Path $stagingRoot "overlay-frames\frame-%06d.png") `
|
||||
-c:v h264_nvenc `
|
||||
-preset p4 `
|
||||
-tune hq `
|
||||
-rc vbr `
|
||||
-cq 21 `
|
||||
-b:v 0 `
|
||||
-pix_fmt yuv420p `
|
||||
-movflags +faststart `
|
||||
$videoPath
|
||||
Assert-LastExitCode "LAB E5 video encoding"
|
||||
$encodeWatch.Stop()
|
||||
Write-Output "PHASE=e5-video-encoding-complete"
|
||||
|
||||
$contactSheetPath = Join-Path $stagingRoot "contact-sheet.png"
|
||||
$tileColumns = if ($clipFrameCount -le 60) { 4 } else { 3 }
|
||||
$tileRows = 2
|
||||
$sampleInterval = [math]::Max(0.25, $clipSpanSeconds / ($tileColumns * $tileRows))
|
||||
$sampleText = $sampleInterval.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture)
|
||||
$contactFilter = "fps=1/{0},scale=400:-1,tile={1}x{2}:padding=8:margin=8:color=black" -f `
|
||||
$sampleText, $tileColumns, $tileRows
|
||||
& ffmpeg `
|
||||
-hide_banner -loglevel error `
|
||||
-i $videoPath `
|
||||
-vf $contactFilter `
|
||||
-frames:v 1 `
|
||||
$contactSheetPath
|
||||
Assert-LastExitCode "LAB E5 contact-sheet generation"
|
||||
|
||||
$probe = & ffprobe `
|
||||
-v error `
|
||||
-select_streams v:0 `
|
||||
-count_frames `
|
||||
-show_entries stream=codec_name,width,height,nb_read_frames `
|
||||
-of json `
|
||||
$videoPath | ConvertFrom-Json
|
||||
Assert-LastExitCode "LAB E5 result video probe"
|
||||
$videoStream = @($probe.streams)[0]
|
||||
if (
|
||||
$videoStream.codec_name -ne "h264" -or
|
||||
[int]$videoStream.width -ne 800 -or
|
||||
[int]$videoStream.height -ne 600 -or
|
||||
[int]$videoStream.nb_read_frames -ne $clipFrameCount
|
||||
) {
|
||||
throw "LAB E5 result video contract changed"
|
||||
}
|
||||
$freeBytesPostArtifacts = Assert-FreeSpace "post-artifacts"
|
||||
|
||||
$totalWatch.Stop()
|
||||
$finalizeArgs = @(
|
||||
"run", "--rm", "--network", "none", "--read-only",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "128", "--tmpfs", "/tmp:rw,noexec,nosuid,size=256m",
|
||||
"-e", "PYTHONPATH=/runner",
|
||||
"-v", ((Convert-ToDockerPath $stagingRoot) + ":/output:rw"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3",
|
||||
$ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "finalize",
|
||||
"--output", "/output",
|
||||
"--video", "/output/tracking.mp4",
|
||||
"--contact-sheet", "/output/contact-sheet.png",
|
||||
"--extract-seconds", $extractWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
|
||||
"--encode-seconds", $encodeWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
|
||||
"--wall-seconds", $totalWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
|
||||
"--encoder", "ffmpeg-h264_nvenc-p4-cq21-yuv420p-faststart",
|
||||
"--disk-free-before-bytes", [string]$freeBytesBefore,
|
||||
"--disk-free-post-extract-bytes", [string]$freeBytesPostExtract,
|
||||
"--disk-free-post-inference-bytes", [string]$freeBytesPostInference,
|
||||
"--disk-free-post-artifacts-bytes", [string]$freeBytesPostArtifacts,
|
||||
"--disk-floor-bytes", [string]([int64]$FreeGiBFloor * 1GB),
|
||||
"--working-set-reserve-bytes", [string]$workingSetReserve
|
||||
)
|
||||
& docker @finalizeArgs
|
||||
Assert-LastExitCode "LAB E5 result finalization"
|
||||
|
||||
$result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$result.schema_version -ne "missioncore.e5-tracking-result/v1" -or
|
||||
$result.result_id -notmatch "^e5-tracking-[a-f0-9]{64}$" -or
|
||||
[int]$result.frames_processed -ne $clipFrameCount -or
|
||||
$result.ground_truth -ne $false -or
|
||||
$result.publication_scope -ne "qualification-clip-only"
|
||||
) {
|
||||
throw "Final LAB E5 result manifest is incompatible"
|
||||
}
|
||||
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
||||
if (Test-Path -LiteralPath $finalRoot) {
|
||||
throw "An immutable result with the same LAB E5 identity already exists: $finalRoot"
|
||||
}
|
||||
$overlayFrames = Join-Path $stagingRoot "overlay-frames"
|
||||
if (Test-Path -LiteralPath $overlayFrames) {
|
||||
Remove-Item -LiteralPath $overlayFrames -Recurse -Force
|
||||
}
|
||||
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
|
||||
Remove-Item -LiteralPath $publishRoot -Force
|
||||
$completed = $true
|
||||
$freeBytesFinal = Assert-FreeSpace "post-publication"
|
||||
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
||||
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_EXTRACT={0}" -f $freeBytesPostExtract)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_INFERENCE={0}" -f $freeBytesPostInference)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_ARTIFACTS={0}" -f $freeBytesPostArtifacts)
|
||||
Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeBytesFinal)
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $workRoot) {
|
||||
Remove-Item -LiteralPath $workRoot -Recurse -Force
|
||||
}
|
||||
if (-not $completed -and (Test-Path -LiteralPath $publishRoot)) {
|
||||
Remove-Item -LiteralPath $publishRoot -Recurse -Force
|
||||
}
|
||||
if ($loadedByRun) {
|
||||
try {
|
||||
Invoke-WebRequest `
|
||||
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/unload" `
|
||||
-Method Post `
|
||||
-ContentType "application/json" `
|
||||
-Body "{}" `
|
||||
-UseBasicParsing `
|
||||
-TimeoutSec 60 *> $null
|
||||
Write-Output "PHASE=e5-model-state-restored"
|
||||
}
|
||||
catch {
|
||||
Write-Warning "LAB E5 could not restore the prior unloaded YOLOX-S state"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,411 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$JobRoot,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RunnerPath,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ProfilePath,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ValidFovRoot,
|
||||
|
||||
[ValidateRange(0, 1000000)]
|
||||
[int]$StartFrame = 1000,
|
||||
|
||||
[ValidateRange(0, 1000000)]
|
||||
[int]$EndFrame = 1600,
|
||||
|
||||
[ValidateRange(1, 1000)]
|
||||
[int]$FreeGiBFloor = 360,
|
||||
|
||||
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
|
||||
|
||||
[string]$ModelRoot = "D:\NDC_MISSIONCORE\runtime\models\yolox_s",
|
||||
|
||||
[string]$TritonContainer = "mission-core-triton",
|
||||
|
||||
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode {
|
||||
param([string]$Operation)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-RegularFile {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if (-not $item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
return $item.FullName
|
||||
}
|
||||
throw "$Label is not a regular file"
|
||||
}
|
||||
|
||||
function Assert-Directory {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if ($item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
return $item.FullName
|
||||
}
|
||||
throw "$Label is not a real directory"
|
||||
}
|
||||
|
||||
function Assert-DDrivePath {
|
||||
param([string]$Path, [string]$Label)
|
||||
$fullPath = [IO.Path]::GetFullPath($Path)
|
||||
if ([IO.Path]::GetPathRoot($fullPath).TrimEnd("\") -ine "D:") {
|
||||
throw "$Label must be stored on D:"
|
||||
}
|
||||
return $fullPath
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath {
|
||||
param([string]$Path)
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
function Get-DFreeBytes {
|
||||
return [int64](Get-PSDrive -Name D).Free
|
||||
}
|
||||
|
||||
function Assert-FreeSpace {
|
||||
param([string]$Phase, [int64]$RequiredAdditionalBytes = 0)
|
||||
$freeBytes = Get-DFreeBytes
|
||||
$floorBytes = [int64]$FreeGiBFloor * 1GB
|
||||
Write-Host (
|
||||
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3} REQUIRED_ADDITIONAL_GIB={4}" -f
|
||||
$Phase,
|
||||
$freeBytes,
|
||||
[math]::Round($freeBytes / 1GB, 3),
|
||||
$FreeGiBFloor,
|
||||
[math]::Round($RequiredAdditionalBytes / 1GB, 3)
|
||||
)
|
||||
if ($freeBytes -lt ($floorBytes + $RequiredAdditionalBytes)) {
|
||||
throw "D: does not have the guarded LAB E8 working-set reserve during $Phase"
|
||||
}
|
||||
return $freeBytes
|
||||
}
|
||||
|
||||
function Test-TritonModelReady {
|
||||
try {
|
||||
$response = Invoke-WebRequest `
|
||||
-Uri "http://127.0.0.1:8000/v2/models/yolox_s/ready" `
|
||||
-Method Get -UseBasicParsing -TimeoutSec 10
|
||||
return $response.StatusCode -eq 200
|
||||
}
|
||||
catch {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
||||
$jobDirectory = Assert-DDrivePath (
|
||||
Assert-Directory (Resolve-Path -LiteralPath $JobRoot).Path "Job root"
|
||||
) "Job root"
|
||||
$runner = Assert-DDrivePath (
|
||||
Assert-RegularFile (Resolve-Path -LiteralPath $RunnerPath).Path "LAB E8 runner"
|
||||
) "LAB E8 runner"
|
||||
$profile = Assert-DDrivePath (
|
||||
Assert-RegularFile (Resolve-Path -LiteralPath $ProfilePath).Path "LAB E8 profile"
|
||||
) "LAB E8 profile"
|
||||
$validFov = Assert-DDrivePath (
|
||||
Assert-Directory (Resolve-Path -LiteralPath $ValidFovRoot).Path "Valid-FOV root"
|
||||
) "Valid-FOV root"
|
||||
$runtime = Assert-DDrivePath (
|
||||
Assert-Directory (Resolve-Path -LiteralPath $RuntimeRoot).Path "Runtime root"
|
||||
) "Runtime root"
|
||||
$model = Assert-DDrivePath (
|
||||
Assert-Directory (Resolve-Path -LiteralPath $ModelRoot).Path "YOLOX model root"
|
||||
) "YOLOX model root"
|
||||
$runnerRoot = Split-Path $runner -Parent
|
||||
if ((Split-Path $profile -Parent) -ne $runnerRoot) {
|
||||
throw "LAB E8 runner and profile must share one read-only mount"
|
||||
}
|
||||
foreach ($dependency in @(
|
||||
"run_e5_instance_tracking.py",
|
||||
"run_recorded_perception_epoch.py",
|
||||
"run_e3_rectified_segmentation.py",
|
||||
"run_evaluation_prelabels.py"
|
||||
)) {
|
||||
$null = Assert-RegularFile (Join-Path $runnerRoot $dependency) "LAB E8 runner dependency"
|
||||
}
|
||||
|
||||
$job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json
|
||||
if ($job.schema_version -ne "missioncore.compute-job/v1" -or $job.job_id -ne (Split-Path $jobDirectory -Leaf)) {
|
||||
throw "Compute job manifest is incompatible"
|
||||
}
|
||||
$sourceId = [string]$job.input.source_id
|
||||
$epoch = [int]$job.input.codec_epoch
|
||||
$fullFrameCount = [int]$job.input.segment_count
|
||||
$clipFrameCount = $EndFrame - $StartFrame + 1
|
||||
$timelineStart = [double]$job.input.timeline.start_seconds
|
||||
if (
|
||||
$sourceId -ne "sensor.camera.right" -or
|
||||
$StartFrame -lt 0 -or
|
||||
$EndFrame -lt $StartFrame -or
|
||||
$EndFrame -ge $fullFrameCount -or
|
||||
$clipFrameCount -lt 2
|
||||
) {
|
||||
throw "LAB E8 clip escapes the camera job"
|
||||
}
|
||||
Write-Output (
|
||||
"PHASE=job-manifest-validated JOB={0} CLIP={1}-{2} FRAMES={3}" -f
|
||||
$job.job_id, $StartFrame, $EndFrame, $clipFrameCount
|
||||
)
|
||||
|
||||
$epochRoot = Assert-DDrivePath (
|
||||
Assert-Directory (
|
||||
Join-Path $jobDirectory ("input\camera\{0}\epoch-{1}" -f $sourceId, $epoch)
|
||||
) "Camera epoch"
|
||||
) "Camera epoch"
|
||||
$initPath = Assert-RegularFile (Join-Path $epochRoot "init.mp4") "Camera init"
|
||||
$segmentsRoot = Assert-Directory (Join-Path $epochRoot "segments") "Camera segments"
|
||||
$derivedRoot = Join-Path $runtime "derived"
|
||||
$tmpRoot = Join-Path $runtime "tmp"
|
||||
$pythonEnvironment = Assert-DDrivePath (
|
||||
Assert-Directory (Join-Path $derivedRoot "perception-p0-env-v1") "Python environment"
|
||||
) "Python environment"
|
||||
$null = New-Item -ItemType Directory -Path $derivedRoot -Force
|
||||
$null = New-Item -ItemType Directory -Path $tmpRoot -Force
|
||||
|
||||
# LAB E8 has temporary decoded input but no per-frame output images. Reserve
|
||||
# decoded RGB payload, partial stream and two GiB for filesystem/codec overhead.
|
||||
$pixelWorkingSet = [int64]$clipFrameCount * 800 * 600 * 3
|
||||
$partialStreamReserve = [int64][math]::Ceiling(
|
||||
([int64]$job.input.byte_length * ($EndFrame + 1) / $fullFrameCount) * 1.2
|
||||
)
|
||||
$workingSetReserve = [int64][math]::Ceiling($pixelWorkingSet * 1.4) + $partialStreamReserve + 2GB
|
||||
$freeBytesBefore = Assert-FreeSpace "preflight" $workingSetReserve
|
||||
|
||||
& docker image inspect $ContainerImage *> $null
|
||||
Assert-LastExitCode "Existing container image inspection"
|
||||
$tritonState = docker inspect --format "{{.State.Running}}" $TritonContainer
|
||||
Assert-LastExitCode "Triton container inspection"
|
||||
if ($tritonState.Trim().ToLowerInvariant() -ne "true") {
|
||||
throw "LAB E8 requires the existing Triton container"
|
||||
}
|
||||
|
||||
$runnerName = Split-Path $runner -Leaf
|
||||
$profileName = Split-Path $profile -Leaf
|
||||
$orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$preflightArgs = @(
|
||||
"run", "--rm", "--network", "none", "--read-only",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "128", "--tmpfs", "/tmp:rw,noexec,nosuid,size=256m",
|
||||
"-e", "PYTHONPATH=/runner:/opt/env",
|
||||
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
||||
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
||||
"-v", ((Convert-ToDockerPath $model) + ":/model:ro"),
|
||||
"-v", ((Convert-ToDockerPath $pythonEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3", $ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "preflight",
|
||||
"--job", "/job",
|
||||
"--profile", ("/runner/{0}" -f $profileName),
|
||||
"--valid-fov-root", "/valid-fov",
|
||||
"--model-root", "/model"
|
||||
)
|
||||
Write-Output "PHASE=e8-preflight-start"
|
||||
& docker @preflightArgs
|
||||
Assert-LastExitCode "LAB E8 preflight"
|
||||
Write-Output "PHASE=e8-preflight-complete"
|
||||
|
||||
$modelWasReady = Test-TritonModelReady
|
||||
$loadedByRun = $false
|
||||
if (-not $modelWasReady) {
|
||||
Invoke-WebRequest `
|
||||
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/load" `
|
||||
-Method Post -ContentType "application/json" -Body "{}" `
|
||||
-UseBasicParsing -TimeoutSec 60 *> $null
|
||||
$loadedByRun = $true
|
||||
$deadline = [DateTime]::UtcNow.AddSeconds(60)
|
||||
while (-not (Test-TritonModelReady)) {
|
||||
if ([DateTime]::UtcNow -ge $deadline) {
|
||||
throw "YOLOX-S did not become ready in Triton"
|
||||
}
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
}
|
||||
Write-Output ("PHASE=e8-model-ready LOADED_BY_RUN={0}" -f $loadedByRun)
|
||||
|
||||
$runToken = [Guid]::NewGuid().ToString("N")
|
||||
$workRoot = Join-Path $tmpRoot ("{0}-e8-{1}" -f $job.job_id, $runToken)
|
||||
$framesRoot = Join-Path $workRoot "frames"
|
||||
$streamPath = Join-Path $workRoot "camera.mp4"
|
||||
$ptsPath = Join-Path $workRoot "pts.json"
|
||||
$timelinePath = Join-Path $workRoot "timeline.jsonl"
|
||||
$publishRoot = Join-Path $derivedRoot (".{0}-e8-{1}.publish" -f $job.job_id, $runToken)
|
||||
$stagingRoot = Join-Path $publishRoot "output"
|
||||
$null = New-Item -ItemType Directory -Path $framesRoot
|
||||
$null = New-Item -ItemType Directory -Path $publishRoot
|
||||
$totalWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
$completed = $false
|
||||
Write-Output "PHASE=e8-private-staging-created"
|
||||
|
||||
try {
|
||||
$stream = [IO.File]::Open(
|
||||
$streamPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None
|
||||
)
|
||||
try {
|
||||
$input = [IO.File]::OpenRead($initPath)
|
||||
try { $input.CopyTo($stream) } finally { $input.Dispose() }
|
||||
for ($sequence = 1; $sequence -le ($EndFrame + 1); $sequence++) {
|
||||
$path = Assert-RegularFile (
|
||||
Join-Path $segmentsRoot ("{0}.m4s" -f $sequence)
|
||||
) "Camera segment"
|
||||
$input = [IO.File]::OpenRead($path)
|
||||
try { $input.CopyTo($stream) } finally { $input.Dispose() }
|
||||
}
|
||||
$stream.Flush($true)
|
||||
}
|
||||
finally {
|
||||
$stream.Dispose()
|
||||
}
|
||||
$null = Assert-FreeSpace "post-stream-reconstruction"
|
||||
|
||||
$extractWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
Write-Output "PHASE=e8-frame-extraction-start"
|
||||
$selectFilter = "select='between(n,{0},{1})'" -f $StartFrame, $EndFrame
|
||||
& ffmpeg -hide_banner -loglevel fatal -i $streamPath -map 0:v:0 `
|
||||
-vf $selectFilter -fps_mode passthrough (Join-Path $framesRoot "frame-%06d.png")
|
||||
Assert-LastExitCode "LAB E8 camera clip extraction"
|
||||
& ffprobe -v error -select_streams v:0 `
|
||||
-show_entries frame=best_effort_timestamp_time -of json $streamPath |
|
||||
Set-Content -LiteralPath $ptsPath -Encoding utf8
|
||||
Assert-LastExitCode "LAB E8 camera timestamp probe"
|
||||
|
||||
$decodedFrames = @(Get-ChildItem -LiteralPath $framesRoot -File -Filter "frame-*.png")
|
||||
$ptsDocument = Get-Content -LiteralPath $ptsPath -Raw | ConvertFrom-Json
|
||||
$pts = @($ptsDocument.frames)
|
||||
if ($decodedFrames.Count -ne $clipFrameCount -or $pts.Count -ne ($EndFrame + 1)) {
|
||||
throw "Decoded LAB E8 frame/timestamp count differs from the selected clip"
|
||||
}
|
||||
$firstEpochSeconds = [double]::Parse(
|
||||
([string]$pts[0].best_effort_timestamp_time).Trim(),
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
)
|
||||
$previousSessionSeconds = -1.0
|
||||
$timelineWriter = [IO.StreamWriter]::new(
|
||||
$timelinePath, $false, [Text.UTF8Encoding]::new($false)
|
||||
)
|
||||
try {
|
||||
for ($localIndex = 0; $localIndex -lt $clipFrameCount; $localIndex++) {
|
||||
$sourceIndex = $StartFrame + $localIndex
|
||||
$epochSeconds = [double]::Parse(
|
||||
([string]$pts[$sourceIndex].best_effort_timestamp_time).Trim(),
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
) - $firstEpochSeconds
|
||||
$sessionSeconds = $timelineStart + $epochSeconds
|
||||
if ($sessionSeconds -le $previousSessionSeconds) {
|
||||
throw "Decoded LAB E8 clip timestamps are not strictly monotonic"
|
||||
}
|
||||
$row = [ordered]@{
|
||||
frame_index = $localIndex
|
||||
sequence = $localIndex + 1
|
||||
source_frame_index = $sourceIndex
|
||||
source_sequence = $sourceIndex + 1
|
||||
epoch_seconds = $epochSeconds
|
||||
session_seconds = $sessionSeconds
|
||||
}
|
||||
$timelineWriter.WriteLine(($row | ConvertTo-Json -Compress))
|
||||
$previousSessionSeconds = $sessionSeconds
|
||||
}
|
||||
$timelineWriter.Flush()
|
||||
}
|
||||
finally {
|
||||
$timelineWriter.Dispose()
|
||||
}
|
||||
$extractWatch.Stop()
|
||||
$freeBytesPostExtract = Assert-FreeSpace "post-frame-extraction"
|
||||
Write-Output ("PHASE=e8-frame-extraction-complete FRAMES={0}" -f $clipFrameCount)
|
||||
|
||||
$runArgs = @(
|
||||
"run", "--rm", "--gpus", "all",
|
||||
"--network", ("container:{0}" -f $TritonContainer),
|
||||
"--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "256", "--shm-size", "1g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=512m",
|
||||
"-e", "PYTHONPATH=/runner:/opt/env",
|
||||
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
||||
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
||||
"-v", ((Convert-ToDockerPath $model) + ":/model:ro"),
|
||||
"-v", ((Convert-ToDockerPath $pythonEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $framesRoot) + ":/frames:ro"),
|
||||
"-v", ((Convert-ToDockerPath $workRoot) + ":/work:ro"),
|
||||
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3", $ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "run",
|
||||
"--job", "/job",
|
||||
"--profile", ("/runner/{0}" -f $profileName),
|
||||
"--valid-fov-root", "/valid-fov",
|
||||
"--model-root", "/model",
|
||||
"--frames", "/frames",
|
||||
"--timeline", "/work/timeline.jsonl",
|
||||
"--output", "/publish/output",
|
||||
"--triton-url", "http://127.0.0.1:8000",
|
||||
"--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB),
|
||||
"--orchestrator-sha256", $orchestratorSha256,
|
||||
"--container-image", $ContainerImage,
|
||||
"--telemetry-interval-seconds", "1"
|
||||
)
|
||||
Write-Output ("PHASE=e8-source-paced-replay-start FRAMES={0}" -f $clipFrameCount)
|
||||
& docker @runArgs
|
||||
Assert-LastExitCode "LAB E8 source-paced detector/tracker"
|
||||
$freeBytesPostReplay = Assert-FreeSpace "post-replay"
|
||||
Write-Output "PHASE=e8-source-paced-replay-complete"
|
||||
|
||||
$result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$result.schema_version -ne "missioncore.e8-realtime-tracking-result/v1" -or
|
||||
$result.result_id -notmatch "^e8-realtime-tracking-[a-f0-9]{64}$" -or
|
||||
$result.acceptance_state -ne "accepted" -or
|
||||
$result.ground_truth -ne $false
|
||||
) {
|
||||
throw "Final LAB E8 result manifest is incompatible or rejected"
|
||||
}
|
||||
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
||||
if (Test-Path -LiteralPath $finalRoot) {
|
||||
throw "An immutable result with the same LAB E8 identity already exists: $finalRoot"
|
||||
}
|
||||
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
|
||||
Remove-Item -LiteralPath $publishRoot -Force
|
||||
$totalWatch.Stop()
|
||||
$completed = $true
|
||||
$freeBytesFinal = Assert-FreeSpace "post-publication"
|
||||
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
||||
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
||||
Write-Output ("PREPARATION_SECONDS={0}" -f $extractWatch.Elapsed.TotalSeconds)
|
||||
Write-Output ("ORCHESTRATOR_WALL_SECONDS={0}" -f $totalWatch.Elapsed.TotalSeconds)
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_EXTRACT={0}" -f $freeBytesPostExtract)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_REPLAY={0}" -f $freeBytesPostReplay)
|
||||
Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeBytesFinal)
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $workRoot) {
|
||||
Remove-Item -LiteralPath $workRoot -Recurse -Force
|
||||
}
|
||||
if (-not $completed -and (Test-Path -LiteralPath $publishRoot)) {
|
||||
Remove-Item -LiteralPath $publishRoot -Recurse -Force
|
||||
}
|
||||
if ($loadedByRun) {
|
||||
try {
|
||||
Invoke-WebRequest `
|
||||
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/unload" `
|
||||
-Method Post -ContentType "application/json" -Body "{}" `
|
||||
-UseBasicParsing -TimeoutSec 60 *> $null
|
||||
Write-Output "PHASE=e8-model-state-restored"
|
||||
}
|
||||
catch {
|
||||
Write-Warning "LAB E8 could not restore the prior unloaded YOLOX-S state"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)] [string]$JobRoot,
|
||||
[Parameter(Mandatory = $true)] [string]$RunnerPath,
|
||||
[Parameter(Mandatory = $true)] [string]$ProfilePath,
|
||||
[Parameter(Mandatory = $true)] [string]$DetectorProfilePath,
|
||||
[Parameter(Mandatory = $true)] [string]$SemanticProfilePath,
|
||||
[Parameter(Mandatory = $true)] [string]$ValidFovRoot,
|
||||
[ValidateRange(0, 1000000)] [int]$StartFrame = 1000,
|
||||
[ValidateRange(0, 1000000)] [int]$EndFrame = 1600,
|
||||
[ValidateRange(1, 1000)] [int]$FreeGiBFloor = 360,
|
||||
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
|
||||
[string]$ModelRoot = "D:\NDC_MISSIONCORE\runtime\models\yolox_s",
|
||||
[string]$TritonContainer = "mission-core-triton",
|
||||
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode {
|
||||
param([string]$Operation)
|
||||
if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" }
|
||||
}
|
||||
|
||||
function Resolve-DDirectory {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
||||
if (-not $item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") {
|
||||
throw "$Label must be a real D: directory"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Resolve-DFile {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
||||
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") {
|
||||
throw "$Label must be a regular D: file"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath {
|
||||
param([string]$Path)
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
function Get-DFreeBytes { return [int64](Get-PSDrive -Name D).Free }
|
||||
|
||||
function Assert-FreeSpace {
|
||||
param([string]$Phase, [int64]$RequiredAdditionalBytes = 0)
|
||||
$freeBytes = Get-DFreeBytes
|
||||
$floorBytes = [int64]$FreeGiBFloor * 1GB
|
||||
Write-Host (
|
||||
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3} REQUIRED_ADDITIONAL_GIB={4}" -f
|
||||
$Phase, $freeBytes, [math]::Round($freeBytes / 1GB, 3), $FreeGiBFloor,
|
||||
[math]::Round($RequiredAdditionalBytes / 1GB, 3)
|
||||
)
|
||||
if ($freeBytes -lt ($floorBytes + $RequiredAdditionalBytes)) {
|
||||
throw "D: does not have the guarded LAB E9 working-set reserve during $Phase"
|
||||
}
|
||||
return $freeBytes
|
||||
}
|
||||
|
||||
function Test-TritonModelReady {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -TimeoutSec 10 `
|
||||
-Uri "http://127.0.0.1:8000/v2/models/yolox_s/ready"
|
||||
return $response.StatusCode -eq 200
|
||||
}
|
||||
catch { return $false }
|
||||
}
|
||||
|
||||
$jobDirectory = Resolve-DDirectory $JobRoot "Job root"
|
||||
$runner = Resolve-DFile $RunnerPath "LAB E9 runner"
|
||||
$profile = Resolve-DFile $ProfilePath "LAB E9 profile"
|
||||
$detectorProfile = Resolve-DFile $DetectorProfilePath "LAB E9 detector profile"
|
||||
$semanticProfile = Resolve-DFile $SemanticProfilePath "LAB E9 semantic profile"
|
||||
$validFov = Resolve-DDirectory $ValidFovRoot "Valid-FOV root"
|
||||
$runtime = Resolve-DDirectory $RuntimeRoot "Runtime root"
|
||||
$model = Resolve-DDirectory $ModelRoot "YOLOX model root"
|
||||
$runnerRoot = Split-Path $runner -Parent
|
||||
foreach ($path in @($profile, $detectorProfile, $semanticProfile)) {
|
||||
if ((Split-Path $path -Parent) -ne $runnerRoot) {
|
||||
throw "LAB E9 runner and profiles must share one read-only mount"
|
||||
}
|
||||
}
|
||||
foreach ($dependency in @(
|
||||
"run_e8_realtime_tracking.py",
|
||||
"run_e5_instance_tracking.py",
|
||||
"run_e4_full_session_segmentation.py",
|
||||
"run_recorded_perception_epoch.py",
|
||||
"run_e3_rectified_segmentation.py",
|
||||
"run_evaluation_prelabels.py"
|
||||
)) {
|
||||
$null = Resolve-DFile (Join-Path $runnerRoot $dependency) "LAB E9 runner dependency"
|
||||
}
|
||||
|
||||
$job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json
|
||||
if ($job.schema_version -ne "missioncore.compute-job/v1" -or $job.job_id -ne (Split-Path $jobDirectory -Leaf)) {
|
||||
throw "Compute job manifest is incompatible"
|
||||
}
|
||||
$sourceId = [string]$job.input.source_id
|
||||
$epoch = [int]$job.input.codec_epoch
|
||||
$fullFrameCount = [int]$job.input.segment_count
|
||||
$clipFrameCount = $EndFrame - $StartFrame + 1
|
||||
$timelineStart = [double]$job.input.timeline.start_seconds
|
||||
if (
|
||||
$sourceId -ne "sensor.camera.right" -or $StartFrame -lt 0 -or
|
||||
$EndFrame -lt $StartFrame -or $EndFrame -ge $fullFrameCount -or $clipFrameCount -lt 2
|
||||
) { throw "LAB E9 clip escapes the camera job" }
|
||||
Write-Output (
|
||||
"PHASE=job-manifest-validated JOB={0} CLIP={1}-{2} FRAMES={3}" -f
|
||||
$job.job_id, $StartFrame, $EndFrame, $clipFrameCount
|
||||
)
|
||||
|
||||
$epochRoot = Resolve-DDirectory (
|
||||
Join-Path $jobDirectory ("input\camera\{0}\epoch-{1}" -f $sourceId, $epoch)
|
||||
) "Camera epoch"
|
||||
$initPath = Resolve-DFile (Join-Path $epochRoot "init.mp4") "Camera init"
|
||||
$segmentsRoot = Resolve-DDirectory (Join-Path $epochRoot "segments") "Camera segments"
|
||||
$derivedRoot = Join-Path $runtime "derived"
|
||||
$tmpRoot = Join-Path $runtime "tmp"
|
||||
$cacheRoot = Resolve-DDirectory (Join-Path $runtime "cache\perception-e3-models-v1") "EoMT cache"
|
||||
$e3Environment = Resolve-DDirectory (Join-Path $derivedRoot "perception-e3-opencv413092-v1") "E3 environment"
|
||||
$torchEnvironment = Resolve-DDirectory (Join-Path $derivedRoot "perception-p0-env-v1") "Torch environment"
|
||||
$transformersEnvironment = Resolve-DDirectory (Join-Path $derivedRoot "perception-p0-transformers4576-v1") "Transformers environment"
|
||||
$null = New-Item -ItemType Directory -Path $derivedRoot -Force
|
||||
$null = New-Item -ItemType Directory -Path $tmpRoot -Force
|
||||
|
||||
$pixelWorkingSet = [int64]$clipFrameCount * 800 * 600 * 3
|
||||
$partialStreamReserve = [int64][math]::Ceiling(
|
||||
([int64]$job.input.byte_length * ($EndFrame + 1) / $fullFrameCount) * 1.2
|
||||
)
|
||||
$workingSetReserve = [int64][math]::Ceiling($pixelWorkingSet * 1.4) + $partialStreamReserve + 3GB
|
||||
$freeBytesBefore = Assert-FreeSpace "preflight" $workingSetReserve
|
||||
|
||||
& docker image inspect $ContainerImage *> $null
|
||||
Assert-LastExitCode "Existing container image inspection"
|
||||
$tritonState = docker inspect --format "{{.State.Running}}" $TritonContainer
|
||||
Assert-LastExitCode "Triton container inspection"
|
||||
if ($tritonState.Trim().ToLowerInvariant() -ne "true") {
|
||||
throw "LAB E9 requires the existing Triton container"
|
||||
}
|
||||
|
||||
$runnerName = Split-Path $runner -Leaf
|
||||
$profileName = Split-Path $profile -Leaf
|
||||
$detectorProfileName = Split-Path $detectorProfile -Leaf
|
||||
$semanticProfileName = Split-Path $semanticProfile -Leaf
|
||||
$orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$commonMounts = @(
|
||||
"-e", "PYTHONPATH=/runner:/opt/transformers:/opt/env",
|
||||
"-e", "TORCH_HOME=/cache/torch",
|
||||
"-e", "HF_HOME=/cache/huggingface",
|
||||
"-e", "HF_HUB_OFFLINE=1",
|
||||
"-e", "TRANSFORMERS_OFFLINE=1",
|
||||
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
||||
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
||||
"-v", ((Convert-ToDockerPath $model) + ":/model:ro"),
|
||||
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"),
|
||||
"-v", ((Convert-ToDockerPath $e3Environment) + ":/environment:ro"),
|
||||
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro")
|
||||
)
|
||||
$commonRunnerArgs = @(
|
||||
"--job", "/job",
|
||||
"--profile", ("/runner/{0}" -f $profileName),
|
||||
"--detector-profile", ("/runner/{0}" -f $detectorProfileName),
|
||||
"--semantic-profile", ("/runner/{0}" -f $semanticProfileName),
|
||||
"--valid-fov-root", "/valid-fov",
|
||||
"--model-root", "/model",
|
||||
"--cache", "/cache",
|
||||
"--environment", "/environment"
|
||||
)
|
||||
$preflightArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", "none", "--read-only",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "512", "--shm-size", "4g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g"
|
||||
) + $commonMounts + @(
|
||||
"--entrypoint", "python3", $ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "preflight"
|
||||
) + $commonRunnerArgs
|
||||
Write-Output "PHASE=e9-preflight-start"
|
||||
& docker @preflightArgs
|
||||
Assert-LastExitCode "LAB E9 preflight"
|
||||
Write-Output "PHASE=e9-preflight-complete"
|
||||
|
||||
$modelWasReady = Test-TritonModelReady
|
||||
$loadedByRun = $false
|
||||
if (-not $modelWasReady) {
|
||||
Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 `
|
||||
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/load" `
|
||||
-Method Post -ContentType "application/json" -Body "{}" *> $null
|
||||
$loadedByRun = $true
|
||||
$deadline = [DateTime]::UtcNow.AddSeconds(60)
|
||||
while (-not (Test-TritonModelReady)) {
|
||||
if ([DateTime]::UtcNow -ge $deadline) { throw "YOLOX-S did not become ready" }
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
}
|
||||
Write-Output ("PHASE=e9-model-ready LOADED_BY_RUN={0}" -f $loadedByRun)
|
||||
|
||||
$runToken = [Guid]::NewGuid().ToString("N")
|
||||
$workRoot = Join-Path $tmpRoot ("{0}-e9-{1}" -f $job.job_id, $runToken)
|
||||
$framesRoot = Join-Path $workRoot "frames"
|
||||
$streamPath = Join-Path $workRoot "camera.mp4"
|
||||
$ptsPath = Join-Path $workRoot "pts.json"
|
||||
$timelinePath = Join-Path $workRoot "timeline.jsonl"
|
||||
$publishRoot = Join-Path $derivedRoot (".{0}-e9-{1}.publish" -f $job.job_id, $runToken)
|
||||
$stagingRoot = Join-Path $publishRoot "output"
|
||||
$null = New-Item -ItemType Directory -Path $framesRoot
|
||||
$null = New-Item -ItemType Directory -Path $publishRoot
|
||||
$completed = $false
|
||||
$totalWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
|
||||
try {
|
||||
$stream = [IO.File]::Open($streamPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
||||
try {
|
||||
$input = [IO.File]::OpenRead($initPath)
|
||||
try { $input.CopyTo($stream) } finally { $input.Dispose() }
|
||||
for ($sequence = 1; $sequence -le ($EndFrame + 1); $sequence++) {
|
||||
$path = Resolve-DFile (Join-Path $segmentsRoot ("{0}.m4s" -f $sequence)) "Camera segment"
|
||||
$input = [IO.File]::OpenRead($path)
|
||||
try { $input.CopyTo($stream) } finally { $input.Dispose() }
|
||||
}
|
||||
$stream.Flush($true)
|
||||
}
|
||||
finally { $stream.Dispose() }
|
||||
$null = Assert-FreeSpace "post-stream-reconstruction"
|
||||
|
||||
$extractWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
$selectFilter = "select='between(n,{0},{1})'" -f $StartFrame, $EndFrame
|
||||
& ffmpeg -hide_banner -loglevel fatal -i $streamPath -map 0:v:0 `
|
||||
-vf $selectFilter -fps_mode passthrough (Join-Path $framesRoot "frame-%06d.png")
|
||||
Assert-LastExitCode "LAB E9 camera clip extraction"
|
||||
& ffprobe -v error -select_streams v:0 -show_entries frame=best_effort_timestamp_time `
|
||||
-of json $streamPath | Set-Content -LiteralPath $ptsPath -Encoding utf8
|
||||
Assert-LastExitCode "LAB E9 camera timestamp probe"
|
||||
|
||||
$decodedFrames = @(Get-ChildItem -LiteralPath $framesRoot -File -Filter "frame-*.png")
|
||||
$pts = @((Get-Content -LiteralPath $ptsPath -Raw | ConvertFrom-Json).frames)
|
||||
if ($decodedFrames.Count -ne $clipFrameCount -or $pts.Count -ne ($EndFrame + 1)) {
|
||||
throw "Decoded LAB E9 frame/timestamp count changed"
|
||||
}
|
||||
$firstEpochSeconds = [double]::Parse(
|
||||
([string]$pts[0].best_effort_timestamp_time).Trim(),
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
)
|
||||
$previousSessionSeconds = -1.0
|
||||
$timelineWriter = [IO.StreamWriter]::new(
|
||||
$timelinePath, $false, [Text.UTF8Encoding]::new($false)
|
||||
)
|
||||
try {
|
||||
for ($localIndex = 0; $localIndex -lt $clipFrameCount; $localIndex++) {
|
||||
$sourceIndex = $StartFrame + $localIndex
|
||||
$epochSeconds = [double]::Parse(
|
||||
([string]$pts[$sourceIndex].best_effort_timestamp_time).Trim(),
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
) - $firstEpochSeconds
|
||||
$sessionSeconds = $timelineStart + $epochSeconds
|
||||
if ($sessionSeconds -le $previousSessionSeconds) { throw "LAB E9 timeline is not monotonic" }
|
||||
$row = [ordered]@{
|
||||
frame_index = $localIndex
|
||||
sequence = $localIndex + 1
|
||||
source_frame_index = $sourceIndex
|
||||
source_sequence = $sourceIndex + 1
|
||||
epoch_seconds = $epochSeconds
|
||||
session_seconds = $sessionSeconds
|
||||
}
|
||||
$timelineWriter.WriteLine(($row | ConvertTo-Json -Compress))
|
||||
$previousSessionSeconds = $sessionSeconds
|
||||
}
|
||||
$timelineWriter.Flush()
|
||||
}
|
||||
finally { $timelineWriter.Dispose() }
|
||||
$extractWatch.Stop()
|
||||
$freeBytesPostExtract = Assert-FreeSpace "post-frame-extraction"
|
||||
Write-Output ("PHASE=e9-frame-extraction-complete FRAMES={0}" -f $clipFrameCount)
|
||||
|
||||
$runArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", ("container:{0}" -f $TritonContainer),
|
||||
"--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "512", "--shm-size", "4g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g"
|
||||
) + $commonMounts + @(
|
||||
"-v", ((Convert-ToDockerPath $framesRoot) + ":/frames:ro"),
|
||||
"-v", ((Convert-ToDockerPath $workRoot) + ":/work:ro"),
|
||||
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
|
||||
"--entrypoint", "python3", $ContainerImage,
|
||||
("/runner/{0}" -f $runnerName), "run"
|
||||
) + $commonRunnerArgs + @(
|
||||
"--frames", "/frames",
|
||||
"--timeline", "/work/timeline.jsonl",
|
||||
"--output", "/publish/output",
|
||||
"--triton-url", "http://127.0.0.1:8000",
|
||||
"--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB),
|
||||
"--orchestrator-sha256", $orchestratorSha256,
|
||||
"--container-image", $ContainerImage,
|
||||
"--telemetry-interval-seconds", "1"
|
||||
)
|
||||
Write-Output ("PHASE=e9-multirate-replay-start FRAMES={0}" -f $clipFrameCount)
|
||||
& docker @runArgs
|
||||
Assert-LastExitCode "LAB E9 multirate replay"
|
||||
$freeBytesPostReplay = Assert-FreeSpace "post-replay"
|
||||
|
||||
$result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$result.schema_version -ne "missioncore.e9-multirate-perception-result/v1" -or
|
||||
$result.result_id -notmatch "^e9-multirate-perception-[a-f0-9]{64}$" -or
|
||||
$result.acceptance_state -notin @("accepted", "rejected") -or
|
||||
$result.ground_truth -ne $false
|
||||
) { throw "LAB E9 result manifest is incompatible" }
|
||||
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
||||
if (Test-Path -LiteralPath $finalRoot) { throw "Immutable LAB E9 result already exists" }
|
||||
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
|
||||
Remove-Item -LiteralPath $publishRoot -Force
|
||||
$completed = $true
|
||||
$totalWatch.Stop()
|
||||
$freeBytesFinal = Assert-FreeSpace "post-publication"
|
||||
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
||||
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
||||
Write-Output ("ACCEPTANCE_STATE={0}" -f $result.acceptance_state)
|
||||
Write-Output ("PREPARATION_SECONDS={0}" -f $extractWatch.Elapsed.TotalSeconds)
|
||||
Write-Output ("ORCHESTRATOR_WALL_SECONDS={0}" -f $totalWatch.Elapsed.TotalSeconds)
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_EXTRACT={0}" -f $freeBytesPostExtract)
|
||||
Write-Output ("DISK_FREE_BYTES_POST_REPLAY={0}" -f $freeBytesPostReplay)
|
||||
Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeBytesFinal)
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $workRoot) { Remove-Item -LiteralPath $workRoot -Recurse -Force }
|
||||
if (-not $completed -and (Test-Path -LiteralPath $publishRoot)) {
|
||||
Remove-Item -LiteralPath $publishRoot -Recurse -Force
|
||||
}
|
||||
if ($loadedByRun) {
|
||||
try {
|
||||
Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 `
|
||||
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/unload" `
|
||||
-Method Post -ContentType "application/json" -Body "{}" *> $null
|
||||
Write-Output "PHASE=e9-model-state-restored"
|
||||
}
|
||||
catch { Write-Warning "LAB E9 could not restore the prior unloaded YOLOX-S state" }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$EvaluationPack,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ValidFovRoot,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RunnerPath,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$BaselineRunnerPath,
|
||||
|
||||
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode {
|
||||
param([string]$Operation)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-RegularFile {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if (-not $item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
return $item.FullName
|
||||
}
|
||||
throw "$Label is not a regular file"
|
||||
}
|
||||
|
||||
function Assert-Directory {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if ($item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
return $item.FullName
|
||||
}
|
||||
throw "$Label is not a real directory"
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath {
|
||||
param([string]$Path)
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
$packRoot = Assert-Directory (Resolve-Path -LiteralPath $EvaluationPack).Path "Evaluation pack"
|
||||
$validFov = Assert-Directory (Resolve-Path -LiteralPath $ValidFovRoot).Path "Valid-FOV root"
|
||||
$runner = Assert-RegularFile (Resolve-Path -LiteralPath $RunnerPath).Path "Prelabel runner"
|
||||
$baselineRunner = Assert-RegularFile (Resolve-Path -LiteralPath $BaselineRunnerPath).Path "Baseline runner"
|
||||
if ((Split-Path $runner -Parent) -ne (Split-Path $baselineRunner -Parent)) {
|
||||
throw "Prelabel and baseline runners must share one read-only mount"
|
||||
}
|
||||
$runtime = Assert-Directory (Resolve-Path -LiteralPath $RuntimeRoot).Path "Runtime root"
|
||||
$pack = Get-Content -LiteralPath (Join-Path $packRoot "manifest.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$pack.schema_version -ne "missioncore.perception-evaluation-pack/v1" -or
|
||||
$pack.generation_id -ne (Split-Path $packRoot -Leaf) -or
|
||||
$pack.identity.preprocessing_profile -ne "fixed-valid-fov-fill/v1"
|
||||
) {
|
||||
throw "Evaluation pack manifest is incompatible"
|
||||
}
|
||||
$null = Assert-RegularFile (Join-Path $validFov "manifest.json") "Valid-FOV manifest"
|
||||
$null = Assert-RegularFile (Join-Path $validFov "mask.png") "Valid-FOV mask"
|
||||
|
||||
$cacheRoot = Assert-Directory (Join-Path $runtime "cache\perception-p0-models-v1") "Model cache"
|
||||
$torchEnvironment = Assert-Directory (Join-Path $runtime "derived\perception-p0-env-v1") "Torch environment"
|
||||
$transformersEnvironment = Assert-Directory (Join-Path $runtime "derived\perception-p0-transformers4576-v1") "Transformers environment"
|
||||
$runnerRoot = Split-Path $runner -Parent
|
||||
$runnerName = Split-Path $runner -Leaf
|
||||
$derivedRoot = Join-Path $runtime "derived\evaluation-prelabels"
|
||||
$null = New-Item -ItemType Directory -Path $derivedRoot -Force
|
||||
$runToken = [Guid]::NewGuid().ToString("N")
|
||||
$publishRoot = Join-Path $derivedRoot (".{0}-{1}.publish" -f $pack.generation_id, $runToken)
|
||||
$stagingRoot = Join-Path $publishRoot "output"
|
||||
$null = New-Item -ItemType Directory -Path $publishRoot
|
||||
|
||||
try {
|
||||
$dockerArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", "none",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--shm-size", "2g",
|
||||
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
|
||||
"-e", "TORCH_HOME=/cache/torch",
|
||||
"-e", "HF_HOME=/cache/huggingface",
|
||||
"-e", "HF_HUB_OFFLINE=1",
|
||||
"-e", "TRANSFORMERS_OFFLINE=1",
|
||||
"-v", ((Convert-ToDockerPath $packRoot) + ":/evaluation-pack:ro"),
|
||||
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
||||
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
|
||||
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:rw"),
|
||||
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3",
|
||||
"nvcr.io/nvidia/tritonserver:26.06-py3",
|
||||
("/runner/{0}" -f $runnerName),
|
||||
"--evaluation-pack", "/evaluation-pack",
|
||||
"--valid-fov-root", "/valid-fov",
|
||||
"--output", "/publish/output",
|
||||
"--cache", "/cache",
|
||||
"--telemetry-interval-seconds", "1"
|
||||
)
|
||||
Write-Output ("PHASE=e2-prelabels-start PACK={0}" -f $pack.generation_id)
|
||||
& docker @dockerArgs
|
||||
Assert-LastExitCode "E2 prelabel generation"
|
||||
|
||||
$result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$result.schema_version -ne "missioncore.perception-evaluation-prelabels/v1" -or
|
||||
$result.result_id -notmatch "^evaluation-prelabels-[a-f0-9]{64}$" -or
|
||||
$result.identity.evaluation_pack_id -ne $pack.generation_id
|
||||
) {
|
||||
throw "E2 prelabel result manifest is incompatible"
|
||||
}
|
||||
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
||||
if (Test-Path -LiteralPath $finalRoot) {
|
||||
throw "An immutable E2 prelabel result with the same identity already exists: $finalRoot"
|
||||
}
|
||||
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
|
||||
Remove-Item -LiteralPath $publishRoot -Force
|
||||
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
||||
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $publishRoot) {
|
||||
Remove-Item -LiteralPath $publishRoot -Recurse -Force
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$JobRoot,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RunnerPath,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$BaselineRunnerPath,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$QualificationManifest,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ValidFovRoot,
|
||||
|
||||
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
|
||||
|
||||
[string]$CalibrationSha256 = "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9",
|
||||
|
||||
[string]$CalibrationSlot = "camera_1"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode {
|
||||
param([string]$Operation)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-RegularFile {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if (-not $item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
return $item.FullName
|
||||
}
|
||||
throw "$Label is not a regular file"
|
||||
}
|
||||
|
||||
function Assert-Directory {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if ($item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
return $item.FullName
|
||||
}
|
||||
throw "$Label is not a real directory"
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath {
|
||||
param([string]$Path)
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
$jobDirectory = Assert-Directory (Resolve-Path -LiteralPath $JobRoot).Path "Job root"
|
||||
$runner = Assert-RegularFile (Resolve-Path -LiteralPath $RunnerPath).Path "Qualification runner"
|
||||
$baselineRunner = Assert-RegularFile (Resolve-Path -LiteralPath $BaselineRunnerPath).Path "Baseline runner"
|
||||
$qualification = Assert-RegularFile (Resolve-Path -LiteralPath $QualificationManifest).Path "Qualification manifest"
|
||||
$validFov = Assert-Directory (Resolve-Path -LiteralPath $ValidFovRoot).Path "Valid-FOV root"
|
||||
$null = Assert-RegularFile (Join-Path $validFov "manifest.json") "Valid-FOV manifest"
|
||||
$null = Assert-RegularFile (Join-Path $validFov "mask.png") "Valid-FOV mask"
|
||||
if ((Split-Path $runner -Parent) -ne (Split-Path $baselineRunner -Parent)) {
|
||||
throw "Qualification and baseline runners must share one read-only mount"
|
||||
}
|
||||
$runtime = Assert-Directory (Resolve-Path -LiteralPath $RuntimeRoot).Path "Runtime root"
|
||||
$job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json
|
||||
if ($job.schema_version -ne "missioncore.compute-job/v1" -or $job.job_id -ne (Split-Path $jobDirectory -Leaf)) {
|
||||
throw "Compute job manifest is incompatible"
|
||||
}
|
||||
if ($CalibrationSha256 -notmatch "^[a-f0-9]{64}$" -or $CalibrationSlot -notmatch "^[A-Za-z0-9._-]+$") {
|
||||
throw "Calibration binding is invalid"
|
||||
}
|
||||
|
||||
$sourceId = [string]$job.input.source_id
|
||||
$epoch = [int]$job.input.codec_epoch
|
||||
$frameCount = [int]$job.input.segment_count
|
||||
$timelineStart = [double]$job.input.timeline.start_seconds
|
||||
$timelineEnd = [double]$job.input.timeline.end_seconds
|
||||
$timelineDuration = $timelineEnd - $timelineStart
|
||||
if ($frameCount -lt 1 -or $timelineDuration -le 0) {
|
||||
throw "Compute job frame/timeline contract is invalid"
|
||||
}
|
||||
$qualificationDocument = Get-Content -LiteralPath $qualification -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$qualificationDocument.schema_version -ne "missioncore.recorded-qualification-slice/v1" -or
|
||||
$qualificationDocument.identity.job_id -ne $job.job_id -or
|
||||
$qualificationDocument.identity.input_sha256 -ne $job.input_sha256
|
||||
) {
|
||||
throw "Qualification manifest is not bound to this job"
|
||||
}
|
||||
Write-Output ("PHASE=inputs-validated QUALIFICATION_FRAMES={0}" -f @($qualificationDocument.frames).Count)
|
||||
|
||||
$epochRoot = Join-Path $jobDirectory ("input\camera\{0}\epoch-{1}" -f $sourceId, $epoch)
|
||||
$epochRoot = Assert-Directory $epochRoot "Camera epoch"
|
||||
$initPath = Assert-RegularFile (Join-Path $epochRoot "init.mp4") "Camera init"
|
||||
$segmentsRoot = Assert-Directory (Join-Path $epochRoot "segments") "Camera segments"
|
||||
$derivedRoot = Join-Path $runtime "derived\qualification"
|
||||
$tmpRoot = Join-Path $runtime "tmp"
|
||||
$cacheRoot = Assert-Directory (Join-Path $runtime "cache\perception-p0-models-v1") "Model cache"
|
||||
$torchEnvironment = Assert-Directory (Join-Path $runtime "derived\perception-p0-env-v1") "Torch environment"
|
||||
$transformersEnvironment = Assert-Directory (Join-Path $runtime "derived\perception-p0-transformers4576-v1") "Transformers environment"
|
||||
$runnerRoot = Split-Path $runner -Parent
|
||||
$runnerName = Split-Path $runner -Leaf
|
||||
$baselineRunnerName = Split-Path $baselineRunner -Leaf
|
||||
$qualificationRoot = Split-Path $qualification -Parent
|
||||
$qualificationName = Split-Path $qualification -Leaf
|
||||
$null = New-Item -ItemType Directory -Path $derivedRoot -Force
|
||||
$null = New-Item -ItemType Directory -Path $tmpRoot -Force
|
||||
|
||||
$preflightArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", "none",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
|
||||
"-e", "TORCH_HOME=/cache/torch",
|
||||
"-e", "HF_HOME=/cache/huggingface",
|
||||
"-e", "HF_HUB_OFFLINE=1",
|
||||
"-e", "TRANSFORMERS_OFFLINE=1",
|
||||
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
||||
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:rw"),
|
||||
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3",
|
||||
"nvcr.io/nvidia/tritonserver:26.06-py3",
|
||||
("/runner/{0}" -f $baselineRunnerName), "preflight",
|
||||
"--job", "/job",
|
||||
"--cache", "/cache"
|
||||
)
|
||||
Write-Output "PHASE=worker-preflight-started"
|
||||
& docker @preflightArgs
|
||||
Assert-LastExitCode "Worker preflight"
|
||||
Write-Output "PHASE=worker-preflight-complete"
|
||||
|
||||
$runToken = [Guid]::NewGuid().ToString("N")
|
||||
$workRoot = Join-Path $tmpRoot ("{0}-e1-{1}" -f $job.job_id, $runToken)
|
||||
$framesRoot = Join-Path $workRoot "frames"
|
||||
$streamPath = Join-Path $workRoot "camera.mp4"
|
||||
$ptsPath = Join-Path $workRoot "pts.json"
|
||||
$timelinePath = Join-Path $workRoot "timeline.jsonl"
|
||||
$publishRoot = Join-Path $derivedRoot (".{0}-e1-{1}.publish" -f $job.job_id, $runToken)
|
||||
$stagingRoot = Join-Path $publishRoot "output"
|
||||
$null = New-Item -ItemType Directory -Path $framesRoot
|
||||
$null = New-Item -ItemType Directory -Path $publishRoot
|
||||
Write-Output "PHASE=private-staging-created"
|
||||
|
||||
try {
|
||||
$stream = [IO.File]::Open($streamPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
||||
try {
|
||||
$input = [IO.File]::OpenRead($initPath)
|
||||
try {
|
||||
$input.CopyTo($stream)
|
||||
}
|
||||
finally {
|
||||
$input.Dispose()
|
||||
}
|
||||
for ($sequence = 1; $sequence -le $frameCount; $sequence++) {
|
||||
$path = Assert-RegularFile (Join-Path $segmentsRoot ("{0}.m4s" -f $sequence)) "Camera segment"
|
||||
$input = [IO.File]::OpenRead($path)
|
||||
try {
|
||||
$input.CopyTo($stream)
|
||||
}
|
||||
finally {
|
||||
$input.Dispose()
|
||||
}
|
||||
if ($sequence % 500 -eq 0 -or $sequence -eq $frameCount) {
|
||||
Write-Output ("PHASE=stream-reconstruction SEGMENTS={0}/{1}" -f $sequence, $frameCount)
|
||||
}
|
||||
}
|
||||
$stream.Flush($true)
|
||||
}
|
||||
finally {
|
||||
$stream.Dispose()
|
||||
}
|
||||
|
||||
Write-Output "PHASE=frame-extraction-started"
|
||||
& ffmpeg -hide_banner -loglevel fatal -i $streamPath -map 0:v:0 -fps_mode passthrough (Join-Path $framesRoot "frame-%06d.png")
|
||||
Assert-LastExitCode "Camera extraction"
|
||||
& ffprobe -v error -select_streams v:0 -show_entries frame=best_effort_timestamp_time -of json $streamPath | Set-Content -LiteralPath $ptsPath -Encoding utf8
|
||||
Assert-LastExitCode "Camera timestamp probe"
|
||||
|
||||
$decodedFrames = @(Get-ChildItem -LiteralPath $framesRoot -File -Filter "frame-*.png")
|
||||
$ptsDocument = Get-Content -LiteralPath $ptsPath -Raw | ConvertFrom-Json
|
||||
$pts = @($ptsDocument.frames)
|
||||
if ($decodedFrames.Count -ne $frameCount -or $pts.Count -ne $frameCount) {
|
||||
throw "Decoded frame count differs from the compute job"
|
||||
}
|
||||
$firstEpochSeconds = [double]::Parse(
|
||||
([string]$pts[0].best_effort_timestamp_time).Trim(),
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
)
|
||||
$previousEpochSeconds = -1.0
|
||||
$timelineWriter = [IO.StreamWriter]::new($timelinePath, $false, [Text.UTF8Encoding]::new($false))
|
||||
try {
|
||||
for ($index = 0; $index -lt $frameCount; $index++) {
|
||||
$epochSeconds = [double]::Parse(
|
||||
([string]$pts[$index].best_effort_timestamp_time).Trim(),
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
) - $firstEpochSeconds
|
||||
if ($epochSeconds -le $previousEpochSeconds -or $epochSeconds -gt ($timelineDuration + 0.001)) {
|
||||
throw "Decoded camera timestamps are not strictly monotonic inside the compute job timeline"
|
||||
}
|
||||
$row = [ordered]@{
|
||||
frame_index = $index
|
||||
epoch_seconds = $epochSeconds
|
||||
session_seconds = $timelineStart + $epochSeconds
|
||||
}
|
||||
$timelineWriter.WriteLine(($row | ConvertTo-Json -Compress))
|
||||
$previousEpochSeconds = $epochSeconds
|
||||
}
|
||||
$timelineWriter.Flush()
|
||||
}
|
||||
finally {
|
||||
$timelineWriter.Dispose()
|
||||
}
|
||||
Write-Output ("PHASE=frame-extraction-complete FRAMES={0}" -f $frameCount)
|
||||
|
||||
$dockerArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", "none",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--shm-size", "2g",
|
||||
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
|
||||
"-e", "TORCH_HOME=/cache/torch",
|
||||
"-e", "HF_HOME=/cache/huggingface",
|
||||
"-e", "HF_HUB_OFFLINE=1",
|
||||
"-e", "TRANSFORMERS_OFFLINE=1",
|
||||
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
||||
"-v", ((Convert-ToDockerPath $framesRoot) + ":/frames:ro"),
|
||||
"-v", ((Convert-ToDockerPath $workRoot) + ":/work:ro"),
|
||||
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
|
||||
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:rw"),
|
||||
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"-v", ((Convert-ToDockerPath $qualificationRoot) + ":/qualification:ro"),
|
||||
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
||||
"--entrypoint", "python3",
|
||||
"nvcr.io/nvidia/tritonserver:26.06-py3",
|
||||
("/runner/{0}" -f $runnerName),
|
||||
"--job", "/job",
|
||||
"--frames", "/frames",
|
||||
"--timeline", "/work/timeline.jsonl",
|
||||
"--qualification", ("/qualification/{0}" -f $qualificationName),
|
||||
"--valid-fov-root", "/valid-fov",
|
||||
"--output", "/publish/output",
|
||||
"--cache", "/cache",
|
||||
"--calibration-sha256", $CalibrationSha256,
|
||||
"--calibration-slot", $CalibrationSlot,
|
||||
"--telemetry-interval-seconds", "1"
|
||||
)
|
||||
& docker @dockerArgs
|
||||
Assert-LastExitCode "Preprocessing qualification"
|
||||
|
||||
$result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$result.schema_version -ne "missioncore.perception-preprocessing-qualification-result/v1" -or
|
||||
$result.result_id -notmatch "^qualification-result-[a-f0-9]{64}$"
|
||||
) {
|
||||
throw "Qualification result manifest is incompatible"
|
||||
}
|
||||
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
||||
if (Test-Path -LiteralPath $finalRoot) {
|
||||
throw "An immutable qualification result with the same identity already exists: $finalRoot"
|
||||
}
|
||||
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
|
||||
Remove-Item -LiteralPath $publishRoot -Force
|
||||
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
||||
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $workRoot) {
|
||||
Remove-Item -LiteralPath $workRoot -Recurse -Force
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$JobRoot,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RunnerPath,
|
||||
|
||||
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
|
||||
|
||||
[string]$CalibrationSha256 = "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9",
|
||||
|
||||
[string]$CalibrationSlot = "camera_1"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode {
|
||||
param([string]$Operation)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-RegularFile {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if (-not $item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
return $item.FullName
|
||||
}
|
||||
throw "$Label is not a regular file"
|
||||
}
|
||||
|
||||
function Assert-Directory {
|
||||
param([string]$Path, [string]$Label)
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if ($item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
return $item.FullName
|
||||
}
|
||||
throw "$Label is not a real directory"
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath {
|
||||
param([string]$Path)
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
$jobDirectory = Assert-Directory (Resolve-Path -LiteralPath $JobRoot).Path "Job root"
|
||||
$runner = Assert-RegularFile (Resolve-Path -LiteralPath $RunnerPath).Path "Runner"
|
||||
$runtime = Assert-Directory (Resolve-Path -LiteralPath $RuntimeRoot).Path "Runtime root"
|
||||
$job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json
|
||||
if ($job.schema_version -ne "missioncore.compute-job/v1" -or $job.job_id -ne (Split-Path $jobDirectory -Leaf)) {
|
||||
throw "Compute job manifest is incompatible"
|
||||
}
|
||||
if ($CalibrationSha256 -notmatch "^[a-f0-9]{64}$" -or $CalibrationSlot -notmatch "^[A-Za-z0-9._-]+$") {
|
||||
throw "Calibration binding is invalid"
|
||||
}
|
||||
Write-Output "PHASE=job-manifest-validated"
|
||||
|
||||
$sourceId = [string]$job.input.source_id
|
||||
$epoch = [int]$job.input.codec_epoch
|
||||
$frameCount = [int]$job.input.segment_count
|
||||
$timelineStart = [double]$job.input.timeline.start_seconds
|
||||
$timelineEnd = [double]$job.input.timeline.end_seconds
|
||||
$timelineDuration = $timelineEnd - $timelineStart
|
||||
if ($frameCount -lt 1 -or $timelineDuration -le 0) {
|
||||
throw "Compute job frame/timeline contract is invalid"
|
||||
}
|
||||
|
||||
$epochRoot = Join-Path $jobDirectory ("input\camera\{0}\epoch-{1}" -f $sourceId, $epoch)
|
||||
$epochRoot = Assert-Directory $epochRoot "Camera epoch"
|
||||
$initPath = Assert-RegularFile (Join-Path $epochRoot "init.mp4") "Camera init"
|
||||
$segmentsRoot = Assert-Directory (Join-Path $epochRoot "segments") "Camera segments"
|
||||
$derivedRoot = Join-Path $runtime "derived"
|
||||
$tmpRoot = Join-Path $runtime "tmp"
|
||||
$cacheRoot = Assert-Directory (Join-Path $runtime "cache\perception-p0-models-v1") "Model cache"
|
||||
$torchEnvironment = Assert-Directory (Join-Path $derivedRoot "perception-p0-env-v1") "Torch environment"
|
||||
$transformersEnvironment = Assert-Directory (Join-Path $derivedRoot "perception-p0-transformers4576-v1") "Transformers environment"
|
||||
$runnerRoot = Split-Path $runner -Parent
|
||||
$runnerName = Split-Path $runner -Leaf
|
||||
$null = New-Item -ItemType Directory -Path $derivedRoot -Force
|
||||
$null = New-Item -ItemType Directory -Path $tmpRoot -Force
|
||||
|
||||
$preflightArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", "none",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
|
||||
"-e", "TORCH_HOME=/cache/torch",
|
||||
"-e", "HF_HOME=/cache/huggingface",
|
||||
"-e", "HF_HUB_OFFLINE=1",
|
||||
"-e", "TRANSFORMERS_OFFLINE=1",
|
||||
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
||||
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:rw"),
|
||||
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3",
|
||||
"nvcr.io/nvidia/tritonserver:26.06-py3",
|
||||
("/runner/{0}" -f $runnerName), "preflight",
|
||||
"--job", "/job",
|
||||
"--cache", "/cache"
|
||||
)
|
||||
Write-Output "PHASE=worker-preflight-started"
|
||||
& docker @preflightArgs
|
||||
Assert-LastExitCode "Worker preflight"
|
||||
Write-Output "PHASE=worker-preflight-complete"
|
||||
|
||||
$runToken = [Guid]::NewGuid().ToString("N")
|
||||
$workRoot = Join-Path $tmpRoot ("{0}-panoptic-{1}" -f $job.job_id, $runToken)
|
||||
$framesRoot = Join-Path $workRoot "frames"
|
||||
$streamPath = Join-Path $workRoot "camera.mp4"
|
||||
$ptsPath = Join-Path $workRoot "pts.json"
|
||||
$timelinePath = Join-Path $workRoot "timeline.jsonl"
|
||||
$publishRoot = Join-Path $derivedRoot (".{0}-panoptic-{1}.publish" -f $job.job_id, $runToken)
|
||||
$stagingRoot = Join-Path $publishRoot "output"
|
||||
$null = New-Item -ItemType Directory -Path $framesRoot
|
||||
$null = New-Item -ItemType Directory -Path $publishRoot
|
||||
$totalWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
Write-Output "PHASE=private-staging-created"
|
||||
|
||||
try {
|
||||
$stream = [IO.File]::Open($streamPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
||||
try {
|
||||
foreach ($path in @($initPath)) {
|
||||
$input = [IO.File]::OpenRead($path)
|
||||
try {
|
||||
$input.CopyTo($stream)
|
||||
}
|
||||
finally {
|
||||
$input.Dispose()
|
||||
}
|
||||
}
|
||||
for ($sequence = 1; $sequence -le $frameCount; $sequence++) {
|
||||
$path = Assert-RegularFile (Join-Path $segmentsRoot ("{0}.m4s" -f $sequence)) "Camera segment"
|
||||
$input = [IO.File]::OpenRead($path)
|
||||
try {
|
||||
$input.CopyTo($stream)
|
||||
}
|
||||
finally {
|
||||
$input.Dispose()
|
||||
}
|
||||
if ($sequence % 500 -eq 0 -or $sequence -eq $frameCount) {
|
||||
Write-Output ("PHASE=stream-reconstruction SEGMENTS={0}/{1}" -f $sequence, $frameCount)
|
||||
}
|
||||
}
|
||||
$stream.Flush($true)
|
||||
}
|
||||
finally {
|
||||
$stream.Dispose()
|
||||
}
|
||||
|
||||
$extractWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
Write-Output "PHASE=frame-extraction-started"
|
||||
# This archive contains valid strictly-increasing PTS but image2 rounds DTS
|
||||
# to its own time base and reports harmless duplicate-DTS diagnostics. Exact
|
||||
# frame-count and JSON timestamp checks below are the admission boundary.
|
||||
& ffmpeg -hide_banner -loglevel fatal -i $streamPath -map 0:v:0 -fps_mode passthrough (Join-Path $framesRoot "frame-%06d.png")
|
||||
Assert-LastExitCode "Camera extraction"
|
||||
& ffprobe -v error -select_streams v:0 -show_entries frame=best_effort_timestamp_time -of json $streamPath | Set-Content -LiteralPath $ptsPath -Encoding utf8
|
||||
Assert-LastExitCode "Camera timestamp probe"
|
||||
|
||||
$decodedFrames = @(Get-ChildItem -LiteralPath $framesRoot -File -Filter "frame-*.png")
|
||||
$ptsDocument = Get-Content -LiteralPath $ptsPath -Raw | ConvertFrom-Json
|
||||
$pts = @($ptsDocument.frames)
|
||||
if ($decodedFrames.Count -ne $frameCount -or $pts.Count -ne $frameCount) {
|
||||
throw "Decoded frame count differs from the compute job"
|
||||
}
|
||||
$firstEpochSeconds = [double]::Parse(
|
||||
([string]$pts[0].best_effort_timestamp_time).Trim(),
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
)
|
||||
$previousEpochSeconds = -1.0
|
||||
$timelineWriter = [IO.StreamWriter]::new($timelinePath, $false, [Text.UTF8Encoding]::new($false))
|
||||
try {
|
||||
for ($index = 0; $index -lt $frameCount; $index++) {
|
||||
$epochSeconds = [double]::Parse(
|
||||
([string]$pts[$index].best_effort_timestamp_time).Trim(),
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
) - $firstEpochSeconds
|
||||
if ($epochSeconds -le $previousEpochSeconds -or $epochSeconds -gt ($timelineDuration + 0.001)) {
|
||||
throw "Decoded camera timestamps are not strictly monotonic inside the compute job timeline"
|
||||
}
|
||||
$sessionSeconds = $timelineStart + $epochSeconds
|
||||
$row = [ordered]@{
|
||||
frame_index = $index
|
||||
epoch_seconds = $epochSeconds
|
||||
session_seconds = $sessionSeconds
|
||||
}
|
||||
$timelineWriter.WriteLine(($row | ConvertTo-Json -Compress))
|
||||
$previousEpochSeconds = $epochSeconds
|
||||
}
|
||||
$timelineWriter.Flush()
|
||||
}
|
||||
finally {
|
||||
$timelineWriter.Dispose()
|
||||
}
|
||||
$extractWatch.Stop()
|
||||
Write-Output ("PHASE=frame-extraction-complete FRAMES={0}" -f $frameCount)
|
||||
|
||||
$dockerArgs = @(
|
||||
"run", "--rm", "--gpus", "all", "--network", "none",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--shm-size", "2g",
|
||||
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
|
||||
"-e", "TORCH_HOME=/cache/torch",
|
||||
"-e", "HF_HOME=/cache/huggingface",
|
||||
"-e", "HF_HUB_OFFLINE=1",
|
||||
"-e", "TRANSFORMERS_OFFLINE=1",
|
||||
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
||||
"-v", ((Convert-ToDockerPath $framesRoot) + ":/frames:ro"),
|
||||
"-v", ((Convert-ToDockerPath $workRoot) + ":/work:ro"),
|
||||
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
|
||||
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:rw"),
|
||||
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
||||
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3",
|
||||
"nvcr.io/nvidia/tritonserver:26.06-py3",
|
||||
("/runner/{0}" -f $runnerName), "run",
|
||||
"--job", "/job",
|
||||
"--frames", "/frames",
|
||||
"--timeline", "/work/timeline.jsonl",
|
||||
"--output", "/publish/output",
|
||||
"--cache", "/cache",
|
||||
"--calibration-sha256", $CalibrationSha256,
|
||||
"--calibration-slot", $CalibrationSlot,
|
||||
"--telemetry-interval-seconds", "1"
|
||||
)
|
||||
& docker @dockerArgs
|
||||
Assert-LastExitCode "Panoptic inference"
|
||||
Write-Output "PHASE=panoptic-inference-complete"
|
||||
|
||||
$videoPath = Join-Path $stagingRoot "perception.mp4"
|
||||
$encodeWatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
$averageFps = $frameCount / $timelineDuration
|
||||
$fpsText = $averageFps.ToString("0.#########", [Globalization.CultureInfo]::InvariantCulture)
|
||||
$durationText = $timelineDuration.ToString("0.#########", [Globalization.CultureInfo]::InvariantCulture)
|
||||
& ffmpeg -hide_banner -loglevel error -framerate $fpsText -i (Join-Path $stagingRoot "overlay-frames\frame-%06d.png") -t $durationText -c:v h264_nvenc -preset p4 -tune hq -rc vbr -cq 21 -b:v 0 -pix_fmt yuv420p -movflags +faststart $videoPath
|
||||
Assert-LastExitCode "Panoptic video encoding"
|
||||
$encodeWatch.Stop()
|
||||
Write-Output "PHASE=video-encoding-complete"
|
||||
|
||||
$masksPath = Join-Path $stagingRoot "masks.tar.gz"
|
||||
& tar.exe -czf $masksPath -C $stagingRoot instance-masks semantic-masks
|
||||
Assert-LastExitCode "Mask archive publication"
|
||||
|
||||
$totalWatch.Stop()
|
||||
$finalizeArgs = @(
|
||||
"run", "--rm", "--network", "none",
|
||||
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"-v", ((Convert-ToDockerPath $stagingRoot) + ":/output:rw"),
|
||||
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
||||
"--entrypoint", "python3",
|
||||
"nvcr.io/nvidia/tritonserver:26.06-py3",
|
||||
("/runner/{0}" -f $runnerName), "finalize",
|
||||
"--output", "/output",
|
||||
"--video", "/output/perception.mp4",
|
||||
"--masks", "/output/masks.tar.gz",
|
||||
"--extract-seconds", $extractWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
|
||||
"--encode-seconds", $encodeWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
|
||||
"--wall-seconds", $totalWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
|
||||
"--encoder", "ffmpeg-h264_nvenc-p4-cq21-yuv420p-faststart"
|
||||
)
|
||||
& docker @finalizeArgs
|
||||
Assert-LastExitCode "Result finalization"
|
||||
|
||||
$result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json
|
||||
if ($result.schema_version -ne "missioncore.recorded-perception-result/v2" -or $result.result_id -notmatch "^result-[a-f0-9]{64}$") {
|
||||
throw "Final result manifest is incompatible"
|
||||
}
|
||||
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
||||
if (Test-Path -LiteralPath $finalRoot) {
|
||||
throw "An immutable result with the same identity already exists: $finalRoot"
|
||||
}
|
||||
|
||||
foreach ($temporaryChild in @("overlay-frames", "instance-masks", "semantic-masks")) {
|
||||
$temporaryPath = Join-Path $stagingRoot $temporaryChild
|
||||
if (Test-Path -LiteralPath $temporaryPath) {
|
||||
Remove-Item -LiteralPath $temporaryPath -Recurse -Force
|
||||
}
|
||||
}
|
||||
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
|
||||
Remove-Item -LiteralPath $publishRoot -Force
|
||||
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
||||
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $workRoot) {
|
||||
Remove-Item -LiteralPath $workRoot -Recurse -Force
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
# Mission Core CVAT D-only profile
|
||||
|
||||
This profile installs the annotation control plane in a dedicated WSL2
|
||||
distribution named `MissionCore-CVAT`. Its VHDX, Docker image store, CVAT
|
||||
source, persistent volumes, reports, and imported datasets live below
|
||||
`D:\NDC_MISSIONCORE`. It does not use the Docker Desktop image store that backs
|
||||
the Triton, Frigate, and Ollama containers.
|
||||
|
||||
Pinned inputs:
|
||||
|
||||
- Ubuntu 24.04.4 WSL AMD64 image, SHA-256
|
||||
`9b2f7730dc68227dd04a9f3e5eab86ad85caf556b8606ad94f1f29ff5c4fd3f5`
|
||||
- CVAT `v2.70.0`
|
||||
- D free-space floor: `360 GiB`
|
||||
- WSL VHD logical ceiling: `32 GB` (sparse allocation)
|
||||
|
||||
The official CVAT Compose topology is retained. The override only replaces its
|
||||
named volumes with explicit bind-backed directories inside the D-hosted WSL
|
||||
VHD. Images are pulled serially and the D free-space floor is checked before and
|
||||
after every image.
|
||||
|
||||
Run from the Windows host:
|
||||
|
||||
```powershell
|
||||
wsl -d MissionCore-CVAT -u root -- bash /mnt/d/NDC_MISSIONCORE/workspace/mission-core-compute/cvat/provision_cvat_wsl.sh
|
||||
wsl -d MissionCore-CVAT -u root -- bash /mnt/d/NDC_MISSIONCORE/workspace/mission-core-compute/cvat/ensure_cvat_admin.sh
|
||||
wsl -d MissionCore-CVAT -u root -- bash /mnt/d/NDC_MISSIONCORE/workspace/mission-core-compute/cvat/import_e2_workspace.sh
|
||||
```
|
||||
|
||||
After a Windows reboot, start the existing deployment without pulling images or
|
||||
re-provisioning it:
|
||||
|
||||
```powershell
|
||||
& D:\NDC_MISSIONCORE\workspace\mission-core-compute\cvat\Start-Cvat.ps1
|
||||
```
|
||||
|
||||
The launcher checks the `360 GiB` D-drive floor, starts the pinned Compose
|
||||
topology in the D-hosted WSL distribution, waits for the API, and keeps the WSL
|
||||
runtime alive. It writes startup logs only below
|
||||
`D:\NDC_MISSIONCORE\runtime\annotation\cvat\logs`.
|
||||
|
||||
From the Mission Core repository on the Mac, start CVAT if needed, discover the
|
||||
current WSL address, and open the SSH tunnel without a hard-coded IP:
|
||||
|
||||
```bash
|
||||
bash experiments/perception/worker/cvat/open_cvat_tunnel.sh
|
||||
```
|
||||
|
||||
Keep that terminal open and use `http://localhost:18080`. The same SSH session
|
||||
keeps the WSL runtime alive. Pass `--background` when detached runtime and
|
||||
tunnel sessions are preferred; rerun the helper after a Mac or Windows reboot.
|
||||
|
||||
The generated administrator password is stored only in
|
||||
`D:\NDC_MISSIONCORE\secrets\cvat\admin.env`; it is not printed by the script or
|
||||
committed to Git.
|
||||
|
||||
Traefik binds inside the dedicated WSL environment to `127.0.0.1:8080` and
|
||||
`127.0.0.1:8090`. It also publishes container port `8080` as WSL-internal port
|
||||
`18080` so the existing Windows SSH service can forward it without relying on
|
||||
Windows-to-WSL localhost forwarding. Remote review still uses the exact SSH
|
||||
host alias and a local forward; this profile does not add routes, Windows port
|
||||
proxies, DNS changes, or firewall rules.
|
||||
@@ -0,0 +1,58 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[int]$TimeoutSeconds = 180
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$Distro = 'MissionCore-CVAT'
|
||||
$Root = 'D:\NDC_MISSIONCORE'
|
||||
$RuntimeRoot = Join-Path $Root 'runtime\annotation\cvat'
|
||||
$LogRoot = Join-Path $RuntimeRoot 'logs'
|
||||
$LinuxStartScript = '/mnt/d/NDC_MISSIONCORE/workspace/mission-core-compute/cvat/start_cvat_runtime.sh'
|
||||
$FreeGiBFloor = 360
|
||||
|
||||
$drive = Get-PSDrive -Name D
|
||||
$freeGiB = [math]::Floor($drive.Free / 1GB)
|
||||
if ($freeGiB -lt $FreeGiBFloor) {
|
||||
throw "D: free-space floor crossed ($freeGiB GiB free; $FreeGiBFloor GiB required)"
|
||||
}
|
||||
|
||||
try {
|
||||
$about = Invoke-RestMethod -Uri 'http://localhost:8080/api/server/about' -TimeoutSec 3
|
||||
if ($about.version -eq '2.70.0') {
|
||||
Write-Output "CVAT already ready version=$($about.version) free_gib=$freeGiB"
|
||||
exit 0
|
||||
}
|
||||
} catch {
|
||||
# A stopped WSL distribution is the normal condition after a Windows reboot.
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Path $LogRoot -Force | Out-Null
|
||||
$timestamp = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssZ')
|
||||
$stdoutPath = Join-Path $LogRoot "start-$timestamp.stdout.log"
|
||||
$stderrPath = Join-Path $LogRoot "start-$timestamp.stderr.log"
|
||||
$arguments = @(
|
||||
'-d', $Distro,
|
||||
'-u', 'root',
|
||||
'--', 'bash', $LinuxStartScript, '--keepalive'
|
||||
)
|
||||
|
||||
Start-Process -FilePath 'wsl.exe' -ArgumentList $arguments -WindowStyle Hidden `
|
||||
-RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
Start-Sleep -Seconds 2
|
||||
try {
|
||||
$about = Invoke-RestMethod -Uri 'http://localhost:8080/api/server/about' -TimeoutSec 3
|
||||
if ($about.version -eq '2.70.0') {
|
||||
Write-Output "CVAT ready version=$($about.version) free_gib=$freeGiB"
|
||||
Write-Output "startup_log=$stdoutPath"
|
||||
exit 0
|
||||
}
|
||||
} catch {
|
||||
# Continue until the complete CVAT Compose topology is ready.
|
||||
}
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
|
||||
throw "CVAT did not become ready within $TimeoutSeconds seconds; inspect $stderrPath"
|
||||
@@ -0,0 +1,50 @@
|
||||
services:
|
||||
traefik:
|
||||
ports: !override
|
||||
- 127.0.0.1:8080:8080
|
||||
- 127.0.0.1:8090:8090
|
||||
- 0.0.0.0:18080:8080
|
||||
|
||||
volumes:
|
||||
cvat_db:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: none
|
||||
o: bind
|
||||
device: /srv/mission-core-cvat/volumes/cvat_db
|
||||
cvat_data:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: none
|
||||
o: bind
|
||||
device: /srv/mission-core-cvat/volumes/cvat_data
|
||||
cvat_keys:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: none
|
||||
o: bind
|
||||
device: /srv/mission-core-cvat/volumes/cvat_keys
|
||||
cvat_logs:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: none
|
||||
o: bind
|
||||
device: /srv/mission-core-cvat/volumes/cvat_logs
|
||||
cvat_inmem_db:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: none
|
||||
o: bind
|
||||
device: /srv/mission-core-cvat/volumes/cvat_inmem_db
|
||||
cvat_events_db:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: none
|
||||
o: bind
|
||||
device: /srv/mission-core-cvat/volumes/cvat_events_db
|
||||
cvat_cache_db:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: none
|
||||
o: bind
|
||||
device: /srv/mission-core-cvat/volumes/cvat_cache_db
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
|
||||
readonly SECRET_ROOT="/mnt/d/NDC_MISSIONCORE/secrets/cvat"
|
||||
readonly SECRET_FILE="${SECRET_ROOT}/admin.env"
|
||||
|
||||
if [[ "${EUID}" -ne 0 ]]; then
|
||||
echo "ensure_cvat_admin.sh must run as root" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
install -d -m 0700 "${SECRET_ROOT}"
|
||||
if [[ ! -f "${SECRET_FILE}" ]]; then
|
||||
umask 077
|
||||
password="$(openssl rand -hex 24)"
|
||||
{
|
||||
printf 'CVAT_ADMIN_USERNAME=missioncore\n'
|
||||
printf 'CVAT_ADMIN_EMAIL=missioncore@local.invalid\n'
|
||||
printf 'CVAT_ADMIN_PASSWORD=%s\n' "${password}"
|
||||
} >"${SECRET_FILE}"
|
||||
fi
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "${SECRET_FILE}"
|
||||
set +a
|
||||
|
||||
docker exec \
|
||||
-e "CVAT_ADMIN_USERNAME=${CVAT_ADMIN_USERNAME}" \
|
||||
-e "CVAT_ADMIN_EMAIL=${CVAT_ADMIN_EMAIL}" \
|
||||
-e "CVAT_ADMIN_PASSWORD=${CVAT_ADMIN_PASSWORD}" \
|
||||
cvat_server \
|
||||
python3 /home/django/manage.py shell -c \
|
||||
'import os; from django.contrib.auth import get_user_model; User = get_user_model(); user, _ = User.objects.get_or_create(username=os.environ["CVAT_ADMIN_USERNAME"]); user.email = os.environ["CVAT_ADMIN_EMAIL"]; user.is_staff = True; user.is_superuser = True; user.set_password(os.environ["CVAT_ADMIN_PASSWORD"]); user.save()'
|
||||
|
||||
printf 'admin_ready=true\nusername=%s\nsecret_file=%s\n' \
|
||||
"${CVAT_ADMIN_USERNAME}" "${SECRET_FILE}"
|
||||
@@ -0,0 +1,284 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from cvat_sdk import make_client, models
|
||||
from cvat_sdk.api_client.exceptions import ServiceException
|
||||
from cvat_sdk.core.proxies.tasks import ResourceType
|
||||
|
||||
WORKSPACE_ID = (
|
||||
"annotation-workspace-"
|
||||
"9a950d1c37d56dc12cc285b13c5addd7795285879cbcb1fbb2d5811c3c69821a"
|
||||
)
|
||||
EVALUATION_PACK_ID = (
|
||||
"evaluation-pack-"
|
||||
"7a983bba75d46c7c260252cb2d461e1384dcb92cda9e164397e841e6ebb37789"
|
||||
)
|
||||
EXPECTED_FRAME_COUNT = 64
|
||||
EXPECTED_INSTANCE_COUNT = 775
|
||||
BACKGROUND_LABEL = {
|
||||
"name": "background",
|
||||
"color": "#000000",
|
||||
"attributes": [],
|
||||
}
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _label_specs(raw_labels: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
labels = [
|
||||
{
|
||||
"name": item["name"],
|
||||
"color": item["color"],
|
||||
"attributes": [],
|
||||
}
|
||||
for item in raw_labels
|
||||
]
|
||||
if not any(label["name"] == BACKGROUND_LABEL["name"] for label in labels):
|
||||
labels.insert(0, dict(BACKGROUND_LABEL))
|
||||
return labels
|
||||
|
||||
|
||||
def _annotation_counts(task: Any) -> dict[str, int]:
|
||||
annotations = task.get_annotations()
|
||||
return {
|
||||
"shape_count": len(annotations.shapes),
|
||||
"tag_count": len(annotations.tags),
|
||||
"track_count": len(annotations.tracks),
|
||||
}
|
||||
|
||||
|
||||
def _list_tasks_when_ready(client: Any, *, attempts: int = 60) -> list[Any]:
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
return list(client.tasks.list())
|
||||
except ServiceException as error:
|
||||
if error.status not in {500, 502, 503} or attempt == attempts:
|
||||
raise
|
||||
time.sleep(2)
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
def _task_record(task: Any, *, disposition: str, expected_labels: list[str]) -> dict[str, Any]:
|
||||
task.fetch()
|
||||
task_labels = list(task.get_labels())
|
||||
actual_labels = sorted(label.name for label in task_labels)
|
||||
if task.size != EXPECTED_FRAME_COUNT:
|
||||
raise RuntimeError(
|
||||
f"Task {task.id} has {task.size} frames, expected {EXPECTED_FRAME_COUNT}"
|
||||
)
|
||||
if actual_labels != sorted(expected_labels):
|
||||
raise RuntimeError(
|
||||
f"Task {task.id} labels differ: actual={actual_labels!r} "
|
||||
f"expected={sorted(expected_labels)!r}"
|
||||
)
|
||||
annotation_counts = _annotation_counts(task)
|
||||
label_names_by_id = {label.id: label.name for label in task_labels}
|
||||
annotations = task.get_annotations()
|
||||
shape_counts_by_label = dict(
|
||||
sorted(
|
||||
Counter(
|
||||
label_names_by_id.get(shape.label_id, f"unknown:{shape.label_id}")
|
||||
for shape in annotations.shapes
|
||||
).items()
|
||||
)
|
||||
)
|
||||
return {
|
||||
"id": task.id,
|
||||
"name": task.name,
|
||||
"disposition": disposition,
|
||||
"frame_count": task.size,
|
||||
"label_names": actual_labels,
|
||||
**annotation_counts,
|
||||
"shape_counts_by_label": shape_counts_by_label,
|
||||
"url_path": f"/tasks/{task.id}",
|
||||
}
|
||||
|
||||
|
||||
def _ensure_task(
|
||||
client: Any,
|
||||
*,
|
||||
name: str,
|
||||
labels: list[dict[str, Any]],
|
||||
images_path: Path,
|
||||
annotation_path: Path,
|
||||
annotation_format: str,
|
||||
expected_shape_count: int | None,
|
||||
) -> dict[str, Any]:
|
||||
matches = [task for task in _list_tasks_when_ready(client) if task.name == name]
|
||||
if len(matches) > 1:
|
||||
raise RuntimeError(f"Multiple CVAT tasks have the reserved name {name!r}")
|
||||
|
||||
expected_labels = [label["name"] for label in labels]
|
||||
if matches:
|
||||
task = matches[0]
|
||||
task.fetch()
|
||||
actual_labels = sorted(label.name for label in task.get_labels())
|
||||
expected_without_background = sorted(
|
||||
label for label in expected_labels if label != BACKGROUND_LABEL["name"]
|
||||
)
|
||||
disposition = "reused"
|
||||
if (
|
||||
actual_labels == expected_without_background
|
||||
and BACKGROUND_LABEL["name"] in expected_labels
|
||||
):
|
||||
task.update(
|
||||
models.PatchedTaskWriteRequest(
|
||||
labels=[models.PatchedLabelRequest(**BACKGROUND_LABEL)]
|
||||
)
|
||||
)
|
||||
disposition = "reused-and-background-label-added"
|
||||
|
||||
record = _task_record(
|
||||
task,
|
||||
disposition=disposition,
|
||||
expected_labels=expected_labels,
|
||||
)
|
||||
if record["shape_count"] == 0:
|
||||
task.import_annotations(annotation_format, annotation_path)
|
||||
record = _task_record(
|
||||
task,
|
||||
disposition=f"{disposition}-and-annotations-imported",
|
||||
expected_labels=expected_labels,
|
||||
)
|
||||
if expected_shape_count is not None and record["shape_count"] != expected_shape_count:
|
||||
raise RuntimeError(
|
||||
f"Task {task.id} has {record['shape_count']} shapes, "
|
||||
f"expected {expected_shape_count}"
|
||||
)
|
||||
return record
|
||||
|
||||
task = client.tasks.create_from_data(
|
||||
spec=models.TaskWriteRequest(
|
||||
name=name,
|
||||
labels=labels,
|
||||
segment_size=EXPECTED_FRAME_COUNT,
|
||||
overlap=0,
|
||||
),
|
||||
resources=[images_path],
|
||||
resource_type=ResourceType.LOCAL,
|
||||
data_params={
|
||||
"image_quality": 100,
|
||||
"sorting_method": "lexicographical",
|
||||
"use_cache": False,
|
||||
},
|
||||
annotation_path=annotation_path,
|
||||
annotation_format=annotation_format,
|
||||
status_check_period=2,
|
||||
)
|
||||
record = _task_record(task, disposition="created", expected_labels=expected_labels)
|
||||
if expected_shape_count is not None and record["shape_count"] != expected_shape_count:
|
||||
raise RuntimeError(
|
||||
f"Task {task.id} has {record['shape_count']} shapes, "
|
||||
f"expected {expected_shape_count}"
|
||||
)
|
||||
return record
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--server", required=True)
|
||||
parser.add_argument("--username", required=True)
|
||||
parser.add_argument("--password-env", required=True)
|
||||
parser.add_argument("--workspace", type=Path, required=True)
|
||||
parser.add_argument("--report", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
password = os.environ.get(args.password_env)
|
||||
if not password:
|
||||
raise RuntimeError(f"Password environment variable {args.password_env!r} is empty")
|
||||
|
||||
workspace = args.workspace.resolve()
|
||||
if workspace.name != WORKSPACE_ID:
|
||||
raise RuntimeError(f"Unexpected annotation workspace: {workspace}")
|
||||
|
||||
manifest_path = workspace / "manifest.json"
|
||||
labels_path = workspace / "cvat" / "labels.json"
|
||||
images_path = workspace / "cvat" / "images.zip"
|
||||
instance_path = workspace / "cvat" / "instance-coco.zip"
|
||||
semantic_path = workspace / "cvat" / "semantic-mask.zip"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
labels = json.loads(labels_path.read_text(encoding="utf-8"))
|
||||
|
||||
if manifest["ground_truth"] is not False:
|
||||
raise RuntimeError("LAB E2 import must remain an unreviewed model draft")
|
||||
if manifest["identity"]["frame_count"] != EXPECTED_FRAME_COUNT:
|
||||
raise RuntimeError("Unexpected LAB E2 frame count")
|
||||
if manifest["identity"]["draft_instance_count"] != EXPECTED_INSTANCE_COUNT:
|
||||
raise RuntimeError("Unexpected LAB E2 instance count")
|
||||
|
||||
task_specs = [
|
||||
{
|
||||
"name": "LAB E2 | K1 | instance prelabels | pack 7a983bba",
|
||||
"labels": _label_specs(labels["instance_task"]),
|
||||
"annotation_path": instance_path,
|
||||
"annotation_format": "COCO 1.0",
|
||||
"expected_shape_count": EXPECTED_INSTANCE_COUNT,
|
||||
},
|
||||
{
|
||||
"name": "LAB E2 | K1 | dense semantic prelabels | pack 7a983bba",
|
||||
"labels": _label_specs(labels["semantic_task"]),
|
||||
"annotation_path": semantic_path,
|
||||
"annotation_format": "Segmentation mask 1.1",
|
||||
"expected_shape_count": None,
|
||||
},
|
||||
]
|
||||
|
||||
with make_client(args.server, credentials=(args.username, password)) as client:
|
||||
client.check_server_version(fail_if_unsupported=True)
|
||||
task_records = [
|
||||
_ensure_task(
|
||||
client,
|
||||
name=task_spec["name"],
|
||||
labels=task_spec["labels"],
|
||||
images_path=images_path,
|
||||
annotation_path=task_spec["annotation_path"],
|
||||
annotation_format=task_spec["annotation_format"],
|
||||
expected_shape_count=task_spec["expected_shape_count"],
|
||||
)
|
||||
for task_spec in task_specs
|
||||
]
|
||||
|
||||
report = {
|
||||
"schema_version": "missioncore.lab-e2-cvat-import/v1",
|
||||
"created_at_utc": datetime.now(UTC).isoformat(timespec="milliseconds").replace(
|
||||
"+00:00", "Z"
|
||||
),
|
||||
"server": args.server,
|
||||
"cvat_version": "v2.70.0",
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"evaluation_pack_id": EVALUATION_PACK_ID,
|
||||
"workspace_manifest_sha256": _sha256(manifest_path),
|
||||
"ground_truth": False,
|
||||
"inputs": {
|
||||
"images_zip_sha256": _sha256(images_path),
|
||||
"instance_coco_zip_sha256": _sha256(instance_path),
|
||||
"semantic_mask_zip_sha256": _sha256(semantic_path),
|
||||
},
|
||||
"tasks": task_records,
|
||||
"next_gate": (
|
||||
"two-pass human review and reviewed export; "
|
||||
"do not treat drafts as accuracy evidence"
|
||||
),
|
||||
}
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
|
||||
readonly FREE_GIB_FLOOR="${FREE_GIB_FLOOR:-360}"
|
||||
readonly WINDOWS_ROOT="/mnt/d/NDC_MISSIONCORE"
|
||||
readonly COMPUTE_ROOT="${WINDOWS_ROOT}/workspace/mission-core-compute"
|
||||
readonly TOOL_ROOT="/srv/mission-core-cvat/tools/e2-import"
|
||||
readonly SECRET_FILE="${WINDOWS_ROOT}/secrets/cvat/admin.env"
|
||||
readonly WORKSPACE_ROOT="${WINDOWS_ROOT}/runtime/annotation/imports/LAB-E2/annotation-workspace-9a950d1c37d56dc12cc285b13c5addd7795285879cbcb1fbb2d5811c3c69821a"
|
||||
readonly REPORT_ROOT="${WINDOWS_ROOT}/runtime/annotation/cvat/reports"
|
||||
|
||||
if [[ "${EUID}" -ne 0 ]]; then
|
||||
echo "import_e2_workspace.sh must run as root" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
free_gib() {
|
||||
df -BG --output=avail "${WINDOWS_ROOT}" | tail -n 1 | tr -dc '0-9'
|
||||
}
|
||||
|
||||
guard_disk() {
|
||||
local stage="$1"
|
||||
local free
|
||||
free="$(free_gib)"
|
||||
printf 'disk_guard stage=%s free_gib=%s floor_gib=%s\n' \
|
||||
"${stage}" "${free}" "${FREE_GIB_FLOOR}"
|
||||
if (( free < FREE_GIB_FLOOR )); then
|
||||
echo "D: free-space floor crossed; refusing further writes" >&2
|
||||
exit 3
|
||||
fi
|
||||
}
|
||||
|
||||
wait_for_cvat() {
|
||||
local attempt http_code
|
||||
for attempt in $(seq 1 60); do
|
||||
http_code="$(
|
||||
curl --silent --output /dev/null --write-out '%{http_code}' \
|
||||
"http://localhost:8080/api/server/about" || true
|
||||
)"
|
||||
if [[ "${http_code}" == "200" ]]; then
|
||||
http_code="$(
|
||||
curl --silent --output /dev/null --write-out '%{http_code}' \
|
||||
"http://localhost:8080/api/tasks" || true
|
||||
)"
|
||||
if [[ "${http_code}" == "200" || "${http_code}" == "401" || "${http_code}" == "403" ]]; then
|
||||
printf 'cvat_ready attempt=%s tasks_http=%s\n' "${attempt}" "${http_code}"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "CVAT API did not become ready within 120 seconds" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
test -f "${SECRET_FILE}"
|
||||
test -f "${WORKSPACE_ROOT}/manifest.json"
|
||||
test -f "${COMPUTE_ROOT}/cvat/import_e2_workspace.py"
|
||||
guard_disk preflight
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
apt-get install -y python3-venv
|
||||
install -d -m 0750 "${TOOL_ROOT}" "${REPORT_ROOT}"
|
||||
|
||||
if [[ ! -x "${TOOL_ROOT}/venv/bin/python" ]]; then
|
||||
python3 -m venv "${TOOL_ROOT}/venv"
|
||||
"${TOOL_ROOT}/venv/bin/pip" install --disable-pip-version-check \
|
||||
"cvat-sdk==2.70.0"
|
||||
fi
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "${SECRET_FILE}"
|
||||
set +a
|
||||
|
||||
wait_for_cvat
|
||||
report="${REPORT_ROOT}/lab-e2-cvat-import-$(date -u +%Y%m%dT%H%M%SZ).json"
|
||||
"${TOOL_ROOT}/venv/bin/python" "${COMPUTE_ROOT}/cvat/import_e2_workspace.py" \
|
||||
--server "http://localhost:8080" \
|
||||
--username "${CVAT_ADMIN_USERNAME}" \
|
||||
--password-env CVAT_ADMIN_PASSWORD \
|
||||
--workspace "${WORKSPACE_ROOT}" \
|
||||
--report "${report}"
|
||||
|
||||
guard_disk imported
|
||||
printf 'import_report=%s\n' "${report}"
|
||||
@@ -0,0 +1,179 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from cvat_sdk import make_client
|
||||
from import_e2_workspace import _ensure_task, _label_specs, _sha256
|
||||
|
||||
WORKSPACE_SCHEMA = "missioncore.lab-e3-cvat-review-workspace/v1"
|
||||
IDENTITY_SCHEMA = "missioncore.lab-e3-cvat-review-identity/v1"
|
||||
EXPECTED_PACK_ID = (
|
||||
"evaluation-pack-"
|
||||
"7a983bba75d46c7c260252cb2d461e1384dcb92cda9e164397e841e6ebb37789"
|
||||
)
|
||||
EXPECTED_RESULT_ID = (
|
||||
"e3-segmentation-"
|
||||
"01bd497c44c2b940add145ec784d3418010327bce0baddd4420b0925317e8a16"
|
||||
)
|
||||
EXPECTED_FRAMES = 64
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode()
|
||||
|
||||
|
||||
def _valid_sha256(value: object) -> bool:
|
||||
return (
|
||||
isinstance(value, str)
|
||||
and len(value) == 64
|
||||
and all(character in "0123456789abcdef" for character in value)
|
||||
)
|
||||
|
||||
|
||||
def _read_object(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(value, dict):
|
||||
raise RuntimeError(f"JSON root is not an object: {path}")
|
||||
return value
|
||||
|
||||
|
||||
def _safe_artifact(root: Path, encoded: object) -> Path:
|
||||
if not isinstance(encoded, str):
|
||||
raise RuntimeError("artifact path is not a string")
|
||||
relative = PurePosixPath(encoded)
|
||||
if relative.is_absolute() or ".." in relative.parts or not relative.parts:
|
||||
raise RuntimeError("artifact path is unsafe")
|
||||
path = root.joinpath(*relative.parts).resolve(strict=True)
|
||||
if not path.is_file() or not path.is_relative_to(root):
|
||||
raise RuntimeError("artifact path escaped its root")
|
||||
return path
|
||||
|
||||
|
||||
def _workspace(root: Path, images_zip: Path) -> tuple[dict[str, Any], dict[str, Path]]:
|
||||
manifest_path = root / "manifest.json"
|
||||
manifest = _read_object(manifest_path)
|
||||
identity = manifest.get("identity")
|
||||
identity_sha256 = manifest.get("identity_sha256")
|
||||
if (
|
||||
manifest.get("schema_version") != WORKSPACE_SCHEMA
|
||||
or manifest.get("ground_truth") is not False
|
||||
or not isinstance(identity, dict)
|
||||
or identity.get("schema_version") != IDENTITY_SCHEMA
|
||||
or not _valid_sha256(identity_sha256)
|
||||
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
||||
or manifest.get("workspace_id") != f"e3-cvat-review-{identity_sha256}"
|
||||
or root.name != manifest.get("workspace_id")
|
||||
or identity.get("evaluation_pack_id") != EXPECTED_PACK_ID
|
||||
or identity.get("e3_result_id") != EXPECTED_RESULT_ID
|
||||
or identity.get("frame_count") != EXPECTED_FRAMES
|
||||
or _sha256(images_zip) != identity.get("images_zip_sha256")
|
||||
):
|
||||
raise RuntimeError("LAB E3 CVAT review workspace is incompatible")
|
||||
artifacts: dict[str, Path] = {}
|
||||
descriptors = manifest.get("artifacts")
|
||||
if not isinstance(descriptors, list):
|
||||
raise RuntimeError("LAB E3 CVAT artifact list is invalid")
|
||||
for descriptor in descriptors:
|
||||
if not isinstance(descriptor, dict):
|
||||
raise RuntimeError("LAB E3 CVAT artifact descriptor is invalid")
|
||||
path = _safe_artifact(root, descriptor.get("path"))
|
||||
if (
|
||||
path.stat().st_size != descriptor.get("bytes")
|
||||
or not _valid_sha256(descriptor.get("sha256"))
|
||||
or _sha256(path) != descriptor["sha256"]
|
||||
):
|
||||
raise RuntimeError(f"LAB E3 CVAT artifact changed: {path.name}")
|
||||
artifacts[path.name] = path
|
||||
if set(artifacts) != {
|
||||
"labels.json",
|
||||
"control-fisheye-mask.zip",
|
||||
"challenger-kb4-cubemap5.zip",
|
||||
}:
|
||||
raise RuntimeError("LAB E3 CVAT artifact set changed")
|
||||
return manifest, artifacts
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--server", required=True)
|
||||
parser.add_argument("--username", required=True)
|
||||
parser.add_argument("--password-env", required=True)
|
||||
parser.add_argument("--workspace", type=Path, required=True)
|
||||
parser.add_argument("--images-zip", type=Path, required=True)
|
||||
parser.add_argument("--report", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
password = os.environ.get(args.password_env)
|
||||
if not password:
|
||||
raise RuntimeError(f"Password environment variable {args.password_env!r} is empty")
|
||||
workspace = args.workspace.resolve(strict=True)
|
||||
images_zip = args.images_zip.resolve(strict=True)
|
||||
manifest, artifacts = _workspace(workspace, images_zip)
|
||||
labels_document = _read_object(artifacts["labels.json"])
|
||||
semantic_labels = labels_document.get("semantic_task")
|
||||
if not isinstance(semantic_labels, list):
|
||||
raise RuntimeError("semantic labels are absent")
|
||||
labels = _label_specs(semantic_labels)
|
||||
task_specs = (
|
||||
{
|
||||
"name": "LAB E3 | K1 | EoMT fisheye control | pack 7a983bba",
|
||||
"annotation": artifacts["control-fisheye-mask.zip"],
|
||||
"role": "control",
|
||||
},
|
||||
{
|
||||
"name": "LAB E3 | K1 | EoMT KB4 cubemap5 challenger | pack 7a983bba",
|
||||
"annotation": artifacts["challenger-kb4-cubemap5.zip"],
|
||||
"role": "challenger",
|
||||
},
|
||||
)
|
||||
with make_client(args.server, credentials=(args.username, password)) as client:
|
||||
client.check_server_version(fail_if_unsupported=True)
|
||||
tasks = []
|
||||
for spec in task_specs:
|
||||
record = _ensure_task(
|
||||
client,
|
||||
name=spec["name"],
|
||||
labels=labels,
|
||||
images_path=images_zip,
|
||||
annotation_path=spec["annotation"],
|
||||
annotation_format="Segmentation mask 1.1",
|
||||
expected_shape_count=None,
|
||||
)
|
||||
record["role"] = spec["role"]
|
||||
tasks.append(record)
|
||||
|
||||
report = {
|
||||
"schema_version": "missioncore.lab-e3-cvat-import/v1",
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"server": args.server,
|
||||
"cvat_version": "v2.70.0",
|
||||
"workspace_id": manifest["workspace_id"],
|
||||
"workspace_manifest_sha256": _sha256(workspace / "manifest.json"),
|
||||
"evaluation_pack_id": EXPECTED_PACK_ID,
|
||||
"e3_result_id": EXPECTED_RESULT_ID,
|
||||
"ground_truth": False,
|
||||
"priority_image_ids": manifest["identity"]["priority_image_ids"],
|
||||
"tasks": tasks,
|
||||
"next_gate": "two-pass human review and reviewed export",
|
||||
}
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
|
||||
readonly FREE_GIB_FLOOR="${FREE_GIB_FLOOR:-360}"
|
||||
readonly WINDOWS_ROOT="/mnt/d/NDC_MISSIONCORE"
|
||||
readonly COMPUTE_ROOT="${WINDOWS_ROOT}/workspace/mission-core-compute"
|
||||
readonly TOOL_ROOT="/srv/mission-core-cvat/tools/e2-import"
|
||||
readonly SECRET_FILE="${WINDOWS_ROOT}/secrets/cvat/admin.env"
|
||||
readonly WORKSPACE_ID="e3-cvat-review-ca599521e345446ca8e9af5a9013062099278e7317f83ff89740c6b092ddc52f"
|
||||
readonly WORKSPACE_ROOT="${WINDOWS_ROOT}/runtime/annotation/imports/LAB-E3/${WORKSPACE_ID}"
|
||||
readonly E2_WORKSPACE_ID="annotation-workspace-9a950d1c37d56dc12cc285b13c5addd7795285879cbcb1fbb2d5811c3c69821a"
|
||||
readonly IMAGES_ZIP="${WINDOWS_ROOT}/runtime/annotation/imports/LAB-E2/${E2_WORKSPACE_ID}/cvat/images.zip"
|
||||
readonly REPORT_ROOT="${WINDOWS_ROOT}/runtime/annotation/cvat/reports"
|
||||
|
||||
if [[ "${EUID}" -ne 0 ]]; then
|
||||
echo "import_e3_review.sh must run as root" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
free_gib() {
|
||||
df -BG --output=avail "${WINDOWS_ROOT}" | tail -n 1 | tr -dc '0-9'
|
||||
}
|
||||
|
||||
guard_disk() {
|
||||
local stage="$1"
|
||||
local free
|
||||
free="$(free_gib)"
|
||||
printf 'disk_guard stage=%s free_gib=%s floor_gib=%s\n' \
|
||||
"${stage}" "${free}" "${FREE_GIB_FLOOR}"
|
||||
if (( free < FREE_GIB_FLOOR )); then
|
||||
echo "D: free-space floor crossed; refusing further writes" >&2
|
||||
exit 3
|
||||
fi
|
||||
}
|
||||
|
||||
wait_for_cvat() {
|
||||
local attempt http_code
|
||||
for attempt in $(seq 1 60); do
|
||||
http_code="$(
|
||||
curl --silent --output /dev/null --write-out '%{http_code}' \
|
||||
"http://localhost:8080/api/server/about" || true
|
||||
)"
|
||||
if [[ "${http_code}" == "200" ]]; then
|
||||
printf 'cvat_ready attempt=%s\n' "${attempt}"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "CVAT API did not become ready within 120 seconds" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
test -x "${TOOL_ROOT}/venv/bin/python"
|
||||
test -f "${SECRET_FILE}"
|
||||
test -f "${WORKSPACE_ROOT}/manifest.json"
|
||||
test -f "${IMAGES_ZIP}"
|
||||
test -f "${COMPUTE_ROOT}/cvat/import_e2_workspace.py"
|
||||
test -f "${COMPUTE_ROOT}/cvat/import_e3_review.py"
|
||||
guard_disk preflight
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "${SECRET_FILE}"
|
||||
set +a
|
||||
|
||||
wait_for_cvat
|
||||
report="${REPORT_ROOT}/lab-e3-cvat-import-$(date -u +%Y%m%dT%H%M%SZ).json"
|
||||
"${TOOL_ROOT}/venv/bin/python" "${COMPUTE_ROOT}/cvat/import_e3_review.py" \
|
||||
--server "http://localhost:8080" \
|
||||
--username "${CVAT_ADMIN_USERNAME}" \
|
||||
--password-env CVAT_ADMIN_PASSWORD \
|
||||
--workspace "${WORKSPACE_ROOT}" \
|
||||
--images-zip "${IMAGES_ZIP}" \
|
||||
--report "${report}"
|
||||
|
||||
guard_disk imported
|
||||
printf 'import_report=%s\n' "${report}"
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly SSH_ALIAS="mission-gpu"
|
||||
readonly LOCAL_PORT="${CVAT_LOCAL_PORT:-18080}"
|
||||
readonly REMOTE_WSL_PORT="18080"
|
||||
readonly LINUX_START_SCRIPT='/mnt/d/NDC_MISSIONCORE/workspace/mission-core-compute/cvat/start_cvat_runtime.sh'
|
||||
|
||||
if curl --max-time 2 --fail --silent \
|
||||
"http://localhost:${LOCAL_PORT}/api/server/about" >/dev/null 2>&1; then
|
||||
printf 'cvat_tunnel_ready url=http://localhost:%s\n' "${LOCAL_PORT}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if command -v lsof >/dev/null \
|
||||
&& lsof -nP -iTCP:"${LOCAL_PORT}" -sTCP:LISTEN >/dev/null 2>&1; then
|
||||
echo "Local port ${LOCAL_PORT} is already occupied by a non-responsive process" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
runtime_command="wsl.exe -d MissionCore-CVAT -u root -- bash ${LINUX_START_SCRIPT} --keepalive"
|
||||
background=false
|
||||
if [[ "${1:-}" == "--background" ]]; then
|
||||
background=true
|
||||
ssh -f "${SSH_ALIAS}" "${runtime_command}"
|
||||
else
|
||||
ssh "${SSH_ALIAS}" "${runtime_command}" &
|
||||
runtime_ssh_pid=$!
|
||||
cleanup() {
|
||||
if [[ -n "${tunnel_ssh_pid:-}" ]]; then
|
||||
kill "${tunnel_ssh_pid}" 2>/dev/null || true
|
||||
fi
|
||||
kill "${runtime_ssh_pid}" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
fi
|
||||
|
||||
for attempt in $(seq 1 30); do
|
||||
cvat_ip="$(
|
||||
ssh "${SSH_ALIAS}" \
|
||||
"wsl.exe -d MissionCore-CVAT -u root -- hostname -I" \
|
||||
| tr -d '\r' | awk '{print $1}'
|
||||
)"
|
||||
if [[ "${cvat_ip}" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [[ ! "${cvat_ip:-}" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "MissionCore-CVAT did not return a valid WSL address" >&2
|
||||
if [[ "${background}" == false ]]; then
|
||||
kill "${runtime_ssh_pid}" 2>/dev/null || true
|
||||
fi
|
||||
exit 3
|
||||
fi
|
||||
|
||||
forward=(
|
||||
-N
|
||||
-L "${LOCAL_PORT}:${cvat_ip}:${REMOTE_WSL_PORT}"
|
||||
-o ExitOnForwardFailure=yes
|
||||
"${SSH_ALIAS}"
|
||||
)
|
||||
|
||||
if [[ "${background}" == true ]]; then
|
||||
ssh -f "${forward[@]}"
|
||||
else
|
||||
ssh "${forward[@]}" &
|
||||
tunnel_ssh_pid=$!
|
||||
fi
|
||||
|
||||
for attempt in $(seq 1 60); do
|
||||
if curl --max-time 2 --fail --silent \
|
||||
"http://localhost:${LOCAL_PORT}/api/server/about" >/dev/null 2>&1; then
|
||||
printf 'cvat_tunnel_ready url=http://localhost:%s wsl_ip=%s\n' \
|
||||
"${LOCAL_PORT}" "${cvat_ip}"
|
||||
if [[ "${background}" == true ]]; then
|
||||
exit 0
|
||||
fi
|
||||
wait "${tunnel_ssh_pid}"
|
||||
tunnel_status=$?
|
||||
kill "${runtime_ssh_pid}" 2>/dev/null || true
|
||||
exit "${tunnel_status}"
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "CVAT tunnel did not become ready within 120 seconds" >&2
|
||||
if [[ "${background}" == false ]]; then
|
||||
kill "${tunnel_ssh_pid}" "${runtime_ssh_pid}" 2>/dev/null || true
|
||||
fi
|
||||
exit 4
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
# Do not inherit Windows executables into this D-only runtime. In particular,
|
||||
# Docker Desktop's docker.exe belongs to a different image store.
|
||||
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
|
||||
readonly CVAT_TAG="${CVAT_TAG:-v2.70.0}"
|
||||
readonly FREE_GIB_FLOOR="${FREE_GIB_FLOOR:-360}"
|
||||
readonly WINDOWS_ROOT="/mnt/d/NDC_MISSIONCORE"
|
||||
readonly COMPUTE_ROOT="${WINDOWS_ROOT}/workspace/mission-core-compute"
|
||||
readonly RUNTIME_ROOT="${WINDOWS_ROOT}/runtime/annotation/cvat"
|
||||
readonly CVAT_ROOT="${RUNTIME_ROOT}/source/cvat-${CVAT_TAG}"
|
||||
readonly STATE_ROOT="/srv/mission-core-cvat"
|
||||
readonly OVERRIDE_FILE="${COMPUTE_ROOT}/cvat/docker-compose.override.yml"
|
||||
|
||||
if [[ "${EUID}" -ne 0 ]]; then
|
||||
echo "provision_cvat_wsl.sh must run as root" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
free_gib() {
|
||||
df -BG --output=avail "${WINDOWS_ROOT}" | tail -n 1 | tr -dc '0-9'
|
||||
}
|
||||
|
||||
guard_disk() {
|
||||
local stage="$1"
|
||||
local free
|
||||
free="$(free_gib)"
|
||||
printf 'disk_guard stage=%s free_gib=%s floor_gib=%s\n' \
|
||||
"${stage}" "${free}" "${FREE_GIB_FLOOR}"
|
||||
if (( free < FREE_GIB_FLOOR )); then
|
||||
echo "D: free-space floor crossed; refusing further writes" >&2
|
||||
exit 3
|
||||
fi
|
||||
}
|
||||
|
||||
guard_disk preflight
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
apt-get install -y ca-certificates curl git
|
||||
|
||||
if ! dpkg-query -W -f='${Status}' docker-ce 2>/dev/null \
|
||||
| grep -qx 'install ok installed'; then
|
||||
install -m 0755 -d /etc/apt/keyrings
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg \
|
||||
-o /etc/apt/keyrings/docker.asc
|
||||
chmod a+r /etc/apt/keyrings/docker.asc
|
||||
# shellcheck disable=SC1091
|
||||
. /etc/os-release
|
||||
arch="$(dpkg --print-architecture)"
|
||||
printf 'deb [arch=%s signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu %s stable\n' \
|
||||
"${arch}" "${VERSION_CODENAME}" >/etc/apt/sources.list.d/docker.list
|
||||
apt-get update
|
||||
apt-get install -y \
|
||||
docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
|
||||
fi
|
||||
|
||||
systemctl enable --now docker
|
||||
docker version
|
||||
docker compose version
|
||||
guard_disk docker-engine
|
||||
|
||||
install -d -m 0750 \
|
||||
"${STATE_ROOT}/volumes/cvat_db" \
|
||||
"${STATE_ROOT}/volumes/cvat_data" \
|
||||
"${STATE_ROOT}/volumes/cvat_keys" \
|
||||
"${STATE_ROOT}/volumes/cvat_logs" \
|
||||
"${STATE_ROOT}/volumes/cvat_inmem_db" \
|
||||
"${STATE_ROOT}/volumes/cvat_events_db" \
|
||||
"${STATE_ROOT}/volumes/cvat_cache_db" \
|
||||
"${RUNTIME_ROOT}/source" \
|
||||
"${RUNTIME_ROOT}/reports"
|
||||
|
||||
if [[ ! -d "${CVAT_ROOT}/.git" ]]; then
|
||||
git clone --depth 1 --branch "${CVAT_TAG}" \
|
||||
https://github.com/cvat-ai/cvat.git "${CVAT_ROOT}"
|
||||
fi
|
||||
|
||||
test "$(git -C "${CVAT_ROOT}" describe --tags --exact-match)" = "${CVAT_TAG}"
|
||||
test -f "${OVERRIDE_FILE}"
|
||||
guard_disk cvat-source
|
||||
|
||||
export CVAT_VERSION="${CVAT_TAG}"
|
||||
export CVAT_HOST="localhost"
|
||||
export CVAT_HTTP_PORT="8080"
|
||||
export COMPOSE_PROJECT_NAME="missioncore-cvat"
|
||||
|
||||
compose=(
|
||||
docker compose
|
||||
--project-directory "${CVAT_ROOT}"
|
||||
-f "${CVAT_ROOT}/docker-compose.yml"
|
||||
-f "${OVERRIDE_FILE}"
|
||||
)
|
||||
|
||||
"${compose[@]}" config --quiet
|
||||
mapfile -t images < <("${compose[@]}" config --images | sort -u)
|
||||
for image in "${images[@]}"; do
|
||||
guard_disk "before-pull:${image}"
|
||||
docker pull "${image}"
|
||||
guard_disk "after-pull:${image}"
|
||||
done
|
||||
|
||||
"${compose[@]}" up -d --no-build
|
||||
guard_disk cvat-started
|
||||
|
||||
deadline=$((SECONDS + 600))
|
||||
until curl --fail --silent --show-error http://localhost:8080/api/server/about \
|
||||
>/dev/null; do
|
||||
if (( SECONDS >= deadline )); then
|
||||
"${compose[@]}" ps
|
||||
echo "CVAT did not become ready within 600 seconds" >&2
|
||||
exit 4
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
|
||||
report="${RUNTIME_ROOT}/reports/deployment-$(date -u +%Y%m%dT%H%M%SZ).txt"
|
||||
{
|
||||
printf 'cvat_tag=%s\n' "${CVAT_TAG}"
|
||||
printf 'cvat_commit=%s\n' "$(git -C "${CVAT_ROOT}" rev-parse HEAD)"
|
||||
printf 'docker_version=%s\n' "$(docker version --format '{{.Server.Version}}')"
|
||||
printf 'compose_version=%s\n' "$(docker compose version --short)"
|
||||
printf 'free_gib=%s\n' "$(free_gib)"
|
||||
printf 'generated_at_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
docker image inspect "${images[@]}" \
|
||||
--format 'image={{index .RepoTags 0}} id={{.Id}} size={{.Size}}'
|
||||
"${compose[@]}" ps
|
||||
} | tee "${report}"
|
||||
|
||||
printf 'CVAT ready at http://localhost:8080\nreport=%s\n' "${report}"
|
||||
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
|
||||
readonly FREE_GIB_FLOOR="${FREE_GIB_FLOOR:-360}"
|
||||
readonly WINDOWS_ROOT="/mnt/d/NDC_MISSIONCORE"
|
||||
readonly COMPUTE_ROOT="${WINDOWS_ROOT}/workspace/mission-core-compute"
|
||||
readonly CVAT_ROOT="${WINDOWS_ROOT}/runtime/annotation/cvat/source/cvat-v2.70.0"
|
||||
readonly OVERRIDE_FILE="${COMPUTE_ROOT}/cvat/docker-compose.override.yml"
|
||||
|
||||
if [[ "${EUID}" -ne 0 ]]; then
|
||||
echo "start_cvat_runtime.sh must run as root" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
free_gib="$(df -BG --output=avail "${WINDOWS_ROOT}" | tail -n 1 | tr -dc '0-9')"
|
||||
printf 'disk_guard stage=start free_gib=%s floor_gib=%s\n' \
|
||||
"${free_gib}" "${FREE_GIB_FLOOR}"
|
||||
if (( free_gib < FREE_GIB_FLOOR )); then
|
||||
echo "D: free-space floor crossed; refusing to start CVAT" >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
test -f "${CVAT_ROOT}/docker-compose.yml"
|
||||
test -f "${OVERRIDE_FILE}"
|
||||
systemctl start docker
|
||||
|
||||
export CVAT_VERSION="v2.70.0"
|
||||
export CVAT_HOST="localhost"
|
||||
export CVAT_HTTP_PORT="8080"
|
||||
export COMPOSE_PROJECT_NAME="missioncore-cvat"
|
||||
|
||||
compose=(
|
||||
docker compose
|
||||
--project-directory "${CVAT_ROOT}"
|
||||
-f "${CVAT_ROOT}/docker-compose.yml"
|
||||
-f "${OVERRIDE_FILE}"
|
||||
)
|
||||
"${compose[@]}" up -d --no-build
|
||||
|
||||
deadline=$((SECONDS + 180))
|
||||
while true; do
|
||||
about_http="$(
|
||||
curl --silent --output /dev/null --write-out '%{http_code}' \
|
||||
http://localhost:8080/api/server/about || true
|
||||
)"
|
||||
tasks_http="$(
|
||||
curl --silent --output /dev/null --write-out '%{http_code}' \
|
||||
http://localhost:8080/api/tasks || true
|
||||
)"
|
||||
if [[ "${about_http}" == "200" ]] \
|
||||
&& [[ "${tasks_http}" == "200" || "${tasks_http}" == "401" || "${tasks_http}" == "403" ]]; then
|
||||
break
|
||||
fi
|
||||
if (( SECONDS >= deadline )); then
|
||||
"${compose[@]}" ps
|
||||
echo "CVAT did not become ready within 180 seconds" >&2
|
||||
exit 4
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
printf 'cvat_ready about_http=%s tasks_http=%s free_gib=%s\n' \
|
||||
"${about_http}" "${tasks_http}" "${free_gib}"
|
||||
|
||||
if [[ "${1:-}" == "--keepalive" ]]; then
|
||||
exec sleep infinity
|
||||
fi
|
||||
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"schema_version": "missioncore.e10-integrated-perception-profile/v1",
|
||||
"mode": "full-session-qualification",
|
||||
"source": {
|
||||
"source_id": "sensor.camera.right",
|
||||
"resolution": [800, 600],
|
||||
"calibration_slot": "camera_1",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
},
|
||||
"selection": {
|
||||
"required_frame_count": 4489,
|
||||
"required_source_start_frame_index": 0,
|
||||
"required_source_end_frame_index": 4488,
|
||||
"minimum_source_span_seconds": 448.0
|
||||
},
|
||||
"replay": {
|
||||
"speed": 1.0,
|
||||
"detector_queue_capacity": 2,
|
||||
"semantic_queue_capacity": 1,
|
||||
"semantic_sample_every_frames": 5,
|
||||
"semantic_ttl_ms": 750.0
|
||||
},
|
||||
"association": {
|
||||
"box_inset": {
|
||||
"bottom_fraction": 0.02,
|
||||
"horizontal_fraction": 0.05,
|
||||
"top_fraction": 0.04
|
||||
},
|
||||
"depth_cluster_gap_fraction": 0.06,
|
||||
"depth_cluster_minimum_gap_m": 0.5,
|
||||
"group_nms_iou_threshold": 0.55,
|
||||
"maximum_cuboid_span_m": 12.0,
|
||||
"maximum_distance_innovation_fraction": 0.25,
|
||||
"maximum_distance_innovation_m": 1.5,
|
||||
"maximum_oriented_extent_m": {
|
||||
"bicycle": [3.5, 2.0, 2.5],
|
||||
"motorcycle": [3.5, 2.0, 2.5],
|
||||
"person": [1.5, 1.5, 2.8],
|
||||
"vehicle": [6.5, 3.5, 4.0]
|
||||
},
|
||||
"minimum_cuboid_extent_m": 0.15,
|
||||
"minimum_support_points": {
|
||||
"bicycle": 3,
|
||||
"motorcycle": 3,
|
||||
"person": 3,
|
||||
"vehicle": 5
|
||||
},
|
||||
"semantic_ids": {
|
||||
"bicycle": [2],
|
||||
"motorcycle": [3],
|
||||
"person": [1],
|
||||
"vehicle": [4, 5]
|
||||
},
|
||||
"spatial_cluster_radius_m": {
|
||||
"bicycle": 0.9,
|
||||
"motorcycle": 0.9,
|
||||
"person": 0.9,
|
||||
"vehicle": 1.5
|
||||
},
|
||||
"vehicle_labels": ["car", "truck", "bus"],
|
||||
"distance_history_frames": 5
|
||||
},
|
||||
"world_state": {
|
||||
"velocity_history_limit_s": 1.0,
|
||||
"clearance": {
|
||||
"sector_count": 72,
|
||||
"minimum_range_m": 0.5,
|
||||
"maximum_range_m": 30.0,
|
||||
"ground_percentile": 5.0,
|
||||
"minimum_height_above_ground_m": 0.2,
|
||||
"maximum_height_above_ground_m": 3.0,
|
||||
"front_half_angle_degrees": 15.0
|
||||
}
|
||||
},
|
||||
"acceptance": {
|
||||
"detector_minimum_effective_fps": 9.5,
|
||||
"detector_maximum_drop_fraction": 0.0,
|
||||
"maximum_p95_world_state_age_ms": 175.0,
|
||||
"semantic_minimum_effective_fps": 1.8,
|
||||
"semantic_maximum_drop_fraction": 0.05,
|
||||
"semantic_maximum_p95_completion_age_ms": 400.0,
|
||||
"minimum_fresh_semantic_coverage": 0.9,
|
||||
"minimum_lidar_fused_frames": 3500,
|
||||
"minimum_accepted_cuboids": 500,
|
||||
"require_zero_failures": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"schema_version": "missioncore.e10-integrated-perception-profile/v1",
|
||||
"mode": "qualification",
|
||||
"source": {
|
||||
"source_id": "sensor.camera.right",
|
||||
"resolution": [800, 600],
|
||||
"calibration_slot": "camera_1",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
},
|
||||
"replay": {
|
||||
"speed": 1.0,
|
||||
"detector_queue_capacity": 2,
|
||||
"semantic_queue_capacity": 1,
|
||||
"semantic_sample_every_frames": 5,
|
||||
"semantic_ttl_ms": 750.0
|
||||
},
|
||||
"association": {
|
||||
"box_inset": {
|
||||
"bottom_fraction": 0.02,
|
||||
"horizontal_fraction": 0.05,
|
||||
"top_fraction": 0.04
|
||||
},
|
||||
"depth_cluster_gap_fraction": 0.06,
|
||||
"depth_cluster_minimum_gap_m": 0.5,
|
||||
"group_nms_iou_threshold": 0.55,
|
||||
"maximum_cuboid_span_m": 12.0,
|
||||
"maximum_distance_innovation_fraction": 0.25,
|
||||
"maximum_distance_innovation_m": 1.5,
|
||||
"maximum_oriented_extent_m": {
|
||||
"bicycle": [3.5, 2.0, 2.5],
|
||||
"motorcycle": [3.5, 2.0, 2.5],
|
||||
"person": [1.5, 1.5, 2.8],
|
||||
"vehicle": [6.5, 3.5, 4.0]
|
||||
},
|
||||
"minimum_cuboid_extent_m": 0.15,
|
||||
"minimum_support_points": {
|
||||
"bicycle": 3,
|
||||
"motorcycle": 3,
|
||||
"person": 3,
|
||||
"vehicle": 5
|
||||
},
|
||||
"semantic_ids": {
|
||||
"bicycle": [2],
|
||||
"motorcycle": [3],
|
||||
"person": [1],
|
||||
"vehicle": [4, 5]
|
||||
},
|
||||
"spatial_cluster_radius_m": {
|
||||
"bicycle": 0.9,
|
||||
"motorcycle": 0.9,
|
||||
"person": 0.9,
|
||||
"vehicle": 1.5
|
||||
},
|
||||
"vehicle_labels": ["car", "truck", "bus"],
|
||||
"distance_history_frames": 5
|
||||
},
|
||||
"world_state": {
|
||||
"velocity_history_limit_s": 1.0,
|
||||
"clearance": {
|
||||
"sector_count": 72,
|
||||
"minimum_range_m": 0.5,
|
||||
"maximum_range_m": 30.0,
|
||||
"ground_percentile": 5.0,
|
||||
"minimum_height_above_ground_m": 0.2,
|
||||
"maximum_height_above_ground_m": 3.0,
|
||||
"front_half_angle_degrees": 15.0
|
||||
}
|
||||
},
|
||||
"acceptance": {
|
||||
"detector_minimum_effective_fps": 9.5,
|
||||
"detector_maximum_drop_fraction": 0.005,
|
||||
"maximum_p95_world_state_age_ms": 175.0,
|
||||
"semantic_minimum_effective_fps": 1.8,
|
||||
"semantic_maximum_drop_fraction": 0.05,
|
||||
"semantic_maximum_p95_completion_age_ms": 400.0,
|
||||
"minimum_fresh_semantic_coverage": 0.9,
|
||||
"minimum_lidar_fused_frames": 450,
|
||||
"minimum_accepted_cuboids": 100,
|
||||
"require_zero_failures": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"schema_version": "missioncore.e10-integrated-perception-profile/v1",
|
||||
"mode": "semantic-loss-negative-control",
|
||||
"source": {
|
||||
"source_id": "sensor.camera.right",
|
||||
"resolution": [800, 600],
|
||||
"calibration_slot": "camera_1",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
},
|
||||
"replay": {
|
||||
"speed": 1.0,
|
||||
"detector_queue_capacity": 2,
|
||||
"semantic_queue_capacity": 1,
|
||||
"semantic_sample_every_frames": 5,
|
||||
"semantic_ttl_ms": 750.0
|
||||
},
|
||||
"semantic_loss": {
|
||||
"stop_after_completed_results": 20
|
||||
},
|
||||
"association": {
|
||||
"box_inset": {
|
||||
"bottom_fraction": 0.02,
|
||||
"horizontal_fraction": 0.05,
|
||||
"top_fraction": 0.04
|
||||
},
|
||||
"depth_cluster_gap_fraction": 0.06,
|
||||
"depth_cluster_minimum_gap_m": 0.5,
|
||||
"group_nms_iou_threshold": 0.55,
|
||||
"maximum_cuboid_span_m": 12.0,
|
||||
"maximum_distance_innovation_fraction": 0.25,
|
||||
"maximum_distance_innovation_m": 1.5,
|
||||
"maximum_oriented_extent_m": {
|
||||
"bicycle": [3.5, 2.0, 2.5],
|
||||
"motorcycle": [3.5, 2.0, 2.5],
|
||||
"person": [1.5, 1.5, 2.8],
|
||||
"vehicle": [6.5, 3.5, 4.0]
|
||||
},
|
||||
"minimum_cuboid_extent_m": 0.15,
|
||||
"minimum_support_points": {
|
||||
"bicycle": 3,
|
||||
"motorcycle": 3,
|
||||
"person": 3,
|
||||
"vehicle": 5
|
||||
},
|
||||
"semantic_ids": {
|
||||
"bicycle": [2],
|
||||
"motorcycle": [3],
|
||||
"person": [1],
|
||||
"vehicle": [4, 5]
|
||||
},
|
||||
"spatial_cluster_radius_m": {
|
||||
"bicycle": 0.9,
|
||||
"motorcycle": 0.9,
|
||||
"person": 0.9,
|
||||
"vehicle": 1.5
|
||||
},
|
||||
"vehicle_labels": ["car", "truck", "bus"],
|
||||
"distance_history_frames": 5
|
||||
},
|
||||
"world_state": {
|
||||
"velocity_history_limit_s": 1.0,
|
||||
"clearance": {
|
||||
"sector_count": 72,
|
||||
"minimum_range_m": 0.5,
|
||||
"maximum_range_m": 30.0,
|
||||
"ground_percentile": 5.0,
|
||||
"minimum_height_above_ground_m": 0.2,
|
||||
"maximum_height_above_ground_m": 3.0,
|
||||
"front_half_angle_degrees": 15.0
|
||||
}
|
||||
},
|
||||
"acceptance": {
|
||||
"detector_minimum_effective_fps": 9.5,
|
||||
"detector_maximum_drop_fraction": 0.005,
|
||||
"maximum_p95_world_state_age_ms": 175.0,
|
||||
"minimum_stale_detector_frames": 450
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
{
|
||||
"schema_version": "missioncore.e10-integrated-perception-profile/v1",
|
||||
"mode": "pilot",
|
||||
"source": {
|
||||
"source_id": "sensor.camera.right",
|
||||
"resolution": [800, 600],
|
||||
"calibration_slot": "camera_1",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
},
|
||||
"replay": {
|
||||
"speed": 1.0,
|
||||
"detector_queue_capacity": 2,
|
||||
"semantic_queue_capacity": 1,
|
||||
"semantic_sample_every_frames": 5,
|
||||
"semantic_ttl_ms": 750.0
|
||||
},
|
||||
"association": {
|
||||
"box_inset": {
|
||||
"bottom_fraction": 0.02,
|
||||
"horizontal_fraction": 0.05,
|
||||
"top_fraction": 0.04
|
||||
},
|
||||
"depth_cluster_gap_fraction": 0.06,
|
||||
"depth_cluster_minimum_gap_m": 0.5,
|
||||
"group_nms_iou_threshold": 0.5,
|
||||
"maximum_cuboid_span_m": 15.0,
|
||||
"maximum_distance_innovation_fraction": 0.25,
|
||||
"maximum_distance_innovation_m": 1.5,
|
||||
"maximum_oriented_extent_m": {
|
||||
"bicycle": [3.5, 2.0, 2.5],
|
||||
"motorcycle": [3.5, 2.0, 2.5],
|
||||
"person": [1.5, 1.5, 2.8],
|
||||
"vehicle": [12.5, 4.0, 4.5]
|
||||
},
|
||||
"minimum_cuboid_extent_m": 0.15,
|
||||
"minimum_support_points": {
|
||||
"bicycle": 4,
|
||||
"motorcycle": 4,
|
||||
"person": 4,
|
||||
"vehicle": 8
|
||||
},
|
||||
"semantic_ids": {
|
||||
"bicycle": [2],
|
||||
"motorcycle": [3],
|
||||
"person": [1],
|
||||
"vehicle": [4, 5]
|
||||
},
|
||||
"spatial_cluster_radius_m": {
|
||||
"bicycle": 0.9,
|
||||
"motorcycle": 0.9,
|
||||
"person": 0.9,
|
||||
"vehicle": 1.5
|
||||
},
|
||||
"vehicle_labels": ["car", "truck", "bus"],
|
||||
"distance_history_frames": 5
|
||||
},
|
||||
"cuboid_completion": {
|
||||
"mode": "class-prior-amodal-v1",
|
||||
"failure_policy": "reject",
|
||||
"classes": {
|
||||
"person": {
|
||||
"nominal_size_m": [0.55, 0.55, 1.72],
|
||||
"minimum_size_m": [0.35, 0.35, 1.3],
|
||||
"maximum_size_m": [1.2, 1.2, 2.3],
|
||||
"support_padding_m": [0.12, 0.12, 0.12]
|
||||
},
|
||||
"bicycle": {
|
||||
"nominal_size_m": [1.8, 0.65, 1.5],
|
||||
"minimum_size_m": [1.2, 0.4, 1.0],
|
||||
"maximum_size_m": [2.5, 1.2, 2.2],
|
||||
"support_padding_m": [0.18, 0.12, 0.12]
|
||||
},
|
||||
"motorcycle": {
|
||||
"nominal_size_m": [2.1, 0.8, 1.45],
|
||||
"minimum_size_m": [1.4, 0.5, 1.0],
|
||||
"maximum_size_m": [3.0, 1.4, 2.2],
|
||||
"support_padding_m": [0.2, 0.14, 0.14]
|
||||
},
|
||||
"car": {
|
||||
"nominal_size_m": [4.5, 1.85, 1.55],
|
||||
"minimum_size_m": [3.2, 1.45, 1.2],
|
||||
"maximum_size_m": [5.8, 2.4, 2.3],
|
||||
"support_padding_m": [0.25, 0.18, 0.15]
|
||||
},
|
||||
"truck": {
|
||||
"nominal_size_m": [7.0, 2.5, 3.0],
|
||||
"minimum_size_m": [4.8, 1.8, 1.8],
|
||||
"maximum_size_m": [12.5, 3.2, 4.2],
|
||||
"support_padding_m": [0.35, 0.22, 0.2]
|
||||
},
|
||||
"bus": {
|
||||
"nominal_size_m": [10.5, 2.55, 3.2],
|
||||
"minimum_size_m": [7.0, 2.1, 2.5],
|
||||
"maximum_size_m": [13.5, 3.2, 4.2],
|
||||
"support_padding_m": [0.4, 0.24, 0.2]
|
||||
}
|
||||
},
|
||||
"ground": {
|
||||
"local_radius_m": 2.5,
|
||||
"lower_percentile": 8.0,
|
||||
"maximum_below_support_m": 1.2,
|
||||
"maximum_above_support_m": 0.15,
|
||||
"fallback_below_support_m": 0.25
|
||||
},
|
||||
"orientation": {
|
||||
"minimum_anisotropy_ratio": 1.35,
|
||||
"face_width_switch_fraction": 1.05
|
||||
},
|
||||
"temporal": {
|
||||
"center_alpha": 0.4,
|
||||
"size_alpha": 0.2,
|
||||
"yaw_alpha": 0.25,
|
||||
"maximum_center_innovation_m": 2.0,
|
||||
"maximum_yaw_innovation_degrees": 55.0,
|
||||
"maximum_idle_s": 1.0,
|
||||
"confirmation_hits": 3
|
||||
},
|
||||
"minimum_support_coverage_fraction": 0.75
|
||||
},
|
||||
"world_state": {
|
||||
"velocity_history_limit_s": 1.0,
|
||||
"clearance": {
|
||||
"sector_count": 72,
|
||||
"minimum_range_m": 0.5,
|
||||
"maximum_range_m": 30.0,
|
||||
"ground_percentile": 5.0,
|
||||
"minimum_height_above_ground_m": 0.2,
|
||||
"maximum_height_above_ground_m": 3.0,
|
||||
"front_half_angle_degrees": 15.0
|
||||
}
|
||||
},
|
||||
"acceptance": {
|
||||
"detector_minimum_effective_fps": 9.5,
|
||||
"detector_maximum_drop_fraction": 0.005,
|
||||
"maximum_p95_world_state_age_ms": 175.0,
|
||||
"semantic_minimum_effective_fps": 1.8,
|
||||
"semantic_maximum_drop_fraction": 0.05,
|
||||
"semantic_maximum_p95_completion_age_ms": 400.0,
|
||||
"minimum_fresh_semantic_coverage": 0.9,
|
||||
"minimum_lidar_fused_frames": 450,
|
||||
"minimum_accepted_cuboids": 100,
|
||||
"require_zero_failures": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
{
|
||||
"schema_version": "missioncore.e10-integrated-perception-profile/v1",
|
||||
"mode": "full-session-qualification",
|
||||
"source": {
|
||||
"source_id": "sensor.camera.right",
|
||||
"resolution": [800, 600],
|
||||
"calibration_slot": "camera_1",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
},
|
||||
"selection": {
|
||||
"required_frame_count": 4489,
|
||||
"required_source_start_frame_index": 0,
|
||||
"required_source_end_frame_index": 4488,
|
||||
"minimum_source_span_seconds": 448.0
|
||||
},
|
||||
"replay": {
|
||||
"speed": 1.0,
|
||||
"detector_queue_capacity": 2,
|
||||
"semantic_queue_capacity": 1,
|
||||
"semantic_sample_every_frames": 5,
|
||||
"semantic_ttl_ms": 750.0
|
||||
},
|
||||
"association": {
|
||||
"box_inset": {
|
||||
"bottom_fraction": 0.02,
|
||||
"horizontal_fraction": 0.05,
|
||||
"top_fraction": 0.04
|
||||
},
|
||||
"depth_cluster_gap_fraction": 0.06,
|
||||
"depth_cluster_minimum_gap_m": 0.5,
|
||||
"group_nms_iou_threshold": 0.5,
|
||||
"maximum_cuboid_span_m": 15.0,
|
||||
"maximum_distance_innovation_fraction": 0.25,
|
||||
"maximum_distance_innovation_m": 1.5,
|
||||
"maximum_oriented_extent_m": {
|
||||
"bicycle": [3.5, 2.0, 2.5],
|
||||
"motorcycle": [3.5, 2.0, 2.5],
|
||||
"person": [1.5, 1.5, 2.8],
|
||||
"vehicle": [12.5, 4.0, 4.5]
|
||||
},
|
||||
"minimum_cuboid_extent_m": 0.15,
|
||||
"minimum_support_points": {
|
||||
"bicycle": 4,
|
||||
"motorcycle": 4,
|
||||
"person": 4,
|
||||
"vehicle": 8
|
||||
},
|
||||
"semantic_ids": {
|
||||
"bicycle": [2],
|
||||
"motorcycle": [3],
|
||||
"person": [1],
|
||||
"vehicle": [4, 5]
|
||||
},
|
||||
"spatial_cluster_radius_m": {
|
||||
"bicycle": 0.9,
|
||||
"motorcycle": 0.9,
|
||||
"person": 0.9,
|
||||
"vehicle": 1.5
|
||||
},
|
||||
"vehicle_labels": ["car", "truck", "bus"],
|
||||
"distance_history_frames": 5
|
||||
},
|
||||
"cuboid_completion": {
|
||||
"mode": "class-prior-amodal-v1",
|
||||
"failure_policy": "reject",
|
||||
"classes": {
|
||||
"person": {
|
||||
"nominal_size_m": [0.55, 0.55, 1.72],
|
||||
"minimum_size_m": [0.35, 0.35, 1.3],
|
||||
"maximum_size_m": [1.2, 1.2, 2.3],
|
||||
"support_padding_m": [0.12, 0.12, 0.12]
|
||||
},
|
||||
"bicycle": {
|
||||
"nominal_size_m": [1.8, 0.65, 1.5],
|
||||
"minimum_size_m": [1.2, 0.4, 1.0],
|
||||
"maximum_size_m": [2.5, 1.2, 2.2],
|
||||
"support_padding_m": [0.18, 0.12, 0.12]
|
||||
},
|
||||
"motorcycle": {
|
||||
"nominal_size_m": [2.1, 0.8, 1.45],
|
||||
"minimum_size_m": [1.4, 0.5, 1.0],
|
||||
"maximum_size_m": [3.0, 1.4, 2.2],
|
||||
"support_padding_m": [0.2, 0.14, 0.14]
|
||||
},
|
||||
"car": {
|
||||
"nominal_size_m": [4.5, 1.85, 1.55],
|
||||
"minimum_size_m": [3.2, 1.45, 1.2],
|
||||
"maximum_size_m": [5.8, 2.4, 2.3],
|
||||
"support_padding_m": [0.25, 0.18, 0.15]
|
||||
},
|
||||
"truck": {
|
||||
"nominal_size_m": [7.0, 2.5, 3.0],
|
||||
"minimum_size_m": [4.8, 1.8, 1.8],
|
||||
"maximum_size_m": [12.5, 3.2, 4.2],
|
||||
"support_padding_m": [0.35, 0.22, 0.2]
|
||||
},
|
||||
"bus": {
|
||||
"nominal_size_m": [10.5, 2.55, 3.2],
|
||||
"minimum_size_m": [7.0, 2.1, 2.5],
|
||||
"maximum_size_m": [13.5, 3.2, 4.2],
|
||||
"support_padding_m": [0.4, 0.24, 0.2]
|
||||
}
|
||||
},
|
||||
"ground": {
|
||||
"local_radius_m": 2.5,
|
||||
"lower_percentile": 8.0,
|
||||
"maximum_below_support_m": 1.2,
|
||||
"maximum_above_support_m": 0.15,
|
||||
"fallback_below_support_m": 0.25
|
||||
},
|
||||
"orientation": {
|
||||
"minimum_anisotropy_ratio": 1.35,
|
||||
"face_width_switch_fraction": 1.05
|
||||
},
|
||||
"temporal": {
|
||||
"center_alpha": 0.4,
|
||||
"size_alpha": 0.2,
|
||||
"yaw_alpha": 0.25,
|
||||
"maximum_center_innovation_m": 2.0,
|
||||
"maximum_yaw_innovation_degrees": 55.0,
|
||||
"maximum_idle_s": 1.0,
|
||||
"confirmation_hits": 3
|
||||
},
|
||||
"minimum_support_coverage_fraction": 0.75
|
||||
},
|
||||
"world_state": {
|
||||
"velocity_history_limit_s": 1.0,
|
||||
"clearance": {
|
||||
"sector_count": 72,
|
||||
"minimum_range_m": 0.5,
|
||||
"maximum_range_m": 30.0,
|
||||
"ground_percentile": 5.0,
|
||||
"minimum_height_above_ground_m": 0.2,
|
||||
"maximum_height_above_ground_m": 3.0,
|
||||
"front_half_angle_degrees": 15.0
|
||||
}
|
||||
},
|
||||
"acceptance": {
|
||||
"detector_minimum_effective_fps": 9.5,
|
||||
"detector_maximum_drop_fraction": 0.005,
|
||||
"maximum_p95_world_state_age_ms": 175.0,
|
||||
"semantic_minimum_effective_fps": 1.8,
|
||||
"semantic_maximum_drop_fraction": 0.05,
|
||||
"semantic_maximum_p95_completion_age_ms": 400.0,
|
||||
"minimum_fresh_semantic_coverage": 0.9,
|
||||
"minimum_lidar_fused_frames": 3500,
|
||||
"minimum_accepted_cuboids": 1500,
|
||||
"require_zero_failures": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"schema_version": "missioncore.e15-shadow-inference-profile/v1",
|
||||
"mode": "replay-shadow-gate",
|
||||
"authority": {
|
||||
"commands_enabled": false,
|
||||
"navigation_or_safety_accepted": false
|
||||
},
|
||||
"source": {
|
||||
"source_id": "sensor.camera.right",
|
||||
"resolution": [800, 600],
|
||||
"calibration_slot": "camera_1",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
},
|
||||
"transport": {
|
||||
"wire_schema": "missioncore.live-perception-wire/v1",
|
||||
"camera_media": "persistent-fmp4-pyav",
|
||||
"pyav_version": "18.0.0",
|
||||
"maximum_media_buffer_bytes": 8388608,
|
||||
"camera_metadata_capacity": 16
|
||||
},
|
||||
"scheduling": {
|
||||
"detector_queue_capacity": 2,
|
||||
"semantic_queue_capacity": 1,
|
||||
"semantic_sample_every_frames": 5,
|
||||
"semantic_ttl_ms": 750.0,
|
||||
"sensor_wait_ms": 90.0
|
||||
},
|
||||
"temporal": {
|
||||
"binding": "nearest-recorded-host-arrival-best-effort",
|
||||
"maximum_lidar_camera_delta_ms": 100.0,
|
||||
"maximum_pose_point_delta_ms": 100.0,
|
||||
"buffer_capacity_per_modality": 32,
|
||||
"retention_seconds": 3.0,
|
||||
"clock_qualification": "not-hardware-synchronized"
|
||||
},
|
||||
"acceptance": {
|
||||
"minimum_camera_frames": 140,
|
||||
"detector_minimum_effective_fps": 9.5,
|
||||
"detector_maximum_drop_fraction": 0.01,
|
||||
"semantic_minimum_effective_fps": 1.8,
|
||||
"semantic_maximum_drop_fraction": 0.05,
|
||||
"semantic_maximum_p95_completion_age_ms": 400.0,
|
||||
"minimum_fresh_semantic_coverage": 0.9,
|
||||
"minimum_fused_fraction": 0.85,
|
||||
"maximum_p95_decode_age_ms": 80.0,
|
||||
"maximum_p95_world_state_age_ms": 200.0,
|
||||
"require_zero_transport_gaps": true,
|
||||
"require_zero_camera_sequence_gaps": true,
|
||||
"require_zero_failures": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
"""Bounded media/runtime primitives for the LAB E15 shadow inference worker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
class ShadowRuntimeError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CameraFragmentMetadata:
|
||||
ingress_sequence: int
|
||||
source_sequence: int
|
||||
captured_at_epoch_ns: int
|
||||
worker_received_monotonic: float
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DecodedCameraFrame:
|
||||
frame_index: int
|
||||
metadata: CameraFragmentMetadata
|
||||
image: np.ndarray
|
||||
decoded_monotonic: float
|
||||
decode_age_ms: float
|
||||
|
||||
|
||||
class IncrementalMediaBuffer:
|
||||
"""A bounded non-seekable blocking byte source accepted by PyAV."""
|
||||
|
||||
def __init__(self, maximum_buffer_bytes: int) -> None:
|
||||
if maximum_buffer_bytes < 1024:
|
||||
raise ValueError("media buffer bound is too small")
|
||||
self._maximum_buffer_bytes = maximum_buffer_bytes
|
||||
self._buffer = bytearray()
|
||||
self._condition = threading.Condition()
|
||||
self._finished = False
|
||||
self._failure: BaseException | None = None
|
||||
self._bytes_published = 0
|
||||
self._bytes_read = 0
|
||||
self._maximum_depth = 0
|
||||
|
||||
def readable(self) -> bool:
|
||||
return True
|
||||
|
||||
def read(self, size: int = -1) -> bytes:
|
||||
with self._condition:
|
||||
self._condition.wait_for(
|
||||
lambda: bool(self._buffer) or self._finished or self._failure is not None
|
||||
)
|
||||
if self._failure is not None:
|
||||
raise ShadowRuntimeError("incremental media source failed") from self._failure
|
||||
if not self._buffer:
|
||||
return b""
|
||||
count = len(self._buffer) if size < 0 else min(size, len(self._buffer))
|
||||
result = bytes(self._buffer[:count])
|
||||
del self._buffer[:count]
|
||||
self._bytes_read += count
|
||||
self._condition.notify_all()
|
||||
return result
|
||||
|
||||
def append(self, payload: bytes) -> None:
|
||||
if not payload:
|
||||
raise ShadowRuntimeError("empty fMP4 payload is not admitted")
|
||||
with self._condition:
|
||||
if self._finished or self._failure is not None:
|
||||
raise ShadowRuntimeError("incremental media source is closed")
|
||||
if len(self._buffer) + len(payload) > self._maximum_buffer_bytes:
|
||||
failure = ShadowRuntimeError("incremental media buffer exceeded its hard bound")
|
||||
self._failure = failure
|
||||
self._condition.notify_all()
|
||||
raise failure
|
||||
self._buffer.extend(payload)
|
||||
self._bytes_published += len(payload)
|
||||
self._maximum_depth = max(self._maximum_depth, len(self._buffer))
|
||||
self._condition.notify_all()
|
||||
|
||||
def finish(self) -> None:
|
||||
with self._condition:
|
||||
self._finished = True
|
||||
self._condition.notify_all()
|
||||
|
||||
def fail(self, failure: BaseException) -> None:
|
||||
with self._condition:
|
||||
if self._failure is None:
|
||||
self._failure = failure
|
||||
self._condition.notify_all()
|
||||
|
||||
def snapshot(self) -> dict[str, int | bool]:
|
||||
with self._condition:
|
||||
return {
|
||||
"maximum_buffer_bytes": self._maximum_buffer_bytes,
|
||||
"depth_bytes": len(self._buffer),
|
||||
"maximum_depth_bytes": self._maximum_depth,
|
||||
"bytes_published": self._bytes_published,
|
||||
"bytes_read": self._bytes_read,
|
||||
"finished": self._finished,
|
||||
"failed": self._failure is not None,
|
||||
}
|
||||
|
||||
|
||||
class CameraMetadataQueue:
|
||||
"""Fail-closed segment metadata queue; dropping a fragment would corrupt H.264."""
|
||||
|
||||
def __init__(self, capacity: int) -> None:
|
||||
if capacity < 2:
|
||||
raise ValueError("camera metadata capacity is too small")
|
||||
self._capacity = capacity
|
||||
self._items: deque[CameraFragmentMetadata] = deque()
|
||||
self._condition = threading.Condition()
|
||||
self._finished = False
|
||||
self._published = 0
|
||||
self._consumed = 0
|
||||
self._maximum_depth = 0
|
||||
|
||||
def publish(self, value: CameraFragmentMetadata) -> None:
|
||||
with self._condition:
|
||||
if self._finished:
|
||||
raise ShadowRuntimeError("camera metadata queue is closed")
|
||||
if len(self._items) >= self._capacity:
|
||||
raise ShadowRuntimeError("camera metadata queue exceeded its hard bound")
|
||||
self._items.append(value)
|
||||
self._published += 1
|
||||
self._maximum_depth = max(self._maximum_depth, len(self._items))
|
||||
self._condition.notify_all()
|
||||
|
||||
def take(self) -> CameraFragmentMetadata:
|
||||
with self._condition:
|
||||
self._condition.wait_for(lambda: bool(self._items) or self._finished)
|
||||
if not self._items:
|
||||
raise ShadowRuntimeError("decoder emitted a frame without segment metadata")
|
||||
self._consumed += 1
|
||||
return self._items.popleft()
|
||||
|
||||
def finish(self) -> None:
|
||||
with self._condition:
|
||||
self._finished = True
|
||||
self._condition.notify_all()
|
||||
|
||||
def snapshot(self) -> dict[str, int | bool]:
|
||||
with self._condition:
|
||||
return {
|
||||
"capacity": self._capacity,
|
||||
"depth": len(self._items),
|
||||
"maximum_depth": self._maximum_depth,
|
||||
"published": self._published,
|
||||
"consumed": self._consumed,
|
||||
"finished": self._finished,
|
||||
}
|
||||
|
||||
|
||||
class PersistentFmp4Decoder:
|
||||
"""Decode one committed fMP4 epoch incrementally without writing frames."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
on_frame: Callable[[DecodedCameraFrame], None],
|
||||
width: int = 800,
|
||||
height: int = 600,
|
||||
maximum_buffer_bytes: int = 8 * 1024 * 1024,
|
||||
metadata_capacity: int = 16,
|
||||
) -> None:
|
||||
if width < 1 or height < 1:
|
||||
raise ValueError("decoder resolution is invalid")
|
||||
self._on_frame = on_frame
|
||||
self._width = width
|
||||
self._height = height
|
||||
self._media = IncrementalMediaBuffer(maximum_buffer_bytes)
|
||||
self._metadata = CameraMetadataQueue(metadata_capacity)
|
||||
self._thread = threading.Thread(
|
||||
target=self._decode,
|
||||
name="lab-e15-fmp4-decoder",
|
||||
daemon=True,
|
||||
)
|
||||
self._started = False
|
||||
self._init_seen = False
|
||||
self._last_source_sequence: int | None = None
|
||||
self._decoded_frames = 0
|
||||
self._failure: BaseException | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
if self._started:
|
||||
raise ShadowRuntimeError("fMP4 decoder was already started")
|
||||
self._started = True
|
||||
self._thread.start()
|
||||
|
||||
def feed_init(self, payload: bytes) -> None:
|
||||
if not self._started or self._init_seen or self._last_source_sequence is not None:
|
||||
raise ShadowRuntimeError("camera init ordering is invalid")
|
||||
self._init_seen = True
|
||||
self._media.append(payload)
|
||||
|
||||
def feed_segment(self, metadata: CameraFragmentMetadata, payload: bytes) -> None:
|
||||
if not self._init_seen:
|
||||
raise ShadowRuntimeError("camera media arrived before init")
|
||||
expected = 1 if self._last_source_sequence is None else self._last_source_sequence + 1
|
||||
if metadata.source_sequence != expected:
|
||||
raise ShadowRuntimeError(
|
||||
f"camera source sequence gap: expected {expected}, got {metadata.source_sequence}"
|
||||
)
|
||||
self._metadata.publish(metadata)
|
||||
try:
|
||||
self._media.append(payload)
|
||||
except BaseException as exc:
|
||||
self._media.fail(exc)
|
||||
self._metadata.finish()
|
||||
raise
|
||||
self._last_source_sequence = metadata.source_sequence
|
||||
|
||||
def finish_input(self) -> None:
|
||||
self._media.finish()
|
||||
self._metadata.finish()
|
||||
|
||||
def join(self, timeout_seconds: float = 30.0) -> None:
|
||||
self._thread.join(timeout=timeout_seconds)
|
||||
if self._thread.is_alive():
|
||||
raise ShadowRuntimeError("persistent fMP4 decoder did not stop")
|
||||
if self._failure is not None:
|
||||
raise ShadowRuntimeError("persistent fMP4 decoder failed") from self._failure
|
||||
metadata = self._metadata.snapshot()
|
||||
if metadata["published"] != metadata["consumed"] or metadata["depth"] != 0:
|
||||
raise ShadowRuntimeError("camera segment/frame accounting differs")
|
||||
|
||||
def snapshot(self) -> dict[str, Any]:
|
||||
return {
|
||||
"init_seen": self._init_seen,
|
||||
"last_source_sequence": self._last_source_sequence,
|
||||
"decoded_frames": self._decoded_frames,
|
||||
"failed": self._failure is not None,
|
||||
"media": self._media.snapshot(),
|
||||
"metadata": self._metadata.snapshot(),
|
||||
}
|
||||
|
||||
def _decode(self) -> None:
|
||||
try:
|
||||
import av
|
||||
|
||||
container = av.open(
|
||||
self._media,
|
||||
mode="r",
|
||||
format="mp4",
|
||||
options={"probesize": "32", "analyzeduration": "0"},
|
||||
)
|
||||
try:
|
||||
for frame in container.decode(video=0):
|
||||
metadata = self._metadata.take()
|
||||
image = frame.to_ndarray(format="rgb24")
|
||||
if image.shape != (self._height, self._width, 3):
|
||||
raise ShadowRuntimeError("decoded camera resolution changed")
|
||||
image.setflags(write=False)
|
||||
decoded_monotonic = time.perf_counter()
|
||||
self._on_frame(
|
||||
DecodedCameraFrame(
|
||||
frame_index=self._decoded_frames,
|
||||
metadata=metadata,
|
||||
image=image,
|
||||
decoded_monotonic=decoded_monotonic,
|
||||
decode_age_ms=max(
|
||||
0.0,
|
||||
(
|
||||
decoded_monotonic
|
||||
- metadata.worker_received_monotonic
|
||||
)
|
||||
* 1000,
|
||||
),
|
||||
)
|
||||
)
|
||||
self._decoded_frames += 1
|
||||
finally:
|
||||
container.close()
|
||||
except BaseException as exc:
|
||||
self._failure = exc
|
||||
self._media.fail(exc)
|
||||
self._metadata.finish()
|
||||
@@ -0,0 +1,145 @@
|
||||
{
|
||||
"schema_version": "missioncore.k1-e3-rectified-segmentation-profile/v1",
|
||||
"profile_id": "k1-camera1-kb4-cubemap5-eomt-cityscapes/v1",
|
||||
"source": {
|
||||
"source_id": "sensor.camera.right",
|
||||
"calibration_slot": "camera_1",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9",
|
||||
"resolution": [
|
||||
800,
|
||||
600
|
||||
],
|
||||
"intrinsic_fx_fy_cx_cy": [
|
||||
194.59817287616025,
|
||||
194.57531427932872,
|
||||
396.31861150187996,
|
||||
301.49644357408005
|
||||
],
|
||||
"distortion_kb4": [
|
||||
-0.023164451386679667,
|
||||
-0.0014974198594105452,
|
||||
-0.001039213149441563,
|
||||
-0.000035237331915978814
|
||||
]
|
||||
},
|
||||
"variants": [
|
||||
"eomt-fisheye-mask",
|
||||
"eomt-fisheye-mask-clahe",
|
||||
"eomt-kb4-cubemap5-clahe"
|
||||
],
|
||||
"rectification": {
|
||||
"projection": "five-perspective-gnomonic/v1",
|
||||
"tile_size": 768,
|
||||
"horizontal_fov_degrees": 100.0,
|
||||
"vertical_fov_degrees": 100.0,
|
||||
"tile_selection": "maximum-optical-axis-cosine/v1",
|
||||
"rgb_sampling": "opencv-remap-linear",
|
||||
"label_sampling": "nearest",
|
||||
"tiles": [
|
||||
{
|
||||
"name": "front",
|
||||
"yaw_degrees": 0.0,
|
||||
"pitch_degrees": 0.0
|
||||
},
|
||||
{
|
||||
"name": "left",
|
||||
"yaw_degrees": -90.0,
|
||||
"pitch_degrees": 0.0
|
||||
},
|
||||
{
|
||||
"name": "right",
|
||||
"yaw_degrees": 90.0,
|
||||
"pitch_degrees": 0.0
|
||||
},
|
||||
{
|
||||
"name": "up",
|
||||
"yaw_degrees": 0.0,
|
||||
"pitch_degrees": 90.0
|
||||
},
|
||||
{
|
||||
"name": "down",
|
||||
"yaw_degrees": 0.0,
|
||||
"pitch_degrees": -90.0
|
||||
}
|
||||
]
|
||||
},
|
||||
"contrast": {
|
||||
"method": "clahe-lab-luminance",
|
||||
"clip_limit": 2.0,
|
||||
"tile_grid_size": [
|
||||
8,
|
||||
8
|
||||
]
|
||||
},
|
||||
"model": {
|
||||
"id": "tue-mps/cityscapes_semantic_eomt_large_1024",
|
||||
"revision": "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f",
|
||||
"architecture": "EomtForUniversalSegmentation",
|
||||
"precision": "fp16-autocast",
|
||||
"batch_size": 1,
|
||||
"files": {
|
||||
"config.json": {
|
||||
"bytes": 1575,
|
||||
"sha256": "7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650"
|
||||
},
|
||||
"preprocessor_config.json": {
|
||||
"bytes": 666,
|
||||
"sha256": "97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7"
|
||||
},
|
||||
"model.safetensors": {
|
||||
"bytes": 1276175488,
|
||||
"sha256": "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782"
|
||||
}
|
||||
}
|
||||
},
|
||||
"opencv": {
|
||||
"distribution": "opencv-python-headless",
|
||||
"version": "4.13.0.92",
|
||||
"runtime_version": "4.13.0",
|
||||
"wheel": {
|
||||
"filename": "opencv_python_headless-4.13.0.92-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl",
|
||||
"bytes": 56016764,
|
||||
"sha256": "0525a3d2c0b46c611e2130b5fdebc94cf404845d8fa64d2f3a3b679572a5bd22",
|
||||
"url": "https://files.pythonhosted.org/packages/92/ce/bd17ff5772938267fd49716e94ca24f616ff4cb1ff4c6be13085108037be/opencv_python_headless-4.13.0.92-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl"
|
||||
}
|
||||
},
|
||||
"target_taxonomy": {
|
||||
"0": "outside_valid_fov",
|
||||
"1": "person",
|
||||
"2": "bicycle",
|
||||
"3": "motorcycle",
|
||||
"4": "car",
|
||||
"5": "heavy_vehicle",
|
||||
"6": "building_structure",
|
||||
"7": "paved_road",
|
||||
"8": "sidewalk_curb",
|
||||
"9": "ground_dirt",
|
||||
"10": "grass_low_vegetation",
|
||||
"11": "tree_woody_vegetation",
|
||||
"12": "sky",
|
||||
"13": "static_obstacle",
|
||||
"14": "animal",
|
||||
"15": "other_background"
|
||||
},
|
||||
"cityscapes_to_target": {
|
||||
"road": 7,
|
||||
"sidewalk": 8,
|
||||
"building": 6,
|
||||
"wall": 6,
|
||||
"fence": 13,
|
||||
"pole": 13,
|
||||
"traffic light": 13,
|
||||
"traffic sign": 13,
|
||||
"vegetation": 11,
|
||||
"terrain": 10,
|
||||
"sky": 12,
|
||||
"person": 1,
|
||||
"rider": 1,
|
||||
"car": 4,
|
||||
"truck": 5,
|
||||
"bus": 5,
|
||||
"train": 5,
|
||||
"motorcycle": 3,
|
||||
"bicycle": 2
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"schema_version": "missioncore.e5-tracking-profile/v1",
|
||||
"source": {
|
||||
"source_id": "sensor.camera.right",
|
||||
"resolution": [800, 600],
|
||||
"calibration_slot": "camera_1",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
},
|
||||
"model": {
|
||||
"id": "yolox_s",
|
||||
"version": 1,
|
||||
"architecture": "YOLOX-S",
|
||||
"source": "Megvii-BaseDetection/YOLOX release 0.1.1rc0",
|
||||
"license": "Apache-2.0",
|
||||
"model_sha256": "c5c2d13e59ae883e6af3b45daea64af4833a4951c92d116ec270d9ddbe998063",
|
||||
"config_sha256": "5795c737a7935a655961b069e8404d336d891f9762fb6dffb93956a076479604",
|
||||
"input_name": "images",
|
||||
"output_name": "output",
|
||||
"input_shape": [1, 3, 640, 640],
|
||||
"classes": "COCO-80"
|
||||
},
|
||||
"preprocessing": {
|
||||
"color_order": "BGR",
|
||||
"resize": "bilinear-letterbox-top-left",
|
||||
"pad_value": 114,
|
||||
"valid_fov_fill_value": 114
|
||||
},
|
||||
"detection": {
|
||||
"minimum_score": 0.1,
|
||||
"nms_iou_threshold": 0.45,
|
||||
"nms_containment_threshold": 0.8,
|
||||
"target_class_ids": [0, 1, 2, 3, 5, 7],
|
||||
"minimum_box_area_pixels": 64.0,
|
||||
"maximum_box_area_fraction": 0.5,
|
||||
"minimum_valid_fov_fraction": 0.5,
|
||||
"require_center_inside_valid_fov": true
|
||||
},
|
||||
"tracking": {
|
||||
"algorithm": "bytetrack-style-two-stage-iou/v1",
|
||||
"high_score_threshold": 0.25,
|
||||
"new_track_threshold": 0.25,
|
||||
"primary_match_iou": 0.2,
|
||||
"secondary_match_iou": 0.1,
|
||||
"lost_track_buffer_frames": 15,
|
||||
"minimum_confirmed_hits": 2,
|
||||
"assignment_solver": "scipy-linear-sum-assignment"
|
||||
},
|
||||
"overlay": {
|
||||
"line_width": 3,
|
||||
"trail_length": 24,
|
||||
"show_unconfirmed": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"schema_version": "missioncore.e8-realtime-tracking-profile/v1",
|
||||
"mode": "overload-negative-control",
|
||||
"source": {
|
||||
"source_id": "sensor.camera.right",
|
||||
"resolution": [800, 600],
|
||||
"calibration_slot": "camera_1",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
},
|
||||
"model": {
|
||||
"id": "yolox_s",
|
||||
"version": 1,
|
||||
"architecture": "YOLOX-S",
|
||||
"source": "Megvii-BaseDetection/YOLOX release 0.1.1rc0",
|
||||
"license": "Apache-2.0",
|
||||
"model_sha256": "c5c2d13e59ae883e6af3b45daea64af4833a4951c92d116ec270d9ddbe998063",
|
||||
"config_sha256": "5795c737a7935a655961b069e8404d336d891f9762fb6dffb93956a076479604",
|
||||
"input_name": "images",
|
||||
"output_name": "output",
|
||||
"input_shape": [1, 3, 640, 640],
|
||||
"classes": "COCO-80"
|
||||
},
|
||||
"preprocessing": {
|
||||
"color_order": "BGR",
|
||||
"resize": "bilinear-letterbox-top-left",
|
||||
"pad_value": 114,
|
||||
"valid_fov_fill_value": 114
|
||||
},
|
||||
"detection": {
|
||||
"minimum_score": 0.1,
|
||||
"nms_iou_threshold": 0.45,
|
||||
"nms_containment_threshold": 0.8,
|
||||
"target_class_ids": [0, 1, 2, 3, 5, 7],
|
||||
"minimum_box_area_pixels": 64.0,
|
||||
"maximum_box_area_fraction": 0.5,
|
||||
"minimum_valid_fov_fraction": 0.5,
|
||||
"require_center_inside_valid_fov": true
|
||||
},
|
||||
"tracking": {
|
||||
"algorithm": "bytetrack-style-two-stage-iou/v1",
|
||||
"high_score_threshold": 0.25,
|
||||
"new_track_threshold": 0.25,
|
||||
"primary_match_iou": 0.2,
|
||||
"secondary_match_iou": 0.1,
|
||||
"lost_track_buffer_frames": 15,
|
||||
"minimum_confirmed_hits": 2,
|
||||
"assignment_solver": "scipy-linear-sum-assignment"
|
||||
},
|
||||
"realtime": {
|
||||
"queue_policy": "bounded-latest-wins",
|
||||
"queue_capacity": 2,
|
||||
"speed": 1.0,
|
||||
"consumer_delay_ms": 140.0,
|
||||
"stale_after_ms": 150.0,
|
||||
"unavailable_after_ms": 500.0
|
||||
},
|
||||
"acceptance": {
|
||||
"minimum_effective_fps": 0.0,
|
||||
"maximum_drop_fraction": 1.0,
|
||||
"maximum_p95_result_age_ms": 5000.0,
|
||||
"require_zero_failures": true,
|
||||
"expect_overload": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"schema_version": "missioncore.e8-realtime-tracking-profile/v1",
|
||||
"mode": "qualification",
|
||||
"source": {
|
||||
"source_id": "sensor.camera.right",
|
||||
"resolution": [800, 600],
|
||||
"calibration_slot": "camera_1",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
},
|
||||
"model": {
|
||||
"id": "yolox_s",
|
||||
"version": 1,
|
||||
"architecture": "YOLOX-S",
|
||||
"source": "Megvii-BaseDetection/YOLOX release 0.1.1rc0",
|
||||
"license": "Apache-2.0",
|
||||
"model_sha256": "c5c2d13e59ae883e6af3b45daea64af4833a4951c92d116ec270d9ddbe998063",
|
||||
"config_sha256": "5795c737a7935a655961b069e8404d336d891f9762fb6dffb93956a076479604",
|
||||
"input_name": "images",
|
||||
"output_name": "output",
|
||||
"input_shape": [1, 3, 640, 640],
|
||||
"classes": "COCO-80"
|
||||
},
|
||||
"preprocessing": {
|
||||
"color_order": "BGR",
|
||||
"resize": "bilinear-letterbox-top-left",
|
||||
"pad_value": 114,
|
||||
"valid_fov_fill_value": 114
|
||||
},
|
||||
"detection": {
|
||||
"minimum_score": 0.1,
|
||||
"nms_iou_threshold": 0.45,
|
||||
"nms_containment_threshold": 0.8,
|
||||
"target_class_ids": [0, 1, 2, 3, 5, 7],
|
||||
"minimum_box_area_pixels": 64.0,
|
||||
"maximum_box_area_fraction": 0.5,
|
||||
"minimum_valid_fov_fraction": 0.5,
|
||||
"require_center_inside_valid_fov": true
|
||||
},
|
||||
"tracking": {
|
||||
"algorithm": "bytetrack-style-two-stage-iou/v1",
|
||||
"high_score_threshold": 0.25,
|
||||
"new_track_threshold": 0.25,
|
||||
"primary_match_iou": 0.2,
|
||||
"secondary_match_iou": 0.1,
|
||||
"lost_track_buffer_frames": 15,
|
||||
"minimum_confirmed_hits": 2,
|
||||
"assignment_solver": "scipy-linear-sum-assignment"
|
||||
},
|
||||
"realtime": {
|
||||
"queue_policy": "bounded-latest-wins",
|
||||
"queue_capacity": 2,
|
||||
"speed": 1.0,
|
||||
"consumer_delay_ms": 0.0,
|
||||
"stale_after_ms": 150.0,
|
||||
"unavailable_after_ms": 500.0
|
||||
},
|
||||
"acceptance": {
|
||||
"minimum_effective_fps": 9.5,
|
||||
"maximum_drop_fraction": 0.005,
|
||||
"maximum_p95_result_age_ms": 100.0,
|
||||
"require_zero_failures": true,
|
||||
"expect_overload": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"schema_version": "missioncore.e9-multirate-perception-profile/v1",
|
||||
"mode": "qualification",
|
||||
"source": {
|
||||
"source_id": "sensor.camera.right",
|
||||
"resolution": [800, 600],
|
||||
"calibration_slot": "camera_1",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
},
|
||||
"replay": {
|
||||
"speed": 1.0,
|
||||
"detector_queue_capacity": 2,
|
||||
"semantic_queue_capacity": 1,
|
||||
"semantic_sample_every_frames": 5,
|
||||
"semantic_ttl_ms": 750.0
|
||||
},
|
||||
"acceptance": {
|
||||
"detector_minimum_effective_fps": 9.5,
|
||||
"detector_maximum_drop_fraction": 0.005,
|
||||
"detector_maximum_p95_result_age_ms": 100.0,
|
||||
"semantic_minimum_effective_fps": 1.8,
|
||||
"semantic_maximum_drop_fraction": 0.05,
|
||||
"semantic_maximum_p95_completion_age_ms": 350.0,
|
||||
"minimum_fresh_semantic_coverage": 0.9,
|
||||
"require_zero_failures": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"schema_version": "missioncore.e9-multirate-perception-profile/v1",
|
||||
"mode": "qualification",
|
||||
"source": {
|
||||
"source_id": "sensor.camera.right",
|
||||
"resolution": [800, 600],
|
||||
"calibration_slot": "camera_1",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
},
|
||||
"replay": {
|
||||
"speed": 1.0,
|
||||
"detector_queue_capacity": 2,
|
||||
"semantic_queue_capacity": 1,
|
||||
"semantic_sample_every_frames": 5,
|
||||
"semantic_ttl_ms": 750.0
|
||||
},
|
||||
"acceptance": {
|
||||
"detector_minimum_effective_fps": 9.5,
|
||||
"detector_maximum_drop_fraction": 0.005,
|
||||
"detector_maximum_p95_result_age_ms": 150.0,
|
||||
"semantic_minimum_effective_fps": 1.8,
|
||||
"semantic_maximum_drop_fraction": 0.05,
|
||||
"semantic_maximum_p95_completion_age_ms": 350.0,
|
||||
"minimum_fresh_semantic_coverage": 0.9,
|
||||
"require_zero_failures": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,976 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run camera inference, LiDAR fusion and world-state publication in one paced loop."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import resource
|
||||
import shutil
|
||||
import threading
|
||||
import time
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import run_e9_multirate_perception as e9
|
||||
from e10_fusion_runtime import (
|
||||
CuboidCompletionTracker,
|
||||
LidarReplayPack,
|
||||
WorldStateProjector,
|
||||
_completion_profile,
|
||||
canonical_json,
|
||||
clearance,
|
||||
distance_history,
|
||||
fuse_tracks,
|
||||
fusion_document,
|
||||
project_points,
|
||||
sha256,
|
||||
)
|
||||
from run_e4_full_session_segmentation import TARGET_CLASS_COUNT, _dependency_manifest, _load_model
|
||||
from run_e4_full_session_segmentation import _profile as read_semantic_profile
|
||||
from run_e4_full_session_segmentation import _validate_source as validate_semantic_source
|
||||
from run_e5_instance_tracking import (
|
||||
TwoStageTracker,
|
||||
_detections,
|
||||
_infer,
|
||||
_load_valid_fov,
|
||||
_preprocess,
|
||||
_read_timeline,
|
||||
_track_document,
|
||||
_validate_source,
|
||||
_verify_model,
|
||||
)
|
||||
from run_e8_realtime_tracking import LatestWinsQueue
|
||||
from run_e8_realtime_tracking import _read_profile as read_detector_profile
|
||||
from run_recorded_perception_epoch import (
|
||||
_artifact,
|
||||
_GpuTelemetry,
|
||||
_percentiles,
|
||||
_valid_sha256,
|
||||
_validate_job,
|
||||
_write_json,
|
||||
)
|
||||
|
||||
PROFILE_SCHEMA = "missioncore.e10-integrated-perception-profile/v1"
|
||||
RESULT_SCHEMA = "missioncore.e10-integrated-perception-result/v1"
|
||||
REPORT_SCHEMA = "missioncore.e10-integrated-perception-report/v1"
|
||||
IDENTITY_SCHEMA = "missioncore.e10-integrated-perception-identity/v1"
|
||||
SEMANTIC_SCHEMA = "missioncore.e10-semantic-frame/v1"
|
||||
FUSION_SCHEMA = "missioncore.e10-fusion-frame/v1"
|
||||
WORLD_SCHEMA = "missioncore.live-perception-world-state/v1"
|
||||
PIPELINE_ID = "source-paced-yolox-eomt-kb4-lidar-world-state/v1"
|
||||
|
||||
|
||||
def arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
for name in ("preflight", "run"):
|
||||
command = commands.add_parser(name)
|
||||
command.add_argument("--job", type=Path, required=True)
|
||||
command.add_argument("--profile", type=Path, required=True)
|
||||
command.add_argument("--detector-profile", type=Path, required=True)
|
||||
command.add_argument("--semantic-profile", type=Path, required=True)
|
||||
command.add_argument("--valid-fov-root", type=Path, required=True)
|
||||
command.add_argument("--model-root", type=Path, required=True)
|
||||
command.add_argument("--cache", type=Path, required=True)
|
||||
command.add_argument("--environment", type=Path, required=True)
|
||||
command.add_argument("--lidar-pack", type=Path, required=True)
|
||||
if name == "run":
|
||||
command.add_argument("--frames", type=Path, required=True)
|
||||
command.add_argument("--timeline", type=Path, required=True)
|
||||
command.add_argument("--output", type=Path, required=True)
|
||||
command.add_argument("--triton-url", required=True)
|
||||
command.add_argument("--free-bytes-floor", type=int, default=0)
|
||||
command.add_argument("--orchestrator-sha256", required=True)
|
||||
command.add_argument("--container-image", required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def read_object(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise RuntimeError(f"JSON root is not an object: {path}")
|
||||
return value
|
||||
|
||||
|
||||
def read_profile(path: Path) -> tuple[dict[str, Any], str]:
|
||||
resolved = path.resolve(strict=True)
|
||||
profile = read_object(resolved)
|
||||
replay = profile.get("replay")
|
||||
source = profile.get("source")
|
||||
association = profile.get("association")
|
||||
world = profile.get("world_state")
|
||||
acceptance = profile.get("acceptance")
|
||||
if (
|
||||
profile.get("schema_version") != PROFILE_SCHEMA
|
||||
or profile.get("mode")
|
||||
not in {
|
||||
"pilot",
|
||||
"qualification",
|
||||
"full-session-qualification",
|
||||
"semantic-loss-negative-control",
|
||||
}
|
||||
or not all(
|
||||
isinstance(value, dict) for value in (replay, source, association, world, acceptance)
|
||||
)
|
||||
or source.get("source_id") != "sensor.camera.right"
|
||||
or source.get("resolution") != [800, 600]
|
||||
or set(association.get("semantic_ids", {}))
|
||||
!= {"person", "bicycle", "motorcycle", "vehicle"}
|
||||
or not isinstance(world.get("clearance"), dict)
|
||||
):
|
||||
raise RuntimeError("LAB E10 profile contract is invalid")
|
||||
if (
|
||||
not 0.1 <= float(replay.get("speed", 0)) <= 10
|
||||
or not 1 <= int(replay.get("detector_queue_capacity", 0)) <= 8
|
||||
or not 1 <= int(replay.get("semantic_queue_capacity", 0)) <= 4
|
||||
or not 2 <= int(replay.get("semantic_sample_every_frames", 0)) <= 30
|
||||
or not 100 <= float(replay.get("semantic_ttl_ms", 0)) <= 5000
|
||||
):
|
||||
raise RuntimeError("LAB E10 scheduling contract is invalid")
|
||||
if profile["mode"] == "semantic-loss-negative-control":
|
||||
semantic_loss = profile.get("semantic_loss")
|
||||
if (
|
||||
not isinstance(semantic_loss, dict)
|
||||
or not 1 <= int(semantic_loss.get("stop_after_completed_results", 0)) <= 100
|
||||
or not 1 <= int(acceptance.get("minimum_stale_detector_frames", 0)) <= 1_000_000
|
||||
):
|
||||
raise RuntimeError("LAB E10 semantic-loss contract is invalid")
|
||||
if profile["mode"] == "full-session-qualification":
|
||||
selection = profile.get("selection")
|
||||
if (
|
||||
not isinstance(selection, dict)
|
||||
or int(selection.get("required_frame_count", 0)) < 2
|
||||
or int(selection.get("required_source_start_frame_index", -1)) != 0
|
||||
or int(selection.get("required_source_end_frame_index", -1))
|
||||
!= int(selection["required_frame_count"]) - 1
|
||||
or float(selection.get("minimum_source_span_seconds", 0)) <= 0
|
||||
):
|
||||
raise RuntimeError("LAB E10 full-session selection contract is invalid")
|
||||
completion = profile.get("cuboid_completion")
|
||||
if completion is not None:
|
||||
if not isinstance(completion, dict):
|
||||
raise RuntimeError("LAB E13 cuboid completion contract is invalid")
|
||||
_completion_profile(completion)
|
||||
return profile, sha256(resolved)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SemanticResult:
|
||||
frame_index: int
|
||||
source_frame_index: int
|
||||
session_seconds: float
|
||||
completion_age_ms: float
|
||||
completed_monotonic: float
|
||||
mask: np.ndarray
|
||||
mask_sha256: str
|
||||
class_pixels: dict[str, int]
|
||||
|
||||
|
||||
class LatestSemantic:
|
||||
def __init__(self) -> None:
|
||||
self.lock = threading.Lock()
|
||||
self.value: SemanticResult | None = None
|
||||
|
||||
def publish(self, value: SemanticResult) -> None:
|
||||
with self.lock:
|
||||
if self.value is not None and value.frame_index <= self.value.frame_index:
|
||||
raise RuntimeError("LAB E10 semantic results are not monotonic")
|
||||
self.value = value
|
||||
|
||||
def snapshot(self) -> SemanticResult | None:
|
||||
with self.lock:
|
||||
return self.value
|
||||
|
||||
|
||||
def semantic_worker(
|
||||
*,
|
||||
queue: LatestWinsQueue,
|
||||
latest: LatestSemantic,
|
||||
valid_mask: np.ndarray,
|
||||
target_lut: np.ndarray,
|
||||
target_names: dict[int, str],
|
||||
infer: Any,
|
||||
latency: dict[str, list[float]],
|
||||
completed: list[SemanticResult],
|
||||
failures: list[BaseException],
|
||||
stop_after_results: int | None,
|
||||
) -> None:
|
||||
try:
|
||||
while (envelope := queue.take()) is not None:
|
||||
started = time.perf_counter()
|
||||
latency["queue_wait_ms"].append(max(0.0, (started - envelope.decoded_monotonic) * 1000))
|
||||
fill_started = time.perf_counter()
|
||||
model_input = np.where(valid_mask[..., None], envelope.image, 0).astype(np.uint8)
|
||||
latency["valid_fov_fill_ms"].append((time.perf_counter() - fill_started) * 1000)
|
||||
semantic, measured = infer(model_input)
|
||||
if int(semantic.max()) >= len(target_lut):
|
||||
raise RuntimeError("LAB E10 EoMT emitted an unknown category")
|
||||
target = target_lut[semantic].copy()
|
||||
target[~valid_mask] = 0
|
||||
for name, value in measured.items():
|
||||
latency[name].append(float(value))
|
||||
finished = time.perf_counter()
|
||||
age = max(0.0, (finished - envelope.scheduled_monotonic) * 1000)
|
||||
latency["completion_age_ms"].append(age)
|
||||
latency["processing_ms"].append((finished - started) * 1000)
|
||||
counts = np.bincount(target[valid_mask], minlength=TARGET_CLASS_COUNT)
|
||||
result = SemanticResult(
|
||||
frame_index=envelope.frame_index,
|
||||
source_frame_index=int(envelope.timeline["source_frame_index"]),
|
||||
session_seconds=float(envelope.timeline["session_seconds"]),
|
||||
completion_age_ms=age,
|
||||
completed_monotonic=finished,
|
||||
mask=target,
|
||||
mask_sha256=hashlib.sha256(target.tobytes()).hexdigest(),
|
||||
class_pixels={
|
||||
target_names[index]: int(counts[index])
|
||||
for index in range(1, TARGET_CLASS_COUNT)
|
||||
if int(counts[index]) > 0
|
||||
},
|
||||
)
|
||||
completed.append(result)
|
||||
latest.publish(result)
|
||||
if stop_after_results is not None and len(completed) >= stop_after_results:
|
||||
return
|
||||
except BaseException as exc:
|
||||
failures.append(exc)
|
||||
|
||||
|
||||
def semantic_binding(
|
||||
semantic: SemanticResult | None, frame_seconds: float, ttl_ms: float
|
||||
) -> tuple[str, float | None]:
|
||||
if semantic is None:
|
||||
return "unavailable", None
|
||||
age = max(0.0, (frame_seconds - semantic.session_seconds) * 1000)
|
||||
return ("fresh" if age <= ttl_ms else "stale"), age
|
||||
|
||||
|
||||
def preflight(args: argparse.Namespace) -> int:
|
||||
import torch
|
||||
|
||||
job = _validate_job(args.job.resolve(strict=True))
|
||||
profile, profile_sha256 = read_profile(args.profile)
|
||||
detector, detector_sha256 = read_detector_profile(args.detector_profile)
|
||||
semantic, semantic_sha256 = read_semantic_profile(args.semantic_profile)
|
||||
_validate_source(job, detector)
|
||||
validate_semantic_source(job, semantic)
|
||||
if profile["source"] != detector["source"]:
|
||||
raise RuntimeError("LAB E10 detector source binding changed")
|
||||
if profile["source"] != {
|
||||
key: semantic["source"][key]
|
||||
for key in ("source_id", "resolution", "calibration_slot", "calibration_sha256")
|
||||
}:
|
||||
raise RuntimeError("LAB E10 semantic source binding changed")
|
||||
lidar = LidarReplayPack(args.lidar_pack, expected_job_id=job["job_id"])
|
||||
try:
|
||||
if lidar.identity["calibration_sha256"] != profile["source"]["calibration_sha256"]:
|
||||
raise RuntimeError("LAB E10 LiDAR calibration binding changed")
|
||||
if profile["mode"] == "full-session-qualification":
|
||||
selection = profile["selection"]
|
||||
if (
|
||||
lidar.frame_count != int(selection["required_frame_count"])
|
||||
or int(lidar.source_frame_indices[0])
|
||||
!= int(selection["required_source_start_frame_index"])
|
||||
or int(lidar.source_frame_indices[-1])
|
||||
!= int(selection["required_source_end_frame_index"])
|
||||
or float(lidar.session_seconds[-1] - lidar.session_seconds[0])
|
||||
< float(selection["minimum_source_span_seconds"])
|
||||
):
|
||||
raise RuntimeError("LAB E10 full-session LiDAR selection changed")
|
||||
_load_valid_fov(args.valid_fov_root, job, detector)
|
||||
detector_files = _verify_model(detector, args.model_root)
|
||||
dependency = _dependency_manifest(args.environment.resolve(strict=True))
|
||||
if dependency["identity"]["profile_sha256"] != semantic_sha256:
|
||||
raise RuntimeError("LAB E10 semantic dependency identity changed")
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("CUDA is required for LAB E10")
|
||||
device = torch.device("cuda:0")
|
||||
processor, model, _labels, _lut, semantic_files = _load_model(
|
||||
semantic, args.cache.resolve(strict=True), device
|
||||
)
|
||||
del processor, model, _labels, _lut
|
||||
torch.cuda.empty_cache()
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"state": "preflight-ready",
|
||||
"job_id": job["job_id"],
|
||||
"profile_sha256": profile_sha256,
|
||||
"detector_profile_sha256": detector_sha256,
|
||||
"semantic_profile_sha256": semantic_sha256,
|
||||
"lidar_pack_id": lidar.pack_id,
|
||||
"detector_files": detector_files,
|
||||
"semantic_files": semantic_files,
|
||||
"cuda_device": torch.cuda.get_device_name(),
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
finally:
|
||||
lidar.close()
|
||||
return 0
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> int:
|
||||
import torch
|
||||
import transformers
|
||||
from PIL import Image
|
||||
from scipy.optimize import linear_sum_assignment
|
||||
|
||||
if not _valid_sha256(args.orchestrator_sha256):
|
||||
raise RuntimeError("LAB E10 orchestrator SHA-256 is invalid")
|
||||
job = _validate_job(args.job.resolve(strict=True))
|
||||
profile, profile_sha256 = read_profile(args.profile)
|
||||
detector, detector_sha256 = read_detector_profile(args.detector_profile)
|
||||
semantic, semantic_sha256 = read_semantic_profile(args.semantic_profile)
|
||||
_validate_source(job, detector)
|
||||
validate_semantic_source(job, semantic)
|
||||
valid_mask, valid_fov = _load_valid_fov(args.valid_fov_root, job, detector)
|
||||
detector_files = _verify_model(detector, args.model_root)
|
||||
dependency = _dependency_manifest(args.environment.resolve(strict=True))
|
||||
if dependency["identity"]["profile_sha256"] != semantic_sha256:
|
||||
raise RuntimeError("LAB E10 semantic dependency identity changed")
|
||||
lidar = LidarReplayPack(args.lidar_pack, expected_job_id=job["job_id"])
|
||||
frames = sorted(args.frames.resolve(strict=True).glob("frame-*.png"))
|
||||
timeline_path = args.timeline.resolve(strict=True)
|
||||
timeline = _read_timeline(timeline_path, len(frames))
|
||||
if (
|
||||
len(frames) != lidar.frame_count
|
||||
or [int(row["source_frame_index"]) for row in timeline]
|
||||
!= lidar.source_frame_indices.tolist()
|
||||
or not np.allclose(
|
||||
np.asarray([float(row["session_seconds"]) for row in timeline]),
|
||||
lidar.session_seconds,
|
||||
rtol=0,
|
||||
atol=1e-6,
|
||||
)
|
||||
):
|
||||
lidar.close()
|
||||
raise RuntimeError("LAB E10 camera and LiDAR replay selections differ")
|
||||
if profile["mode"] == "full-session-qualification":
|
||||
selection = profile["selection"]
|
||||
source_span_seconds = float(timeline[-1]["session_seconds"]) - float(
|
||||
timeline[0]["session_seconds"]
|
||||
)
|
||||
if (
|
||||
len(frames) != int(selection["required_frame_count"])
|
||||
or int(timeline[0]["source_frame_index"])
|
||||
!= int(selection["required_source_start_frame_index"])
|
||||
or int(timeline[-1]["source_frame_index"])
|
||||
!= int(selection["required_source_end_frame_index"])
|
||||
or source_span_seconds < float(selection["minimum_source_span_seconds"])
|
||||
):
|
||||
lidar.close()
|
||||
raise RuntimeError("LAB E10 full-session camera selection changed")
|
||||
output = args.output.resolve()
|
||||
if output.exists():
|
||||
raise RuntimeError("LAB E10 output must be absent")
|
||||
output.mkdir(mode=0o700, parents=True, exist_ok=False)
|
||||
assert_disk(output, args.free_bytes_floor, 0)
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
processor, semantic_model, _semantic_labels, target_lut, semantic_files = _load_model(
|
||||
semantic, args.cache.resolve(strict=True), device
|
||||
)
|
||||
target_names = {int(key): str(value) for key, value in semantic["target_taxonomy"].items()}
|
||||
infer_semantic = e9._semantic_infer_factory(processor, semantic_model, device)
|
||||
tracker = TwoStageTracker(detector["tracking"])
|
||||
linear_sum_assignment(np.zeros((1, 1), dtype=np.float64))
|
||||
with Image.open(frames[0]) as opened:
|
||||
warm = np.asarray(opened.convert("RGB"), dtype=np.uint8)
|
||||
_infer(args.triton_url, detector["model"], _preprocess(warm, valid_mask, detector))
|
||||
infer_semantic(np.where(valid_mask[..., None], warm, 0).astype(np.uint8))
|
||||
del warm
|
||||
|
||||
replay = profile["replay"]
|
||||
detector_queue = LatestWinsQueue(int(replay["detector_queue_capacity"]))
|
||||
semantic_queue = LatestWinsQueue(int(replay["semantic_queue_capacity"]))
|
||||
latest = LatestSemantic()
|
||||
completed_semantics: list[SemanticResult] = []
|
||||
producer_errors: list[BaseException] = []
|
||||
semantic_errors: list[BaseException] = []
|
||||
semantic_latency = {
|
||||
name: []
|
||||
for name in (
|
||||
"queue_wait_ms",
|
||||
"valid_fov_fill_ms",
|
||||
"processor_ms",
|
||||
"host_to_device_ms",
|
||||
"forward_ms",
|
||||
"model_postprocess_ms",
|
||||
"processing_ms",
|
||||
"completion_age_ms",
|
||||
)
|
||||
}
|
||||
latency = {
|
||||
name: []
|
||||
for name in (
|
||||
"decode_ms",
|
||||
"queue_wait_ms",
|
||||
"detector_ms",
|
||||
"projection_ms",
|
||||
"association_ms",
|
||||
"clearance_ms",
|
||||
"world_state_ms",
|
||||
"world_state_age_ms",
|
||||
)
|
||||
}
|
||||
status_counts: Counter[str] = Counter()
|
||||
fusion_state_counts: Counter[str] = Counter()
|
||||
rejection_counts: Counter[str] = Counter()
|
||||
accepted_cuboids = 0
|
||||
fused_frames = 0
|
||||
detector_failures = 0
|
||||
fusion_freshness_violations = 0
|
||||
history = distance_history(int(profile["association"]["distance_history_frames"]))
|
||||
projector = WorldStateProjector(float(profile["world_state"]["velocity_history_limit_s"]))
|
||||
completion_tracker = (
|
||||
CuboidCompletionTracker(profile["cuboid_completion"])
|
||||
if "cuboid_completion" in profile
|
||||
else None
|
||||
)
|
||||
semantic_path = output / "semantic-frames.jsonl"
|
||||
fusion_path = output / "fusion-frames.jsonl"
|
||||
world_path = output / "world-state.jsonl"
|
||||
gpu_path = output / "gpu-telemetry.jsonl"
|
||||
support_offsets = [0]
|
||||
support_points: list[np.ndarray] = []
|
||||
support_colors: list[np.ndarray] = []
|
||||
box_offsets = [0]
|
||||
box_centers: list[tuple[float, float, float]] = []
|
||||
box_half_sizes: list[tuple[float, float, float]] = []
|
||||
box_quaternions: list[tuple[float, float, float, float]] = []
|
||||
box_colors: list[tuple[int, int, int, int]] = []
|
||||
frame_times_ns: list[int] = []
|
||||
disk_before = shutil.disk_usage(output).free
|
||||
|
||||
with (
|
||||
semantic_path.open("x", encoding="utf-8", newline="\n") as semantic_stream,
|
||||
fusion_path.open("x", encoding="utf-8", newline="\n") as fusion_stream,
|
||||
world_path.open("x", encoding="utf-8", newline="\n") as world_stream,
|
||||
gpu_path.open("x", encoding="utf-8", newline="\n") as gpu_stream,
|
||||
_GpuTelemetry(gpu_stream, 1.0) as gpu,
|
||||
):
|
||||
semantic_thread = threading.Thread(
|
||||
target=semantic_worker,
|
||||
kwargs={
|
||||
"queue": semantic_queue,
|
||||
"latest": latest,
|
||||
"valid_mask": valid_mask,
|
||||
"target_lut": target_lut,
|
||||
"target_names": target_names,
|
||||
"infer": infer_semantic,
|
||||
"latency": semantic_latency,
|
||||
"completed": completed_semantics,
|
||||
"failures": semantic_errors,
|
||||
"stop_after_results": (
|
||||
int(profile["semantic_loss"]["stop_after_completed_results"])
|
||||
if profile["mode"] == "semantic-loss-negative-control"
|
||||
else None
|
||||
),
|
||||
},
|
||||
name="lab-e10-semantic",
|
||||
daemon=True,
|
||||
)
|
||||
semantic_thread.start()
|
||||
replay_started = time.perf_counter() + 0.25
|
||||
producer = threading.Thread(
|
||||
target=e9._producer,
|
||||
kwargs={
|
||||
"detector_queue": detector_queue,
|
||||
"semantic_queue": semantic_queue,
|
||||
"semantic_stride": int(replay["semantic_sample_every_frames"]),
|
||||
"frame_paths": frames,
|
||||
"timeline_rows": timeline,
|
||||
"replay_started": replay_started,
|
||||
"speed": float(replay["speed"]),
|
||||
"error": producer_errors,
|
||||
},
|
||||
name="lab-e10-source",
|
||||
daemon=True,
|
||||
)
|
||||
producer.start()
|
||||
|
||||
while (envelope := detector_queue.take()) is not None:
|
||||
started = time.perf_counter()
|
||||
latency["decode_ms"].append(envelope.decode_ms)
|
||||
latency["queue_wait_ms"].append(max(0.0, (started - envelope.decoded_monotonic) * 1000))
|
||||
try:
|
||||
detector_started = time.perf_counter()
|
||||
tensor = _preprocess(envelope.image, valid_mask, detector)
|
||||
output_tensor, _request_ms = _infer(args.triton_url, detector["model"], tensor)
|
||||
detections, _rejected = _detections(output_tensor, detector, valid_mask)
|
||||
tracks = tracker.update(detections, envelope.frame_index)
|
||||
latency["detector_ms"].append((time.perf_counter() - detector_started) * 1000)
|
||||
|
||||
frame_seconds = float(envelope.timeline["session_seconds"])
|
||||
current_semantic = latest.snapshot()
|
||||
semantic_status, semantic_age = semantic_binding(
|
||||
current_semantic,
|
||||
frame_seconds,
|
||||
float(replay["semantic_ttl_ms"]),
|
||||
)
|
||||
status_counts[semantic_status] += 1
|
||||
lidar_frame = lidar.frame(envelope.frame_index)
|
||||
fusions = ()
|
||||
points_lidar = np.empty((0, 3), dtype=np.float64)
|
||||
if lidar_frame is None:
|
||||
fusion_state = "depth-unavailable-sync-gate"
|
||||
elif semantic_status != "fresh" or current_semantic is None:
|
||||
fusion_state = f"semantic-{semantic_status}"
|
||||
else:
|
||||
points_map, position, quaternion = lidar_frame
|
||||
projection_started = time.perf_counter()
|
||||
pixels, depths, source_indices, points_lidar = project_points(
|
||||
points_map, position, quaternion, lidar.profile
|
||||
)
|
||||
latency["projection_ms"].append(
|
||||
(time.perf_counter() - projection_started) * 1000
|
||||
)
|
||||
association_started = time.perf_counter()
|
||||
fusions = fuse_tracks(
|
||||
tracks=[_track_document(track) for track in tracks],
|
||||
semantic_map=current_semantic.mask,
|
||||
pixels=pixels,
|
||||
depths=depths,
|
||||
source_indices=source_indices,
|
||||
points_map=points_map,
|
||||
points_lidar=points_lidar,
|
||||
association=profile["association"],
|
||||
distance_history=history,
|
||||
completion_tracker=completion_tracker,
|
||||
sensor_position_map=position,
|
||||
session_seconds=frame_seconds,
|
||||
)
|
||||
latency["association_ms"].append(
|
||||
(time.perf_counter() - association_started) * 1000
|
||||
)
|
||||
fusion_state = "fused"
|
||||
fused_frames += 1
|
||||
if fusion_state == "fused" and semantic_status != "fresh":
|
||||
fusion_freshness_violations += 1
|
||||
fusion_state_counts[fusion_state] += 1
|
||||
accepted = [item for item in fusions if item.cuboid is not None]
|
||||
accepted_cuboids += len(accepted)
|
||||
for item in fusions:
|
||||
rejection_counts[item.status] += 1
|
||||
frame_support = []
|
||||
frame_support_colors = []
|
||||
for item in accepted:
|
||||
color_seed = hashlib.sha256(
|
||||
f"e10:{item.association_group}:{item.track_id}".encode()
|
||||
).digest()
|
||||
color = tuple(64 + value % 176 for value in color_seed[:3])
|
||||
if lidar_frame is None:
|
||||
continue
|
||||
points_map = lidar_frame[0]
|
||||
values = points_map[item.source_indices].astype(np.float32)
|
||||
frame_support.append(values)
|
||||
frame_support_colors.append(
|
||||
np.tile(np.asarray([color], dtype=np.uint8), (values.shape[0], 1))
|
||||
)
|
||||
box_centers.append(item.cuboid.center_map)
|
||||
box_half_sizes.append(item.cuboid.half_size)
|
||||
box_quaternions.append(item.cuboid.quaternion_xyzw)
|
||||
box_colors.append((*color, 88))
|
||||
if frame_support:
|
||||
support = np.concatenate(frame_support)
|
||||
colors = np.concatenate(frame_support_colors)
|
||||
support_points.append(support)
|
||||
support_colors.append(colors)
|
||||
support_offsets.append(support_offsets[-1] + support.shape[0])
|
||||
else:
|
||||
support_offsets.append(support_offsets[-1])
|
||||
box_offsets.append(box_offsets[-1] + len(accepted))
|
||||
clearance_started = time.perf_counter()
|
||||
clearance_state = clearance(points_lidar, profile["world_state"]["clearance"])
|
||||
latency["clearance_ms"].append((time.perf_counter() - clearance_started) * 1000)
|
||||
world_started = time.perf_counter()
|
||||
result_age = max(0.0, (world_started - envelope.scheduled_monotonic) * 1000)
|
||||
if result_age >= 1000:
|
||||
health = "unavailable"
|
||||
elif result_age >= float(profile["acceptance"]["maximum_p95_world_state_age_ms"]):
|
||||
health = "stale"
|
||||
elif fusion_state != "fused":
|
||||
health = "degraded"
|
||||
else:
|
||||
health = "healthy"
|
||||
delivery = {
|
||||
"health": health,
|
||||
"result_age_ms": result_age,
|
||||
"semantic_status": semantic_status,
|
||||
"semantic_source_age_ms": semantic_age,
|
||||
}
|
||||
world = projector.project(
|
||||
frame_index=envelope.frame_index,
|
||||
source_frame_index=int(envelope.timeline["source_frame_index"]),
|
||||
session_seconds=frame_seconds,
|
||||
fusion_state=fusion_state,
|
||||
fusions=fusions,
|
||||
points_lidar=points_lidar,
|
||||
clearance_state=clearance_state,
|
||||
delivery=delivery,
|
||||
)
|
||||
latency["world_state_ms"].append((time.perf_counter() - world_started) * 1000)
|
||||
result_age = max(0.0, (time.perf_counter() - envelope.scheduled_monotonic) * 1000)
|
||||
world["delivery"]["result_age_ms"] = result_age
|
||||
latency["world_state_age_ms"].append(result_age)
|
||||
frame_times_ns.append(round(frame_seconds * 1e9))
|
||||
fusion_stream.write(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": FUSION_SCHEMA,
|
||||
"frame_index": envelope.frame_index,
|
||||
"source_frame_index": int(envelope.timeline["source_frame_index"]),
|
||||
"session_seconds": frame_seconds,
|
||||
"fusion_state": fusion_state,
|
||||
"semantic_status": semantic_status,
|
||||
"semantic_source_frame_index": None
|
||||
if current_semantic is None
|
||||
else current_semantic.source_frame_index,
|
||||
"objects": [fusion_document(item) for item in fusions],
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
world_stream.write(
|
||||
json.dumps(world, sort_keys=True, separators=(",", ":"), allow_nan=False) + "\n"
|
||||
)
|
||||
except Exception:
|
||||
detector_failures += 1
|
||||
raise
|
||||
consumed = int(detector_queue.snapshot()["consumed"])
|
||||
if consumed % 100 == 0:
|
||||
fusion_stream.flush()
|
||||
world_stream.flush()
|
||||
assert_disk(output, args.free_bytes_floor, consumed)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"phase": "e10-integrated",
|
||||
"processed": consumed,
|
||||
"detector_dropped": detector_queue.snapshot()["dropped_overflow"],
|
||||
"semantic_processed": semantic_queue.snapshot()["consumed"],
|
||||
"fused_frames": fused_frames,
|
||||
"accepted_cuboids": accepted_cuboids,
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
producer.join(timeout=5)
|
||||
semantic_thread.join(timeout=30)
|
||||
if producer.is_alive() or producer_errors:
|
||||
raise RuntimeError("LAB E10 producer failed")
|
||||
if semantic_thread.is_alive() or semantic_errors:
|
||||
raise RuntimeError("LAB E10 semantic worker failed")
|
||||
for result in completed_semantics:
|
||||
semantic_stream.write(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": SEMANTIC_SCHEMA,
|
||||
"frame_index": result.frame_index,
|
||||
"source_frame_index": result.source_frame_index,
|
||||
"session_seconds": result.session_seconds,
|
||||
"completion_age_ms": result.completion_age_ms,
|
||||
"mask_sha256": result.mask_sha256,
|
||||
"class_pixels": result.class_pixels,
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
for stream in (semantic_stream, fusion_stream, world_stream):
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
replay_finished = time.perf_counter()
|
||||
detector_state = detector_queue.snapshot()
|
||||
semantic_state = semantic_queue.snapshot()
|
||||
source_span = (
|
||||
float(timeline[-1]["session_seconds"]) - float(timeline[0]["session_seconds"])
|
||||
) / float(replay["speed"])
|
||||
replay_wall = replay_finished - replay_started
|
||||
detector_fps = int(detector_state["consumed"]) / max(source_span, replay_wall, 1e-9)
|
||||
semantic_fps = int(semantic_state["consumed"]) / max(source_span, replay_wall, 1e-9)
|
||||
scheduled_semantic = ((len(frames) - 1) // int(replay["semantic_sample_every_frames"])) + 1
|
||||
fresh_coverage = status_counts["fresh"] / max(1, int(detector_state["consumed"]))
|
||||
latency_summary = {name: _percentiles(values) for name, values in latency.items()}
|
||||
semantic_summary = {name: _percentiles(values) for name, values in semantic_latency.items()}
|
||||
|
||||
arrays_path = output / "transient-perception.npz"
|
||||
np.savez_compressed(
|
||||
arrays_path,
|
||||
frame_times_ns=np.asarray(frame_times_ns, dtype=np.int64),
|
||||
semantic_frame_indices=np.asarray(
|
||||
[value.frame_index for value in completed_semantics], dtype=np.int64
|
||||
),
|
||||
semantic_masks=np.stack([value.mask for value in completed_semantics]).astype(np.uint8),
|
||||
support_offsets=np.asarray(support_offsets, dtype=np.int64),
|
||||
support_points=np.concatenate(support_points)
|
||||
if support_points
|
||||
else np.empty((0, 3), dtype=np.float32),
|
||||
support_colors=np.concatenate(support_colors)
|
||||
if support_colors
|
||||
else np.empty((0, 3), dtype=np.uint8),
|
||||
box_offsets=np.asarray(box_offsets, dtype=np.int64),
|
||||
box_centers=np.asarray(box_centers, dtype=np.float32).reshape((-1, 3)),
|
||||
box_half_sizes=np.asarray(box_half_sizes, dtype=np.float32).reshape((-1, 3)),
|
||||
box_quaternions=np.asarray(box_quaternions, dtype=np.float32).reshape((-1, 4)),
|
||||
box_colors=np.asarray(box_colors, dtype=np.uint8).reshape((-1, 4)),
|
||||
)
|
||||
acceptance = profile["acceptance"]
|
||||
detector_checks = {
|
||||
"detector_accounting": int(detector_state["consumed"])
|
||||
+ int(detector_state["dropped_overflow"])
|
||||
== len(frames),
|
||||
"detector_minimum_effective_fps": detector_fps
|
||||
>= float(acceptance["detector_minimum_effective_fps"]),
|
||||
"detector_maximum_drop_fraction": int(detector_state["dropped_overflow"]) / len(frames)
|
||||
<= float(acceptance["detector_maximum_drop_fraction"]),
|
||||
"maximum_p95_world_state_age_ms": float(latency_summary["world_state_age_ms"]["p95"])
|
||||
<= float(acceptance["maximum_p95_world_state_age_ms"]),
|
||||
"zero_detector_failures": detector_failures == 0 and not producer_errors,
|
||||
}
|
||||
if profile["mode"] == "semantic-loss-negative-control":
|
||||
stop_after = int(profile["semantic_loss"]["stop_after_completed_results"])
|
||||
checks = {
|
||||
**detector_checks,
|
||||
"semantic_loss_triggered": len(completed_semantics) == stop_after,
|
||||
"semantic_queue_accounting": int(semantic_state["consumed"])
|
||||
+ int(semantic_state["dropped_overflow"])
|
||||
+ int(semantic_state["final_depth"])
|
||||
== scheduled_semantic,
|
||||
"minimum_stale_detector_frames": status_counts["stale"]
|
||||
>= int(acceptance["minimum_stale_detector_frames"]),
|
||||
"no_fusion_with_nonfresh_semantics": fusion_freshness_violations == 0,
|
||||
"semantic_worker_stopped_without_error": not semantic_errors,
|
||||
}
|
||||
else:
|
||||
checks = {
|
||||
**detector_checks,
|
||||
"semantic_accounting": int(semantic_state["consumed"])
|
||||
+ int(semantic_state["dropped_overflow"])
|
||||
== scheduled_semantic,
|
||||
"semantic_minimum_effective_fps": semantic_fps
|
||||
>= float(acceptance["semantic_minimum_effective_fps"]),
|
||||
"semantic_maximum_drop_fraction": int(semantic_state["dropped_overflow"])
|
||||
/ scheduled_semantic
|
||||
<= float(acceptance["semantic_maximum_drop_fraction"]),
|
||||
"semantic_maximum_p95_completion_age_ms": float(
|
||||
semantic_summary["completion_age_ms"]["p95"]
|
||||
)
|
||||
<= float(acceptance["semantic_maximum_p95_completion_age_ms"]),
|
||||
"minimum_fresh_semantic_coverage": fresh_coverage
|
||||
>= float(acceptance["minimum_fresh_semantic_coverage"]),
|
||||
"minimum_lidar_fused_frames": fused_frames
|
||||
>= int(acceptance["minimum_lidar_fused_frames"]),
|
||||
"minimum_accepted_cuboids": accepted_cuboids
|
||||
>= int(acceptance["minimum_accepted_cuboids"]),
|
||||
"zero_semantic_failures": not semantic_errors,
|
||||
}
|
||||
accepted = all(checks.values())
|
||||
identity = {
|
||||
"schema_version": IDENTITY_SCHEMA,
|
||||
"job_id": job["job_id"],
|
||||
"input_sha256": job["input_sha256"],
|
||||
"session_id": job["input"]["session_id"],
|
||||
"source_id": job["input"]["source_id"],
|
||||
"lidar_pack_id": lidar.pack_id,
|
||||
"selection": {
|
||||
"frame_count": len(frames),
|
||||
"source_start_frame_index": timeline[0]["source_frame_index"],
|
||||
"source_end_frame_index": timeline[-1]["source_frame_index"],
|
||||
"timeline_start_seconds": timeline[0]["session_seconds"],
|
||||
"timeline_end_seconds": timeline[-1]["session_seconds"],
|
||||
"timeline_sha256": sha256(timeline_path),
|
||||
},
|
||||
"configuration": {
|
||||
"pipeline": PIPELINE_ID,
|
||||
"profile": profile,
|
||||
"profile_sha256": profile_sha256,
|
||||
"detector_profile_sha256": detector_sha256,
|
||||
"semantic_profile_sha256": semantic_sha256,
|
||||
"runner_sha256": sha256(Path(__file__).resolve(strict=True)),
|
||||
"fusion_runtime_sha256": sha256(
|
||||
Path(__file__).with_name("e10_fusion_runtime.py").resolve(strict=True)
|
||||
),
|
||||
"orchestrator_sha256": args.orchestrator_sha256,
|
||||
"container_image": args.container_image,
|
||||
"valid_fov": valid_fov,
|
||||
},
|
||||
"models": {
|
||||
"detector": detector["model"],
|
||||
"detector_files": detector_files,
|
||||
"semantic": semantic["model"],
|
||||
"semantic_files": semantic_files,
|
||||
},
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest()
|
||||
result_id = f"e10-integrated-perception-{identity_sha256}"
|
||||
metrics = {
|
||||
"source_span_seconds": source_span,
|
||||
"replay_wall_seconds": replay_wall,
|
||||
"detector": {
|
||||
"frames_processed": detector_state["consumed"],
|
||||
"frames_dropped": detector_state["dropped_overflow"],
|
||||
"effective_fps": detector_fps,
|
||||
"queue": detector_state,
|
||||
},
|
||||
"semantic": {
|
||||
"frames_processed": semantic_state["consumed"],
|
||||
"frames_dropped": semantic_state["dropped_overflow"],
|
||||
"effective_fps": semantic_fps,
|
||||
"fresh_coverage": fresh_coverage,
|
||||
"status_counts": dict(status_counts),
|
||||
"latency_ms": semantic_summary,
|
||||
},
|
||||
"fusion": {
|
||||
"fused_frames": fused_frames,
|
||||
"fusion_state_counts": dict(fusion_state_counts),
|
||||
"accepted_cuboids": accepted_cuboids,
|
||||
"rejection_counts": dict(rejection_counts),
|
||||
"freshness_violations": fusion_freshness_violations,
|
||||
},
|
||||
"semantic_loss": {
|
||||
"configured": profile["mode"] == "semantic-loss-negative-control",
|
||||
"stop_after_completed_results": (
|
||||
int(profile["semantic_loss"]["stop_after_completed_results"])
|
||||
if profile["mode"] == "semantic-loss-negative-control"
|
||||
else None
|
||||
),
|
||||
"triggered": (
|
||||
len(completed_semantics)
|
||||
== int(profile["semantic_loss"]["stop_after_completed_results"])
|
||||
if profile["mode"] == "semantic-loss-negative-control"
|
||||
else False
|
||||
),
|
||||
},
|
||||
"latency_ms": latency_summary,
|
||||
"gpu_telemetry": gpu.summary(),
|
||||
"process_peak_rss_mib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024,
|
||||
"cuda_peak_memory_allocated_mib": torch.cuda.max_memory_allocated() / 2**20,
|
||||
"cuda_peak_memory_reserved_mib": torch.cuda.max_memory_reserved() / 2**20,
|
||||
"disk": {
|
||||
"free_bytes_before": disk_before,
|
||||
"free_bytes_after": shutil.disk_usage(output).free,
|
||||
"free_bytes_floor": args.free_bytes_floor,
|
||||
},
|
||||
}
|
||||
report = {
|
||||
"schema_version": REPORT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"state": "accepted" if accepted else "rejected",
|
||||
"ground_truth": False,
|
||||
"identity": identity,
|
||||
"runtime": {
|
||||
"hostname": platform.node(),
|
||||
"python": platform.python_version(),
|
||||
"numpy": np.__version__,
|
||||
"torch": torch.__version__,
|
||||
"transformers": transformers.__version__,
|
||||
"scipy": importlib.metadata.version("scipy"),
|
||||
"gpu": torch.cuda.get_device_name(),
|
||||
},
|
||||
"metrics": metrics,
|
||||
"acceptance": {
|
||||
"accepted": accepted,
|
||||
"checks": checks,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
"limitations": [
|
||||
"Recorded source-paced replay, not direct live K1 transport.",
|
||||
"Host-arrival camera/LiDAR synchronization is not a hardware-clock proof.",
|
||||
"COCO and Cityscapes models are not forest-domain or safety validated.",
|
||||
(
|
||||
"Completed 3D cuboids mix visible LiDAR support with class-size priors; "
|
||||
"unobserved volume is inferred, not measured or ground truth."
|
||||
if "cuboid_completion" in profile
|
||||
else (
|
||||
"3D cuboids describe visible LiDAR-supported surfaces, "
|
||||
"not complete object volume."
|
||||
)
|
||||
),
|
||||
*(
|
||||
["Semantic inference was intentionally stopped for a negative-control run."]
|
||||
if profile["mode"] == "semantic-loss-negative-control"
|
||||
else []
|
||||
),
|
||||
],
|
||||
}
|
||||
report_path = output / "run-report.json"
|
||||
_write_json(report_path, report)
|
||||
artifacts = [
|
||||
_artifact(semantic_path, "e10-semantic-frames", "application/x-ndjson", SEMANTIC_SCHEMA),
|
||||
_artifact(fusion_path, "e10-fusion-frames", "application/x-ndjson", FUSION_SCHEMA),
|
||||
_artifact(world_path, "e10-world-state", "application/x-ndjson", WORLD_SCHEMA),
|
||||
_artifact(arrays_path, "e10-transient-perception", "application/x-npz"),
|
||||
_artifact(gpu_path, "worker-gpu-telemetry", "application/x-ndjson"),
|
||||
_artifact(report_path, "e10-run-report", "application/json", REPORT_SCHEMA),
|
||||
]
|
||||
result = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": report["created_at_utc"],
|
||||
"acceptance_state": report["state"],
|
||||
"ground_truth": False,
|
||||
"publication_scope": (
|
||||
"recorded-integrated-semantic-loss-negative-control-only"
|
||||
if profile["mode"] == "semantic-loss-negative-control"
|
||||
else "recorded-integrated-realtime-qualification-only"
|
||||
),
|
||||
"frames_processed": detector_state["consumed"],
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
_write_json(output / "result.json", result)
|
||||
lidar.close()
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result_id,
|
||||
"accepted": accepted,
|
||||
"detector_fps": detector_fps,
|
||||
"semantic_fps": semantic_fps,
|
||||
"world_state_age_p95_ms": latency_summary["world_state_age_ms"]["p95"],
|
||||
"fused_frames": fused_frames,
|
||||
"accepted_cuboids": accepted_cuboids,
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def assert_disk(path: Path, floor: int, frame: int) -> None:
|
||||
if floor < 0 or (floor and shutil.disk_usage(path).free < floor):
|
||||
raise RuntimeError(f"LAB E10 crossed the D-backed free-space floor at frame {frame}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = arguments()
|
||||
return preflight(args) if args.command == "preflight" else run(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,304 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Receive and qualify the Mission Core live shadow stream without K1 access."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import select
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from contextlib import suppress
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, BinaryIO
|
||||
|
||||
WIRE_SCHEMA = "missioncore.live-perception-wire/v1"
|
||||
REPORT_SCHEMA = "missioncore.e12-shadow-transport-report/v1"
|
||||
WEBSOCKET_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
|
||||
MAX_HEADER_BYTES = 64 * 1024
|
||||
MAX_FRAME_BYTES = 3 * 1024 * 1024
|
||||
|
||||
|
||||
class ProbeError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _read_exact(stream: BinaryIO, size: int) -> bytes:
|
||||
parts: list[bytes] = []
|
||||
remaining = size
|
||||
while remaining:
|
||||
chunk = stream.read(remaining)
|
||||
if not chunk:
|
||||
raise ProbeError("shadow transport ended unexpectedly")
|
||||
parts.append(chunk)
|
||||
remaining -= len(chunk)
|
||||
return b"".join(parts)
|
||||
|
||||
|
||||
def _read_http_headers(stream: BinaryIO) -> tuple[str, dict[str, str]]:
|
||||
encoded = bytearray()
|
||||
while not encoded.endswith(b"\r\n\r\n"):
|
||||
encoded.extend(_read_exact(stream, 1))
|
||||
if len(encoded) > MAX_HEADER_BYTES:
|
||||
raise ProbeError("websocket response headers exceed the bound")
|
||||
try:
|
||||
lines = encoded.decode("ascii").split("\r\n")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ProbeError("websocket response headers are not ASCII") from exc
|
||||
headers: dict[str, str] = {}
|
||||
for line in lines[1:]:
|
||||
if not line:
|
||||
continue
|
||||
name, separator, value = line.partition(":")
|
||||
if not separator:
|
||||
raise ProbeError("malformed websocket response header")
|
||||
headers[name.strip().lower()] = value.strip()
|
||||
return lines[0], headers
|
||||
|
||||
|
||||
def _send_client_frame(stream: BinaryIO, opcode: int, payload: bytes) -> None:
|
||||
if len(payload) > 125:
|
||||
raise ProbeError("client control frame exceeds websocket bound")
|
||||
mask = os.urandom(4)
|
||||
masked = bytes(value ^ mask[index % 4] for index, value in enumerate(payload))
|
||||
stream.write(bytes((0x80 | opcode, 0x80 | len(payload))) + mask + masked)
|
||||
stream.flush()
|
||||
|
||||
|
||||
def _read_server_frame(stream: BinaryIO) -> tuple[int, bytes]:
|
||||
first, second = _read_exact(stream, 2)
|
||||
if not first & 0x80:
|
||||
raise ProbeError("fragmented websocket frames are not admitted")
|
||||
opcode = first & 0x0F
|
||||
masked = bool(second & 0x80)
|
||||
length = second & 0x7F
|
||||
if length == 126:
|
||||
length = struct.unpack("!H", _read_exact(stream, 2))[0]
|
||||
elif length == 127:
|
||||
length = struct.unpack("!Q", _read_exact(stream, 8))[0]
|
||||
if length > MAX_FRAME_BYTES:
|
||||
raise ProbeError("websocket frame exceeds the shadow transport bound")
|
||||
mask = _read_exact(stream, 4) if masked else None
|
||||
payload = _read_exact(stream, length)
|
||||
if mask is not None:
|
||||
payload = bytes(value ^ mask[index % 4] for index, value in enumerate(payload))
|
||||
return opcode, payload
|
||||
|
||||
|
||||
def _connect(
|
||||
host: str,
|
||||
port: int,
|
||||
path: str,
|
||||
token: str,
|
||||
timeout_seconds: float,
|
||||
) -> tuple[socket.socket, BinaryIO]:
|
||||
connection = socket.create_connection((host, port), timeout=timeout_seconds)
|
||||
connection.settimeout(timeout_seconds)
|
||||
stream = connection.makefile("rwb", buffering=0)
|
||||
key = base64.b64encode(os.urandom(16)).decode("ascii")
|
||||
request = (
|
||||
f"GET {path} HTTP/1.1\r\n"
|
||||
f"Host: {host}:{port}\r\n"
|
||||
"Upgrade: websocket\r\n"
|
||||
"Connection: Upgrade\r\n"
|
||||
f"Sec-WebSocket-Key: {key}\r\n"
|
||||
"Sec-WebSocket-Version: 13\r\n"
|
||||
f"Authorization: Bearer {token}\r\n"
|
||||
"\r\n"
|
||||
).encode("ascii")
|
||||
stream.write(request)
|
||||
stream.flush()
|
||||
status, headers = _read_http_headers(stream)
|
||||
expected_accept = base64.b64encode(
|
||||
hashlib.sha1(f"{key}{WEBSOCKET_GUID}".encode("ascii")).digest()
|
||||
).decode("ascii")
|
||||
if not status.startswith("HTTP/1.1 101 "):
|
||||
raise ProbeError(f"websocket upgrade failed: {status}")
|
||||
if headers.get("sec-websocket-accept") != expected_accept:
|
||||
raise ProbeError("websocket accept identity mismatch")
|
||||
connection.settimeout(None)
|
||||
return connection, stream
|
||||
|
||||
|
||||
def _decode_event(frame: bytes) -> tuple[dict[str, Any], bytes]:
|
||||
if len(frame) < 4:
|
||||
raise ProbeError("shadow event is truncated")
|
||||
header_bytes = struct.unpack("!I", frame[:4])[0]
|
||||
if header_bytes < 2 or header_bytes > MAX_HEADER_BYTES:
|
||||
raise ProbeError("shadow event header length is invalid")
|
||||
boundary = 4 + header_bytes
|
||||
if boundary > len(frame):
|
||||
raise ProbeError("shadow event header is truncated")
|
||||
try:
|
||||
header = json.loads(frame[4:boundary])
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise ProbeError("shadow event header is invalid") from exc
|
||||
if not isinstance(header, dict) or header.get("schema_version") != WIRE_SCHEMA:
|
||||
raise ProbeError("shadow event schema is incompatible")
|
||||
payload = frame[boundary:]
|
||||
if int(header.get("payload_bytes", -1)) != len(payload):
|
||||
raise ProbeError("shadow event payload length mismatch")
|
||||
if hashlib.sha256(payload).hexdigest() != header.get("payload_sha256"):
|
||||
raise ProbeError("shadow event payload digest mismatch")
|
||||
if header.get("commands_enabled") is not False:
|
||||
raise ProbeError("shadow transport unexpectedly enables commands")
|
||||
if header.get("navigation_or_safety_accepted") is not False:
|
||||
raise ProbeError("shadow transport unexpectedly claims safety acceptance")
|
||||
return header, payload
|
||||
|
||||
|
||||
def _write_report(output_root: Path, report: dict[str, Any]) -> Path:
|
||||
root = output_root.expanduser().resolve()
|
||||
if os.name == "nt" and root.drive.upper() != "D:":
|
||||
raise ProbeError("Windows probe output must stay on drive D")
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
temporary = root / f".{stamp}-e12-shadow-transport.json.tmp"
|
||||
destination = root / f"{stamp}-e12-shadow-transport.json"
|
||||
encoded = json.dumps(report, indent=2, sort_keys=True).encode("utf-8") + b"\n"
|
||||
with temporary.open("xb") as stream:
|
||||
stream.write(encoded)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
temporary.replace(destination)
|
||||
return destination
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> dict[str, Any]:
|
||||
token = sys.stdin.readline().strip() if args.token_stdin else args.token
|
||||
if not token or len(token) < 40:
|
||||
raise ProbeError("shadow bearer token is missing or invalid")
|
||||
started_epoch_ns = time.time_ns()
|
||||
started_monotonic_ns = time.monotonic_ns()
|
||||
counts: Counter[str] = Counter()
|
||||
payload_bytes: Counter[str] = Counter()
|
||||
first_ingress_sequence: int | None = None
|
||||
last_ingress_sequence: int | None = None
|
||||
ingress_gaps = 0
|
||||
session_ids: set[str] = set()
|
||||
source_ids: set[str] = set()
|
||||
session_end_seen = False
|
||||
|
||||
connection, stream = _connect(
|
||||
args.host,
|
||||
args.port,
|
||||
args.path,
|
||||
token,
|
||||
args.socket_timeout_seconds,
|
||||
)
|
||||
try:
|
||||
deadline = time.monotonic() + args.max_duration_seconds
|
||||
while time.monotonic() < deadline:
|
||||
readable, _, _ = select.select(
|
||||
[connection],
|
||||
[],
|
||||
[],
|
||||
min(0.5, max(0.0, deadline - time.monotonic())),
|
||||
)
|
||||
if not readable:
|
||||
continue
|
||||
opcode, frame = _read_server_frame(stream)
|
||||
if opcode == 0x8:
|
||||
break
|
||||
if opcode == 0x9:
|
||||
_send_client_frame(stream, 0xA, frame)
|
||||
continue
|
||||
if opcode != 0x2:
|
||||
raise ProbeError(f"unexpected websocket opcode: {opcode}")
|
||||
header, payload = _decode_event(frame)
|
||||
sequence = int(header["ingress_sequence"])
|
||||
if last_ingress_sequence is not None and sequence > last_ingress_sequence + 1:
|
||||
ingress_gaps += sequence - last_ingress_sequence - 1
|
||||
if last_ingress_sequence is not None and sequence <= last_ingress_sequence:
|
||||
raise ProbeError("shadow ingress sequence is not strictly increasing")
|
||||
first_ingress_sequence = first_ingress_sequence or sequence
|
||||
last_ingress_sequence = sequence
|
||||
modality = str(header["modality"])
|
||||
counts[modality] += 1
|
||||
payload_bytes[modality] += len(payload)
|
||||
session_ids.add(str(header["session_id"]))
|
||||
source_ids.add(str(header["source_id"]))
|
||||
if modality == "control":
|
||||
try:
|
||||
control = json.loads(payload)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ProbeError("control event payload is invalid") from exc
|
||||
if control.get("event") == "session-end":
|
||||
session_end_seen = True
|
||||
break
|
||||
finally:
|
||||
with suppress(Exception):
|
||||
_send_client_frame(stream, 0x8, b"")
|
||||
stream.close()
|
||||
connection.close()
|
||||
|
||||
completed_epoch_ns = time.time_ns()
|
||||
report = {
|
||||
"schema_version": REPORT_SCHEMA,
|
||||
"state": "completed" if session_end_seen else "timed-out",
|
||||
"started_at_epoch_ns": started_epoch_ns,
|
||||
"completed_at_epoch_ns": completed_epoch_ns,
|
||||
"wall_seconds": (time.monotonic_ns() - started_monotonic_ns) / 1_000_000_000,
|
||||
"transport": {
|
||||
"kind": "ssh-reverse-tunnel-websocket",
|
||||
"endpoint": f"{args.host}:{args.port}",
|
||||
"worker_has_k1_connection": False,
|
||||
"payload_integrity": "sha256-verified-per-event",
|
||||
"clock_qualification": "not-qualified-cross-host",
|
||||
},
|
||||
"authority": {
|
||||
"mode": "shadow-diagnostic-only",
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
"events": {
|
||||
"counts": dict(sorted(counts.items())),
|
||||
"payload_bytes": dict(sorted(payload_bytes.items())),
|
||||
"first_ingress_sequence": first_ingress_sequence,
|
||||
"last_ingress_sequence": last_ingress_sequence,
|
||||
"ingress_sequence_gaps": ingress_gaps,
|
||||
"session_end_seen": session_end_seen,
|
||||
"session_ids": sorted(session_ids),
|
||||
"source_ids": sorted(source_ids),
|
||||
},
|
||||
}
|
||||
destination = _write_report(Path(args.output_root), report)
|
||||
report["report_path"] = str(destination)
|
||||
return report
|
||||
|
||||
|
||||
def _arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host", default="host.docker.internal")
|
||||
parser.add_argument("--port", type=int, default=18012)
|
||||
parser.add_argument(
|
||||
"--path",
|
||||
default=(
|
||||
"/api/v1/device-plugins/"
|
||||
"nodedc.device.xgrids-lixelkity-k1/live-perception-shadow"
|
||||
),
|
||||
)
|
||||
parser.add_argument("--token", default="")
|
||||
parser.add_argument("--token-stdin", action="store_true")
|
||||
parser.add_argument("--max-duration-seconds", type=float, default=900.0)
|
||||
parser.add_argument("--socket-timeout-seconds", type=float, default=30.0)
|
||||
parser.add_argument(
|
||||
"--output-root",
|
||||
default="D:/NDC_MISSIONCORE/runtime/derived/e12-shadow-transport",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
print(json.dumps(run(_arguments()), indent=2, sort_keys=True))
|
||||
except Exception as exc:
|
||||
print(f"E12_SHADOW_TRANSPORT_ERROR={type(exc).__name__}:{exc}", file=sys.stderr)
|
||||
raise SystemExit(1) from exc
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,768 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run and seal LAB E4 full-session EoMT semantic segmentation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import resource
|
||||
import shutil
|
||||
import time
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from run_e3_rectified_segmentation import (
|
||||
_dependency_manifest,
|
||||
_model_snapshot,
|
||||
_profile,
|
||||
_resource_delta,
|
||||
_resource_snapshot,
|
||||
_verify_model_files,
|
||||
)
|
||||
from run_recorded_perception_epoch import (
|
||||
FRAME_SCHEMA,
|
||||
REPORT_SCHEMA,
|
||||
RESULT_SCHEMA,
|
||||
_artifact,
|
||||
_canonical_json,
|
||||
_GpuTelemetry,
|
||||
_palette,
|
||||
_percentiles,
|
||||
_read_timeline,
|
||||
_sha256,
|
||||
_valid_sha256,
|
||||
_validate_job,
|
||||
_write_json,
|
||||
)
|
||||
|
||||
IDENTITY_SCHEMA = "missioncore.recorded-perception-identity/v2"
|
||||
VALID_FOV_SCHEMA = "missioncore.k1-valid-fov-mask/v1"
|
||||
PIPELINE_ID = "recorded-semantic-eomt-fisheye-mask/v1"
|
||||
SEMANTIC_ALPHA = 0.48
|
||||
TARGET_CLASS_COUNT = 16
|
||||
|
||||
|
||||
def _arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
preflight = commands.add_parser("preflight")
|
||||
preflight.add_argument("--job", type=Path, required=True)
|
||||
preflight.add_argument("--profile", type=Path, required=True)
|
||||
preflight.add_argument("--valid-fov-root", type=Path, required=True)
|
||||
preflight.add_argument("--cache", type=Path, required=True)
|
||||
preflight.add_argument("--environment", type=Path, required=True)
|
||||
|
||||
run = commands.add_parser("run")
|
||||
run.add_argument("--job", type=Path, required=True)
|
||||
run.add_argument("--profile", type=Path, required=True)
|
||||
run.add_argument("--valid-fov-root", type=Path, required=True)
|
||||
run.add_argument("--frames", type=Path, required=True)
|
||||
run.add_argument("--timeline", type=Path, required=True)
|
||||
run.add_argument("--cache", type=Path, required=True)
|
||||
run.add_argument("--environment", type=Path, required=True)
|
||||
run.add_argument("--output", type=Path, required=True)
|
||||
run.add_argument("--frame-limit", type=int, default=0)
|
||||
run.add_argument("--free-bytes-floor", type=int, default=0)
|
||||
run.add_argument("--orchestrator-sha256", required=True)
|
||||
run.add_argument("--container-image", required=True)
|
||||
run.add_argument("--telemetry-interval-seconds", type=float, default=1.0)
|
||||
|
||||
finalize = commands.add_parser("finalize")
|
||||
finalize.add_argument("--output", type=Path, required=True)
|
||||
finalize.add_argument("--video", type=Path, required=True)
|
||||
finalize.add_argument("--masks", type=Path, required=True)
|
||||
finalize.add_argument("--extract-seconds", type=float, required=True)
|
||||
finalize.add_argument("--encode-seconds", type=float, required=True)
|
||||
finalize.add_argument("--archive-seconds", type=float, required=True)
|
||||
finalize.add_argument("--wall-seconds", type=float, required=True)
|
||||
finalize.add_argument("--encoder", required=True)
|
||||
finalize.add_argument("--disk-free-before-bytes", type=int, required=True)
|
||||
finalize.add_argument("--disk-free-post-extract-bytes", type=int, required=True)
|
||||
finalize.add_argument("--disk-free-post-inference-bytes", type=int, required=True)
|
||||
finalize.add_argument("--disk-free-post-artifacts-bytes", type=int, required=True)
|
||||
finalize.add_argument("--disk-floor-bytes", type=int, required=True)
|
||||
finalize.add_argument("--working-set-reserve-bytes", type=int, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _read_object(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise RuntimeError(f"JSON root is not an object: {path}")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_source(job: dict[str, Any], profile: dict[str, Any]) -> None:
|
||||
source = profile["source"]
|
||||
input_document = job["input"]
|
||||
if (
|
||||
input_document.get("kind") != "canonical-camera-epoch"
|
||||
or input_document.get("source_id") != source["source_id"]
|
||||
or source.get("resolution") != [800, 600]
|
||||
):
|
||||
raise RuntimeError("LAB E4 profile does not match the camera job")
|
||||
|
||||
|
||||
def _load_valid_fov(
|
||||
root: Path,
|
||||
job: dict[str, Any],
|
||||
profile: dict[str, Any],
|
||||
) -> tuple[Any, dict[str, Any]]:
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
resolved = root.resolve(strict=True)
|
||||
manifest = _read_object(resolved / "manifest.json")
|
||||
identity = manifest.get("identity")
|
||||
identity_sha256 = manifest.get("identity_sha256")
|
||||
artifact = manifest.get("artifact")
|
||||
source = profile["source"]
|
||||
if (
|
||||
manifest.get("schema_version") != VALID_FOV_SCHEMA
|
||||
or not isinstance(identity, dict)
|
||||
or not _valid_sha256(identity_sha256)
|
||||
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
||||
or manifest.get("generation_id") != f"valid-fov-mask-{identity_sha256}"
|
||||
or identity.get("calibration_sha256") != source["calibration_sha256"]
|
||||
or identity.get("calibration_slot") != source["calibration_slot"]
|
||||
or identity.get("source_id") != job["input"]["source_id"]
|
||||
or identity.get("admitted_resolution") != source["resolution"]
|
||||
or not isinstance(artifact, dict)
|
||||
or artifact.get("path") != "mask.png"
|
||||
or artifact.get("mode") != "L"
|
||||
or artifact.get("inside_value") != 255
|
||||
or artifact.get("outside_value") != 0
|
||||
or not _valid_sha256(artifact.get("sha256"))
|
||||
):
|
||||
raise RuntimeError("LAB E4 valid-FOV binding is invalid")
|
||||
mask_path = (resolved / "mask.png").resolve(strict=True)
|
||||
if (
|
||||
mask_path.parent != resolved
|
||||
or mask_path.stat().st_size != artifact.get("byte_length")
|
||||
or _sha256(mask_path) != artifact["sha256"]
|
||||
):
|
||||
raise RuntimeError("LAB E4 valid-FOV artifact changed")
|
||||
with Image.open(mask_path) as opened:
|
||||
mask = np.asarray(opened, dtype=np.uint8)
|
||||
if mask.shape != (600, 800) or not np.isin(mask, (0, 255)).all():
|
||||
raise RuntimeError("LAB E4 valid-FOV mask pixels are invalid")
|
||||
valid = mask > 0
|
||||
geometry = manifest.get("geometry")
|
||||
if (
|
||||
not isinstance(geometry, dict)
|
||||
or geometry.get("valid_pixel_count") != int(valid.sum())
|
||||
or int(valid.sum()) < 1
|
||||
):
|
||||
raise RuntimeError("LAB E4 valid-FOV geometry changed")
|
||||
return valid, {
|
||||
"generation_id": manifest["generation_id"],
|
||||
"identity_sha256": identity_sha256,
|
||||
"mask_sha256": artifact["sha256"],
|
||||
"valid_pixel_count": int(valid.sum()),
|
||||
"total_pixel_count": int(valid.size),
|
||||
}
|
||||
|
||||
|
||||
def _load_model(
|
||||
profile: dict[str, Any],
|
||||
cache: Path,
|
||||
device: Any,
|
||||
) -> tuple[Any, Any, dict[int, str], Any, list[dict[str, Any]]]:
|
||||
import numpy as np
|
||||
import torch
|
||||
from transformers import AutoImageProcessor, EomtForUniversalSegmentation
|
||||
|
||||
snapshot = _model_snapshot(profile, cache, offline=True)
|
||||
model_files = _verify_model_files(profile, snapshot)
|
||||
processor = AutoImageProcessor.from_pretrained(
|
||||
snapshot,
|
||||
local_files_only=True,
|
||||
use_fast=True,
|
||||
)
|
||||
model, loading = EomtForUniversalSegmentation.from_pretrained(
|
||||
snapshot,
|
||||
local_files_only=True,
|
||||
output_loading_info=True,
|
||||
)
|
||||
problems = {
|
||||
name: loading.get(name, [])
|
||||
for name in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs")
|
||||
if loading.get(name)
|
||||
}
|
||||
if problems:
|
||||
raise RuntimeError("EoMT checkpoint did not load exactly: " + json.dumps(problems))
|
||||
model = model.to(device=device, dtype=torch.float16).eval()
|
||||
labels = {int(key): str(value) for key, value in model.config.id2label.items()}
|
||||
mapping = profile["cityscapes_to_target"]
|
||||
if set(labels.values()) != set(mapping):
|
||||
raise RuntimeError("EoMT runtime taxonomy changed")
|
||||
target_lut = np.asarray([mapping[labels[index]] for index in range(19)], dtype=np.uint8)
|
||||
return processor, model, labels, target_lut, model_files
|
||||
|
||||
|
||||
def _preflight(args: argparse.Namespace) -> int:
|
||||
import torch
|
||||
|
||||
job = _validate_job(args.job.resolve(strict=True))
|
||||
profile, profile_sha256 = _profile(args.profile)
|
||||
_validate_source(job, profile)
|
||||
dependency = _dependency_manifest(args.environment.resolve(strict=True))
|
||||
if dependency["identity"]["profile_sha256"] != profile_sha256:
|
||||
raise RuntimeError("LAB E4 dependencies belong to another profile")
|
||||
_load_valid_fov(args.valid_fov_root, job, profile)
|
||||
if not torch.cuda.is_available() or torch.cuda.device_count() < 1:
|
||||
raise RuntimeError("CUDA device 0 is unavailable")
|
||||
device = torch.device("cuda:0")
|
||||
processor, model, _labels, _target_lut, model_files = _load_model(
|
||||
profile,
|
||||
args.cache.resolve(strict=True),
|
||||
device,
|
||||
)
|
||||
del processor, model, _labels, _target_lut
|
||||
torch.cuda.empty_cache()
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"state": "preflight-ready",
|
||||
"job_id": job["job_id"],
|
||||
"source_id": job["input"]["source_id"],
|
||||
"frames": job["input"]["segment_count"],
|
||||
"pipeline": PIPELINE_ID,
|
||||
"model_files": model_files,
|
||||
"cuda_device": torch.cuda.get_device_name(),
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def _class_document(
|
||||
semantic: Any,
|
||||
valid_mask: Any,
|
||||
target_names: dict[int, str],
|
||||
) -> list[dict[str, Any]]:
|
||||
import numpy as np
|
||||
|
||||
counts = np.bincount(semantic[valid_mask].reshape(-1), minlength=TARGET_CLASS_COUNT)
|
||||
total = int(valid_mask.sum())
|
||||
return [
|
||||
{
|
||||
"id": index,
|
||||
"label": target_names[index],
|
||||
"pixels": int(counts[index]),
|
||||
"fraction_of_valid_fov": round(float(counts[index]) / total, 9),
|
||||
}
|
||||
for index in range(1, TARGET_CLASS_COUNT)
|
||||
if counts[index] > 0
|
||||
]
|
||||
|
||||
|
||||
def _overlay(image: Any, semantic: Any) -> Any:
|
||||
import numpy as np
|
||||
|
||||
colors = np.zeros_like(image)
|
||||
for category_id in range(1, TARGET_CLASS_COUNT):
|
||||
colors[semantic == category_id] = _palette(category_id)
|
||||
blended = np.rint(
|
||||
image.astype(np.float32) * (1.0 - SEMANTIC_ALPHA)
|
||||
+ colors.astype(np.float32) * SEMANTIC_ALPHA
|
||||
)
|
||||
return (
|
||||
np.where(
|
||||
(semantic > 0)[..., None],
|
||||
blended,
|
||||
image,
|
||||
)
|
||||
.clip(0, 255)
|
||||
.astype(np.uint8)
|
||||
)
|
||||
|
||||
|
||||
def _write_png(path: Path, array: Any) -> None:
|
||||
from PIL import Image
|
||||
|
||||
Image.fromarray(array).save(path, format="PNG", optimize=False)
|
||||
|
||||
|
||||
def _assert_disk_floor(path: Path, floor: int, frame: int) -> None:
|
||||
if floor < 0:
|
||||
raise RuntimeError("disk free-space floor is invalid")
|
||||
free = shutil.disk_usage(path).free
|
||||
if floor and free < floor:
|
||||
raise RuntimeError(f"D-backed output crossed its free-space floor at frame {frame}")
|
||||
|
||||
|
||||
def _run(args: argparse.Namespace) -> int:
|
||||
import numpy as np
|
||||
import torch
|
||||
import transformers
|
||||
from PIL import Image
|
||||
|
||||
job_root = args.job.resolve(strict=True)
|
||||
if not _valid_sha256(args.orchestrator_sha256):
|
||||
raise RuntimeError("LAB E4 orchestrator SHA-256 is invalid")
|
||||
if not 1 <= len(args.container_image) <= 256:
|
||||
raise RuntimeError("LAB E4 container image identity is invalid")
|
||||
job = _validate_job(job_root)
|
||||
profile, profile_sha256 = _profile(args.profile)
|
||||
_validate_source(job, profile)
|
||||
dependency = _dependency_manifest(args.environment.resolve(strict=True))
|
||||
if dependency["identity"]["profile_sha256"] != profile_sha256:
|
||||
raise RuntimeError("LAB E4 dependencies belong to another profile")
|
||||
valid_mask, valid_fov = _load_valid_fov(args.valid_fov_root, job, profile)
|
||||
|
||||
input_document = job["input"]
|
||||
full_frame_count = int(input_document["segment_count"])
|
||||
if args.frame_limit < 0 or args.frame_limit > full_frame_count:
|
||||
raise RuntimeError("LAB E4 frame limit escapes the camera job")
|
||||
frame_count = args.frame_limit or full_frame_count
|
||||
frames_root = args.frames.resolve(strict=True)
|
||||
frame_paths = [frames_root / f"frame-{index:06d}.png" for index in range(1, frame_count + 1)]
|
||||
if not all(path.is_file() for path in frame_paths):
|
||||
raise RuntimeError("decoded LAB E4 frame set is incomplete")
|
||||
if len(list(frames_root.glob("frame-*.png"))) != frame_count:
|
||||
raise RuntimeError("decoded LAB E4 frame set contains unexpected files")
|
||||
timeline = input_document["timeline"]
|
||||
timestamps = _read_timeline(
|
||||
args.timeline.resolve(strict=True),
|
||||
frame_count,
|
||||
float(timeline["start_seconds"]),
|
||||
float(timeline["end_seconds"]),
|
||||
)
|
||||
|
||||
output = args.output.resolve()
|
||||
if output.exists():
|
||||
raise RuntimeError("LAB E4 output must be absent")
|
||||
output.mkdir(mode=0o700, parents=True, exist_ok=False)
|
||||
masks_root = output / "semantic-masks"
|
||||
overlays_root = output / "overlay-frames"
|
||||
masks_root.mkdir(mode=0o700)
|
||||
overlays_root.mkdir(mode=0o700)
|
||||
_assert_disk_floor(output, args.free_bytes_floor, 0)
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("CUDA is required for LAB E4")
|
||||
device = torch.device("cuda:0")
|
||||
processor, model, model_labels, target_lut, model_files = _load_model(
|
||||
profile,
|
||||
args.cache.resolve(strict=True),
|
||||
device,
|
||||
)
|
||||
target_names = {int(key): str(value) for key, value in profile["target_taxonomy"].items()}
|
||||
if set(target_names) != set(range(TARGET_CLASS_COUNT)):
|
||||
raise RuntimeError("LAB E4 target taxonomy changed")
|
||||
|
||||
def infer(image: Any) -> tuple[Any, dict[str, float]]:
|
||||
processor_started = time.perf_counter()
|
||||
inputs = processor(images=Image.fromarray(image), return_tensors="pt")
|
||||
processor_ms = (time.perf_counter() - processor_started) * 1000.0
|
||||
torch.cuda.synchronize()
|
||||
transfer_started = time.perf_counter()
|
||||
inputs = {
|
||||
name: value.to(device) if isinstance(value, torch.Tensor) else value
|
||||
for name, value in inputs.items()
|
||||
}
|
||||
torch.cuda.synchronize()
|
||||
transfer_ms = (time.perf_counter() - transfer_started) * 1000.0
|
||||
forward_started = time.perf_counter()
|
||||
with torch.inference_mode(), torch.amp.autocast("cuda", dtype=torch.float16):
|
||||
outputs = model(**inputs)
|
||||
torch.cuda.synchronize()
|
||||
forward_ms = (time.perf_counter() - forward_started) * 1000.0
|
||||
post_started = time.perf_counter()
|
||||
semantic = processor.post_process_semantic_segmentation(
|
||||
outputs,
|
||||
target_sizes=[image.shape[:2]],
|
||||
)[0]
|
||||
semantic = semantic.detach().cpu().numpy().astype(np.uint8)
|
||||
post_ms = (time.perf_counter() - post_started) * 1000.0
|
||||
del inputs, outputs
|
||||
return semantic, {
|
||||
"processor_ms": processor_ms,
|
||||
"host_to_device_ms": transfer_ms,
|
||||
"forward_ms": forward_ms,
|
||||
"model_postprocess_ms": post_ms,
|
||||
}
|
||||
|
||||
with Image.open(frame_paths[0]) as opened:
|
||||
warm_image = np.asarray(opened.convert("RGB"), dtype=np.uint8)
|
||||
if warm_image.shape != (600, 800, 3):
|
||||
raise RuntimeError("LAB E4 input resolution changed")
|
||||
warm_image = np.where(valid_mask[..., None], warm_image, 0).astype(np.uint8)
|
||||
warm_semantic, _warm_latency = infer(warm_image)
|
||||
del warm_semantic, _warm_latency
|
||||
|
||||
latency = {
|
||||
name: []
|
||||
for name in (
|
||||
"image_decode_ms",
|
||||
"valid_fov_fill_ms",
|
||||
"processor_ms",
|
||||
"host_to_device_ms",
|
||||
"forward_ms",
|
||||
"model_postprocess_ms",
|
||||
"overlay_ms",
|
||||
"artifact_write_ms",
|
||||
"end_to_end_ms",
|
||||
)
|
||||
}
|
||||
total_pixels = Counter()
|
||||
metadata_path = output / "frames.jsonl"
|
||||
telemetry_path = output / "gpu-telemetry.jsonl"
|
||||
resource_before = _resource_snapshot()
|
||||
disk_before = shutil.disk_usage(output).free
|
||||
started = time.perf_counter()
|
||||
|
||||
with (
|
||||
telemetry_path.open("x", encoding="utf-8", newline="\n") as telemetry_stream,
|
||||
metadata_path.open("x", encoding="utf-8", newline="\n") as metadata_stream,
|
||||
_GpuTelemetry(telemetry_stream, args.telemetry_interval_seconds) as telemetry,
|
||||
):
|
||||
for frame_index, (path, session_seconds) in enumerate(
|
||||
zip(frame_paths, timestamps, strict=True)
|
||||
):
|
||||
frame_started = time.perf_counter()
|
||||
decode_started = time.perf_counter()
|
||||
with Image.open(path) as opened:
|
||||
image = np.asarray(opened.convert("RGB"), dtype=np.uint8)
|
||||
latency["image_decode_ms"].append((time.perf_counter() - decode_started) * 1000.0)
|
||||
if image.shape != (600, 800, 3):
|
||||
raise RuntimeError("LAB E4 input resolution changed")
|
||||
|
||||
fill_started = time.perf_counter()
|
||||
model_input = np.where(valid_mask[..., None], image, 0).astype(np.uint8)
|
||||
latency["valid_fov_fill_ms"].append((time.perf_counter() - fill_started) * 1000.0)
|
||||
semantic, measured = infer(model_input)
|
||||
if semantic.max() >= len(target_lut):
|
||||
raise RuntimeError("EoMT emitted an unknown category")
|
||||
target = target_lut[semantic]
|
||||
target = target.copy()
|
||||
target[~valid_mask] = 0
|
||||
if target.shape != valid_mask.shape or np.any(target[~valid_mask] != 0):
|
||||
raise RuntimeError("LAB E4 semantic mask escaped valid FOV")
|
||||
for name, value in measured.items():
|
||||
latency[name].append(value)
|
||||
|
||||
overlay_started = time.perf_counter()
|
||||
overlay = _overlay(image, target)
|
||||
classes = _class_document(target, valid_mask, target_names)
|
||||
latency["overlay_ms"].append((time.perf_counter() - overlay_started) * 1000.0)
|
||||
counts = np.bincount(target[valid_mask], minlength=TARGET_CLASS_COUNT)
|
||||
for category_id in range(1, TARGET_CLASS_COUNT):
|
||||
total_pixels[target_names[category_id]] += int(counts[category_id])
|
||||
|
||||
write_started = time.perf_counter()
|
||||
_write_png(masks_root / f"frame-{frame_index + 1:06d}.png", target)
|
||||
_write_png(overlays_root / f"frame-{frame_index + 1:06d}.png", overlay)
|
||||
metadata_stream.write(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": FRAME_SCHEMA,
|
||||
"frame_index": frame_index,
|
||||
"sequence": frame_index + 1,
|
||||
"session_seconds": round(session_seconds, 9),
|
||||
"instances": [],
|
||||
"semantic_classes": classes,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
latency["artifact_write_ms"].append((time.perf_counter() - write_started) * 1000.0)
|
||||
latency["end_to_end_ms"].append((time.perf_counter() - frame_started) * 1000.0)
|
||||
|
||||
completed = frame_index + 1
|
||||
if completed % 100 == 0 or completed == frame_count:
|
||||
metadata_stream.flush()
|
||||
telemetry_stream.flush()
|
||||
_assert_disk_floor(output, args.free_bytes_floor, completed)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"phase": "semantic",
|
||||
"frames_processed": completed,
|
||||
"frames_total": frame_count,
|
||||
"free_bytes": shutil.disk_usage(output).free,
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
metadata_stream.flush()
|
||||
os.fsync(metadata_stream.fileno())
|
||||
|
||||
inference_elapsed = time.perf_counter() - started
|
||||
resource_after = _resource_snapshot()
|
||||
disk_after = shutil.disk_usage(output).free
|
||||
report = {
|
||||
"schema_version": REPORT_SCHEMA,
|
||||
"run_id": f"{job['job_id']}-e4-eomt-full-session-v1",
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"state": (
|
||||
"inference-complete-awaiting-publication"
|
||||
if frame_count == full_frame_count
|
||||
else "technical-pilot-complete-not-publishable"
|
||||
),
|
||||
"ground_truth": False,
|
||||
"input": {
|
||||
"job_id": job["job_id"],
|
||||
"input_sha256": job["input_sha256"],
|
||||
"session_id": input_document["session_id"],
|
||||
"source_id": input_document["source_id"],
|
||||
"codec_epoch": input_document["codec_epoch"],
|
||||
"segment_count": full_frame_count,
|
||||
"frames_admitted": frame_count,
|
||||
"byte_length": input_document["byte_length"],
|
||||
"timeline_start_seconds": timeline["start_seconds"],
|
||||
"timeline_end_seconds": timeline["end_seconds"],
|
||||
"frame_timeline_sha256": _sha256(args.timeline.resolve(strict=True)),
|
||||
},
|
||||
"calibration": {
|
||||
"content_identity_sha256": profile["source"]["calibration_sha256"],
|
||||
"camera_slot": profile["source"]["calibration_slot"],
|
||||
},
|
||||
"configuration": {
|
||||
"pipeline": PIPELINE_ID,
|
||||
"semantic_alpha": SEMANTIC_ALPHA,
|
||||
"batch_size": 1,
|
||||
"precision": "fp16-autocast",
|
||||
"frame_policy": (
|
||||
"all-frames-no-sampling"
|
||||
if frame_count == full_frame_count
|
||||
else f"technical-pilot-first-{frame_count}-frames"
|
||||
),
|
||||
"instance_branch": "disabled",
|
||||
"profile_sha256": profile_sha256,
|
||||
"dependency_identity_sha256": dependency["identity_sha256"],
|
||||
"valid_fov": valid_fov,
|
||||
"runner_sha256": _sha256(Path(__file__).resolve(strict=True)),
|
||||
"orchestrator_sha256": args.orchestrator_sha256,
|
||||
"container_image": args.container_image,
|
||||
},
|
||||
"models": {
|
||||
"semantic": {
|
||||
"id": profile["model"]["id"],
|
||||
"revision": profile["model"]["revision"],
|
||||
"architecture": profile["model"]["architecture"],
|
||||
"checkpoint_load": "exact-no-missing-unexpected-or-mismatched-keys",
|
||||
"runtime_labels": {
|
||||
str(index): label for index, label in sorted(model_labels.items())
|
||||
},
|
||||
},
|
||||
"files": model_files,
|
||||
},
|
||||
"runtime": {
|
||||
"hostname": platform.node(),
|
||||
"platform": platform.platform(),
|
||||
"python": platform.python_version(),
|
||||
"torch": torch.__version__,
|
||||
"transformers": transformers.__version__,
|
||||
"cuda_runtime": torch.version.cuda,
|
||||
"gpu": torch.cuda.get_device_name(),
|
||||
"gpu_compute_capability": list(torch.cuda.get_device_capability()),
|
||||
},
|
||||
"metrics": {
|
||||
"frames_expected": frame_count,
|
||||
"frames_processed": frame_count,
|
||||
"frames_failed": 0,
|
||||
"frames_skipped": 0,
|
||||
"instances": 0,
|
||||
"inference_wall_seconds": round(inference_elapsed, 6),
|
||||
"inference_frames_per_second": round(frame_count / inference_elapsed, 6),
|
||||
"latency_ms": {name: _percentiles(values) for name, values in latency.items()},
|
||||
"semantic_pixel_totals_inside_valid_fov": dict(sorted(total_pixels.items())),
|
||||
"cuda_peak_memory_allocated_mib": round(
|
||||
torch.cuda.max_memory_allocated() / 2**20,
|
||||
3,
|
||||
),
|
||||
"cuda_peak_memory_reserved_mib": round(
|
||||
torch.cuda.max_memory_reserved() / 2**20,
|
||||
3,
|
||||
),
|
||||
"process_peak_rss_mib": round(
|
||||
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024,
|
||||
3,
|
||||
),
|
||||
"resource_delta": _resource_delta(resource_before, resource_after),
|
||||
"gpu_telemetry": telemetry.summary(),
|
||||
"disk": {
|
||||
"free_bytes_before_inference": disk_before,
|
||||
"free_bytes_after_inference": disk_after,
|
||||
"free_bytes_floor": args.free_bytes_floor,
|
||||
},
|
||||
},
|
||||
"versions": {
|
||||
name: importlib.metadata.version(name)
|
||||
for name in ("numpy", "pillow", "torch", "transformers")
|
||||
},
|
||||
}
|
||||
_write_json(output / "run-report.partial.json", report)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"state": report["state"],
|
||||
"frames": frame_count,
|
||||
"inference_seconds": round(inference_elapsed, 6),
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def _finalize(args: argparse.Namespace) -> int:
|
||||
output = args.output.resolve(strict=True)
|
||||
video = args.video.resolve(strict=True)
|
||||
masks = args.masks.resolve(strict=True)
|
||||
if video.parent != output or masks.parent != output:
|
||||
raise RuntimeError("published LAB E4 artifacts must be direct result children")
|
||||
partial_path = output / "run-report.partial.json"
|
||||
frames_path = output / "frames.jsonl"
|
||||
telemetry_path = output / "gpu-telemetry.jsonl"
|
||||
partial = _read_object(partial_path)
|
||||
input_document = partial.get("input")
|
||||
metrics = partial.get("metrics")
|
||||
if (
|
||||
partial.get("schema_version") != REPORT_SCHEMA
|
||||
or partial.get("state") != "inference-complete-awaiting-publication"
|
||||
or not isinstance(input_document, dict)
|
||||
or not isinstance(metrics, dict)
|
||||
or input_document.get("frames_admitted") != input_document.get("segment_count")
|
||||
or metrics.get("frames_expected") != input_document.get("segment_count")
|
||||
or metrics.get("frames_processed") != input_document.get("segment_count")
|
||||
):
|
||||
raise RuntimeError("only a complete LAB E4 camera epoch can be finalized")
|
||||
for value in (
|
||||
args.extract_seconds,
|
||||
args.encode_seconds,
|
||||
args.archive_seconds,
|
||||
args.wall_seconds,
|
||||
):
|
||||
if value <= 0:
|
||||
raise RuntimeError("LAB E4 publication timing is invalid")
|
||||
disk_values = (
|
||||
args.disk_free_before_bytes,
|
||||
args.disk_free_post_extract_bytes,
|
||||
args.disk_free_post_inference_bytes,
|
||||
args.disk_free_post_artifacts_bytes,
|
||||
args.disk_floor_bytes,
|
||||
args.working_set_reserve_bytes,
|
||||
)
|
||||
if any(value < 0 for value in disk_values):
|
||||
raise RuntimeError("LAB E4 publication disk telemetry is invalid")
|
||||
if any(
|
||||
value < args.disk_floor_bytes
|
||||
for value in (
|
||||
args.disk_free_before_bytes,
|
||||
args.disk_free_post_extract_bytes,
|
||||
args.disk_free_post_inference_bytes,
|
||||
args.disk_free_post_artifacts_bytes,
|
||||
)
|
||||
):
|
||||
raise RuntimeError("LAB E4 publication crossed its D: free-space floor")
|
||||
|
||||
identity = {
|
||||
"schema_version": IDENTITY_SCHEMA,
|
||||
"job_id": input_document["job_id"],
|
||||
"input_sha256": input_document["input_sha256"],
|
||||
"calibration": partial["calibration"],
|
||||
"configuration": partial["configuration"],
|
||||
"models": partial["models"],
|
||||
"publication": {
|
||||
"video_encoder": args.encoder,
|
||||
"video_media_type": "video/mp4",
|
||||
"mask_archive_media_type": "application/gzip",
|
||||
"content": "semantic-overlay-and-semantic-masks",
|
||||
},
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
result_id = f"result-{identity_sha256}"
|
||||
final_report = {
|
||||
**partial,
|
||||
"state": "published",
|
||||
"result_id": result_id,
|
||||
"publication": {
|
||||
"extract_seconds": round(args.extract_seconds, 6),
|
||||
"encode_seconds": round(args.encode_seconds, 6),
|
||||
"archive_seconds": round(args.archive_seconds, 6),
|
||||
"wall_seconds": round(args.wall_seconds, 6),
|
||||
"end_to_end_frames_per_second": round(
|
||||
int(metrics["frames_processed"]) / args.wall_seconds,
|
||||
6,
|
||||
),
|
||||
"encoder": args.encoder,
|
||||
"disk": {
|
||||
"free_bytes_before": args.disk_free_before_bytes,
|
||||
"free_bytes_post_extract": args.disk_free_post_extract_bytes,
|
||||
"free_bytes_post_inference": args.disk_free_post_inference_bytes,
|
||||
"free_bytes_post_artifacts": args.disk_free_post_artifacts_bytes,
|
||||
"free_bytes_floor": args.disk_floor_bytes,
|
||||
"working_set_reserve_bytes": args.working_set_reserve_bytes,
|
||||
},
|
||||
},
|
||||
}
|
||||
report_path = output / "run-report.json"
|
||||
_write_json(report_path, final_report)
|
||||
artifacts = [
|
||||
_artifact(video, "panoptic-overlay-video", "video/mp4"),
|
||||
_artifact(masks, "panoptic-mask-archive", "application/gzip"),
|
||||
_artifact(frames_path, "panoptic-frame-metadata", "application/x-ndjson", FRAME_SCHEMA),
|
||||
_artifact(telemetry_path, "worker-gpu-telemetry", "application/x-ndjson"),
|
||||
_artifact(report_path, "perception-run-report", "application/json", REPORT_SCHEMA),
|
||||
]
|
||||
result = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": final_report["created_at_utc"],
|
||||
"job_id": input_document["job_id"],
|
||||
"input_sha256": input_document["input_sha256"],
|
||||
"session_id": input_document["session_id"],
|
||||
"source_id": input_document["source_id"],
|
||||
"codec_epoch": input_document["codec_epoch"],
|
||||
"timestamp_basis": "session-time-seconds",
|
||||
"timeline_start_seconds": input_document["timeline_start_seconds"],
|
||||
"timeline_end_seconds": input_document["timeline_end_seconds"],
|
||||
"frames_processed": metrics["frames_processed"],
|
||||
"ground_truth": False,
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
_write_json(output / "result.json", result)
|
||||
partial_path.unlink()
|
||||
print(
|
||||
json.dumps(
|
||||
{"result_id": result_id, "identity_sha256": identity_sha256},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _arguments()
|
||||
if args.command == "preflight":
|
||||
return _preflight(args)
|
||||
if args.command == "run":
|
||||
return _run(args)
|
||||
return _finalize(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,687 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run source-paced LAB E8 YOLOX plus tracking without synchronous overlays."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import resource
|
||||
import shutil
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from collections import Counter, deque
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from run_e5_instance_tracking import (
|
||||
TwoStageTracker,
|
||||
_detections,
|
||||
_duplicate_pairs,
|
||||
_infer,
|
||||
_load_valid_fov,
|
||||
_preprocess,
|
||||
_read_timeline,
|
||||
_track_document,
|
||||
_validate_source,
|
||||
_verify_model,
|
||||
)
|
||||
from run_e5_instance_tracking import (
|
||||
_read_profile as _read_e5_profile,
|
||||
)
|
||||
from run_recorded_perception_epoch import (
|
||||
_artifact,
|
||||
_canonical_json,
|
||||
_GpuTelemetry,
|
||||
_percentiles,
|
||||
_sha256,
|
||||
_valid_sha256,
|
||||
_validate_job,
|
||||
_write_json,
|
||||
)
|
||||
|
||||
PROFILE_SCHEMA = "missioncore.e8-realtime-tracking-profile/v1"
|
||||
FRAME_SCHEMA = "missioncore.e8-realtime-tracking-frame/v1"
|
||||
TELEMETRY_SCHEMA = "missioncore.e8-realtime-tracking-telemetry/v1"
|
||||
REPORT_SCHEMA = "missioncore.e8-realtime-tracking-report/v1"
|
||||
RESULT_SCHEMA = "missioncore.e8-realtime-tracking-result/v1"
|
||||
IDENTITY_SCHEMA = "missioncore.e8-realtime-tracking-identity/v1"
|
||||
PIPELINE_ID = "source-paced-yolox-bytetrack-latest-wins/v1"
|
||||
|
||||
|
||||
def _arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
for name in ("preflight", "run"):
|
||||
command = commands.add_parser(name)
|
||||
command.add_argument("--job", type=Path, required=True)
|
||||
command.add_argument("--profile", type=Path, required=True)
|
||||
command.add_argument("--valid-fov-root", type=Path, required=True)
|
||||
command.add_argument("--model-root", type=Path, required=True)
|
||||
if name == "run":
|
||||
command.add_argument("--frames", type=Path, required=True)
|
||||
command.add_argument("--timeline", type=Path, required=True)
|
||||
command.add_argument("--output", type=Path, required=True)
|
||||
command.add_argument("--triton-url", required=True)
|
||||
command.add_argument("--free-bytes-floor", type=int, default=0)
|
||||
command.add_argument("--orchestrator-sha256", required=True)
|
||||
command.add_argument("--container-image", required=True)
|
||||
command.add_argument("--telemetry-interval-seconds", type=float, default=1.0)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _read_object(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise RuntimeError(f"JSON root is not an object: {path}")
|
||||
return value
|
||||
|
||||
|
||||
def _read_profile(path: Path) -> tuple[dict[str, Any], str]:
|
||||
resolved = path.resolve(strict=True)
|
||||
profile = _read_object(resolved)
|
||||
if profile.get("schema_version") != PROFILE_SCHEMA:
|
||||
raise RuntimeError("LAB E8 profile schema changed")
|
||||
|
||||
# Reuse the exhaustively tested E5 detector/tracker validation by changing
|
||||
# only the schema in memory. No E5 defaults are inferred.
|
||||
e5_shape = dict(profile)
|
||||
e5_shape["schema_version"] = "missioncore.e5-tracking-profile/v1"
|
||||
temporary: Path | None = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
encoding="utf-8",
|
||||
prefix="lab-e8-e5-validation-",
|
||||
suffix=".json",
|
||||
delete=False,
|
||||
) as stream:
|
||||
json.dump(e5_shape, stream, sort_keys=True, separators=(",", ":"))
|
||||
temporary = Path(stream.name)
|
||||
_read_e5_profile(temporary)
|
||||
finally:
|
||||
if temporary is not None:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
realtime = profile.get("realtime")
|
||||
acceptance = profile.get("acceptance")
|
||||
if not isinstance(realtime, dict) or not isinstance(acceptance, dict):
|
||||
raise RuntimeError("LAB E8 realtime/acceptance configuration is missing")
|
||||
mode = profile.get("mode")
|
||||
capacity = realtime.get("queue_capacity")
|
||||
speed = realtime.get("speed")
|
||||
delay = realtime.get("consumer_delay_ms")
|
||||
stale = realtime.get("stale_after_ms")
|
||||
unavailable = realtime.get("unavailable_after_ms")
|
||||
if (
|
||||
mode not in {"pilot", "qualification", "overload-negative-control"}
|
||||
or realtime.get("queue_policy") != "bounded-latest-wins"
|
||||
or not isinstance(capacity, int)
|
||||
or isinstance(capacity, bool)
|
||||
or not 1 <= capacity <= 8
|
||||
or not isinstance(speed, int | float)
|
||||
or isinstance(speed, bool)
|
||||
or not 0.1 <= float(speed) <= 10.0
|
||||
or not isinstance(delay, int | float)
|
||||
or isinstance(delay, bool)
|
||||
or not 0 <= float(delay) <= 5000
|
||||
or not isinstance(stale, int | float)
|
||||
or not isinstance(unavailable, int | float)
|
||||
or not 0 < float(stale) < float(unavailable)
|
||||
):
|
||||
raise RuntimeError("LAB E8 scheduling configuration is invalid")
|
||||
numeric_acceptance = (
|
||||
acceptance.get("minimum_effective_fps"),
|
||||
acceptance.get("maximum_drop_fraction"),
|
||||
acceptance.get("maximum_p95_result_age_ms"),
|
||||
)
|
||||
if (
|
||||
any(
|
||||
not isinstance(value, int | float)
|
||||
or isinstance(value, bool)
|
||||
or not math.isfinite(float(value))
|
||||
or float(value) < 0
|
||||
for value in numeric_acceptance
|
||||
)
|
||||
or float(acceptance["maximum_drop_fraction"]) > 1
|
||||
or not isinstance(acceptance.get("require_zero_failures"), bool)
|
||||
or not isinstance(acceptance.get("expect_overload"), bool)
|
||||
):
|
||||
raise RuntimeError("LAB E8 acceptance configuration is invalid")
|
||||
return profile, _sha256(resolved)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FrameEnvelope:
|
||||
frame_index: int
|
||||
path: Path
|
||||
timeline: dict[str, Any]
|
||||
scheduled_monotonic: float
|
||||
decoded_monotonic: float
|
||||
image: np.ndarray
|
||||
decode_ms: float
|
||||
source_release_lag_ms: float
|
||||
|
||||
|
||||
class LatestWinsQueue:
|
||||
def __init__(self, capacity: int) -> None:
|
||||
self.capacity = capacity
|
||||
self.items: deque[FrameEnvelope] = deque()
|
||||
self.condition = threading.Condition()
|
||||
self.maximum_depth = 0
|
||||
self.published = 0
|
||||
self.consumed = 0
|
||||
self.dropped_overflow = 0
|
||||
self.closed = False
|
||||
|
||||
def publish(self, item: FrameEnvelope) -> None:
|
||||
with self.condition:
|
||||
if self.closed:
|
||||
raise RuntimeError("cannot publish to a closed LAB E8 queue")
|
||||
if len(self.items) == self.capacity:
|
||||
self.items.popleft()
|
||||
self.dropped_overflow += 1
|
||||
self.items.append(item)
|
||||
self.published += 1
|
||||
self.maximum_depth = max(self.maximum_depth, len(self.items))
|
||||
self.condition.notify()
|
||||
|
||||
def take(self) -> FrameEnvelope | None:
|
||||
with self.condition:
|
||||
self.condition.wait_for(lambda: bool(self.items) or self.closed)
|
||||
if not self.items:
|
||||
return None
|
||||
value = self.items.popleft()
|
||||
self.consumed += 1
|
||||
return value
|
||||
|
||||
def close(self) -> None:
|
||||
with self.condition:
|
||||
self.closed = True
|
||||
self.condition.notify_all()
|
||||
|
||||
def snapshot(self) -> dict[str, int | bool]:
|
||||
with self.condition:
|
||||
return {
|
||||
"capacity": self.capacity,
|
||||
"final_depth": len(self.items),
|
||||
"maximum_depth": self.maximum_depth,
|
||||
"published": self.published,
|
||||
"consumed": self.consumed,
|
||||
"dropped_overflow": self.dropped_overflow,
|
||||
"closed": self.closed,
|
||||
}
|
||||
|
||||
|
||||
def _wait_until(deadline: float) -> None:
|
||||
while True:
|
||||
remaining = deadline - time.perf_counter()
|
||||
if remaining <= 0:
|
||||
return
|
||||
time.sleep(min(remaining, 0.02))
|
||||
|
||||
|
||||
def _health(age_ms: float, realtime: dict[str, Any]) -> str:
|
||||
if age_ms >= float(realtime["unavailable_after_ms"]):
|
||||
return "unavailable"
|
||||
if age_ms >= float(realtime["stale_after_ms"]):
|
||||
return "stale"
|
||||
return "healthy"
|
||||
|
||||
|
||||
def _producer(
|
||||
*,
|
||||
queue: LatestWinsQueue,
|
||||
frame_paths: list[Path],
|
||||
timeline_rows: list[dict[str, Any]],
|
||||
replay_started: float,
|
||||
speed: float,
|
||||
error: list[BaseException],
|
||||
) -> None:
|
||||
from PIL import Image
|
||||
|
||||
first_session = float(timeline_rows[0]["session_seconds"])
|
||||
try:
|
||||
for frame_index, (path, row) in enumerate(zip(frame_paths, timeline_rows, strict=True)):
|
||||
scheduled = replay_started + (float(row["session_seconds"]) - first_session) / speed
|
||||
_wait_until(scheduled)
|
||||
decode_started = time.perf_counter()
|
||||
with Image.open(path) as opened:
|
||||
image = np.asarray(opened.convert("RGB"), dtype=np.uint8)
|
||||
decoded = time.perf_counter()
|
||||
if image.shape != (600, 800, 3):
|
||||
raise RuntimeError("LAB E8 input resolution changed")
|
||||
queue.publish(
|
||||
FrameEnvelope(
|
||||
frame_index=frame_index,
|
||||
path=path,
|
||||
timeline=row,
|
||||
scheduled_monotonic=scheduled,
|
||||
decoded_monotonic=decoded,
|
||||
image=image,
|
||||
decode_ms=(decoded - decode_started) * 1000.0,
|
||||
source_release_lag_ms=max(0.0, (decode_started - scheduled) * 1000.0),
|
||||
)
|
||||
)
|
||||
except BaseException as exc:
|
||||
error.append(exc)
|
||||
finally:
|
||||
queue.close()
|
||||
|
||||
|
||||
def _assert_disk_floor(path: Path, floor: int, frame: int) -> None:
|
||||
if floor < 0:
|
||||
raise RuntimeError("LAB E8 disk floor is invalid")
|
||||
free = shutil.disk_usage(path).free
|
||||
if floor and free < floor:
|
||||
raise RuntimeError(f"LAB E8 crossed the D-backed free-space floor at frame {frame}")
|
||||
|
||||
|
||||
def _preflight(args: argparse.Namespace) -> int:
|
||||
job = _validate_job(args.job.resolve(strict=True))
|
||||
profile, profile_sha256 = _read_profile(args.profile)
|
||||
_validate_source(job, profile)
|
||||
valid_mask, valid_fov = _load_valid_fov(args.valid_fov_root, job, profile)
|
||||
model_files = _verify_model(profile, args.model_root)
|
||||
from scipy.optimize import linear_sum_assignment # noqa: F401
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"state": "preflight-ready",
|
||||
"job_id": job["job_id"],
|
||||
"profile_sha256": profile_sha256,
|
||||
"model_files": model_files,
|
||||
"valid_fov": valid_fov,
|
||||
"valid_pixels": int(valid_mask.sum()),
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def _run(args: argparse.Namespace) -> int:
|
||||
from PIL import Image
|
||||
from scipy.optimize import linear_sum_assignment
|
||||
|
||||
if not _valid_sha256(args.orchestrator_sha256):
|
||||
raise RuntimeError("LAB E8 orchestrator SHA-256 is invalid")
|
||||
if not args.triton_url.startswith("http://") or len(args.triton_url) > 256:
|
||||
raise RuntimeError("LAB E8 Triton URL is invalid")
|
||||
job = _validate_job(args.job.resolve(strict=True))
|
||||
profile, profile_sha256 = _read_profile(args.profile)
|
||||
_validate_source(job, profile)
|
||||
valid_mask, valid_fov = _load_valid_fov(args.valid_fov_root, job, profile)
|
||||
model_files = _verify_model(profile, args.model_root)
|
||||
frames_root = args.frames.resolve(strict=True)
|
||||
frame_paths = sorted(frames_root.glob("frame-*.png"))
|
||||
if not frame_paths or [path.name for path in frame_paths] != [
|
||||
f"frame-{index:06d}.png" for index in range(1, len(frame_paths) + 1)
|
||||
]:
|
||||
raise RuntimeError("LAB E8 decoded frame sequence changed")
|
||||
timeline_path = args.timeline.resolve(strict=True)
|
||||
timeline_rows = _read_timeline(timeline_path, len(frame_paths))
|
||||
input_timeline = job["input"]["timeline"]
|
||||
if float(timeline_rows[0]["session_seconds"]) < float(input_timeline["start_seconds"]) or float(
|
||||
timeline_rows[-1]["session_seconds"]
|
||||
) > float(input_timeline["end_seconds"]):
|
||||
raise RuntimeError("LAB E8 clip escaped the source timeline")
|
||||
|
||||
output = args.output.resolve()
|
||||
if output.exists():
|
||||
raise RuntimeError("LAB E8 output must be absent")
|
||||
output.mkdir(mode=0o700, parents=True, exist_ok=False)
|
||||
_assert_disk_floor(output, args.free_bytes_floor, 0)
|
||||
|
||||
tracker = TwoStageTracker(profile["tracking"])
|
||||
linear_sum_assignment(np.zeros((1, 1), dtype=np.float64))
|
||||
with Image.open(frame_paths[0]) as opened:
|
||||
warm_image = np.asarray(opened.convert("RGB"), dtype=np.uint8)
|
||||
warm_tensor = _preprocess(warm_image, valid_mask, profile)
|
||||
_infer(args.triton_url, profile["model"], warm_tensor)
|
||||
del warm_tensor, warm_image
|
||||
|
||||
realtime = profile["realtime"]
|
||||
queue = LatestWinsQueue(int(realtime["queue_capacity"]))
|
||||
producer_errors: list[BaseException] = []
|
||||
latency = {
|
||||
name: []
|
||||
for name in (
|
||||
"image_decode_ms",
|
||||
"source_release_lag_ms",
|
||||
"queue_wait_ms",
|
||||
"preprocess_ms",
|
||||
"triton_request_ms",
|
||||
"postprocess_ms",
|
||||
"tracking_ms",
|
||||
"processing_ms",
|
||||
"result_age_ms",
|
||||
)
|
||||
}
|
||||
detection_labels: Counter[str] = Counter()
|
||||
track_labels: Counter[str] = Counter()
|
||||
rejection_reasons: Counter[str] = Counter()
|
||||
health_counts: Counter[str] = Counter()
|
||||
unique_track_ids: set[int] = set()
|
||||
duplicate_pairs = 0
|
||||
failures = 0
|
||||
metadata_path = output / "frames.jsonl"
|
||||
telemetry_path = output / "telemetry.jsonl"
|
||||
gpu_path = output / "gpu-telemetry.jsonl"
|
||||
disk_before = shutil.disk_usage(output).free
|
||||
replay_started = time.perf_counter() + 0.25
|
||||
producer = threading.Thread(
|
||||
target=_producer,
|
||||
kwargs={
|
||||
"queue": queue,
|
||||
"frame_paths": frame_paths,
|
||||
"timeline_rows": timeline_rows,
|
||||
"replay_started": replay_started,
|
||||
"speed": float(realtime["speed"]),
|
||||
"error": producer_errors,
|
||||
},
|
||||
name="lab-e8-source-producer",
|
||||
daemon=True,
|
||||
)
|
||||
producer.start()
|
||||
|
||||
with (
|
||||
metadata_path.open("x", encoding="utf-8", newline="\n") as metadata_stream,
|
||||
telemetry_path.open("x", encoding="utf-8", newline="\n") as telemetry_stream,
|
||||
gpu_path.open("x", encoding="utf-8", newline="\n") as gpu_stream,
|
||||
_GpuTelemetry(gpu_stream, args.telemetry_interval_seconds) as gpu_telemetry,
|
||||
):
|
||||
while (envelope := queue.take()) is not None:
|
||||
processing_started = time.perf_counter()
|
||||
latency["image_decode_ms"].append(envelope.decode_ms)
|
||||
latency["source_release_lag_ms"].append(envelope.source_release_lag_ms)
|
||||
latency["queue_wait_ms"].append(
|
||||
max(0.0, (processing_started - envelope.decoded_monotonic) * 1000.0)
|
||||
)
|
||||
try:
|
||||
preprocess_started = time.perf_counter()
|
||||
tensor = _preprocess(envelope.image, valid_mask, profile)
|
||||
latency["preprocess_ms"].append((time.perf_counter() - preprocess_started) * 1000.0)
|
||||
output_tensor, request_ms = _infer(args.triton_url, profile["model"], tensor)
|
||||
latency["triton_request_ms"].append(request_ms)
|
||||
|
||||
postprocess_started = time.perf_counter()
|
||||
detections, rejected = _detections(output_tensor, profile, valid_mask)
|
||||
latency["postprocess_ms"].append(
|
||||
(time.perf_counter() - postprocess_started) * 1000.0
|
||||
)
|
||||
rejection_reasons.update(rejected)
|
||||
detection_labels.update(str(item["label"]) for item in detections)
|
||||
|
||||
tracking_started = time.perf_counter()
|
||||
tracks = tracker.update(detections, envelope.frame_index)
|
||||
latency["tracking_ms"].append((time.perf_counter() - tracking_started) * 1000.0)
|
||||
duplicate_pairs += _duplicate_pairs(tracks)
|
||||
for track in tracks:
|
||||
unique_track_ids.add(track.track_id)
|
||||
track_labels[track.label] += 1
|
||||
delay_seconds = float(realtime["consumer_delay_ms"]) / 1000.0
|
||||
if delay_seconds:
|
||||
time.sleep(delay_seconds)
|
||||
except Exception:
|
||||
failures += 1
|
||||
raise
|
||||
|
||||
published = time.perf_counter()
|
||||
processing_ms = (published - processing_started) * 1000.0
|
||||
result_age_ms = max(0.0, (published - envelope.scheduled_monotonic) * 1000.0)
|
||||
latency["processing_ms"].append(processing_ms)
|
||||
latency["result_age_ms"].append(result_age_ms)
|
||||
health = _health(result_age_ms, realtime)
|
||||
health_counts[health] += 1
|
||||
frame_document = {
|
||||
"schema_version": FRAME_SCHEMA,
|
||||
"frame_index": envelope.frame_index,
|
||||
"sequence": envelope.frame_index + 1,
|
||||
"source_frame_index": envelope.timeline["source_frame_index"],
|
||||
"source_sequence": envelope.timeline["source_sequence"],
|
||||
"session_seconds": round(float(envelope.timeline["session_seconds"]), 9),
|
||||
"detections": detections,
|
||||
"tracks": [_track_document(track) for track in tracks],
|
||||
"delivery": {"health": health, "result_age_ms": round(result_age_ms, 6)},
|
||||
}
|
||||
metadata_stream.write(
|
||||
json.dumps(
|
||||
frame_document,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
queue_state = queue.snapshot()
|
||||
telemetry_stream.write(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": TELEMETRY_SCHEMA,
|
||||
"frame_index": envelope.frame_index,
|
||||
"session_seconds": frame_document["session_seconds"],
|
||||
"health": health,
|
||||
"result_age_ms": round(result_age_ms, 6),
|
||||
"processing_ms": round(processing_ms, 6),
|
||||
"queue_depth_after_take": queue_state["final_depth"],
|
||||
"queue_dropped_overflow": queue_state["dropped_overflow"],
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
completed = int(queue_state["consumed"])
|
||||
if completed % 100 == 0:
|
||||
metadata_stream.flush()
|
||||
telemetry_stream.flush()
|
||||
gpu_stream.flush()
|
||||
_assert_disk_floor(output, args.free_bytes_floor, completed)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"phase": "realtime-tracking",
|
||||
"frames_consumed": completed,
|
||||
"frames_published": queue_state["published"],
|
||||
"frames_dropped": queue_state["dropped_overflow"],
|
||||
"maximum_queue_depth": queue_state["maximum_depth"],
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
producer.join(timeout=5)
|
||||
if producer.is_alive():
|
||||
raise RuntimeError("LAB E8 producer did not terminate")
|
||||
if producer_errors:
|
||||
raise RuntimeError("LAB E8 producer failed") from producer_errors[0]
|
||||
metadata_stream.flush()
|
||||
telemetry_stream.flush()
|
||||
os.fsync(metadata_stream.fileno())
|
||||
os.fsync(telemetry_stream.fileno())
|
||||
|
||||
finished = time.perf_counter()
|
||||
queue_state = queue.snapshot()
|
||||
source_span = (
|
||||
float(timeline_rows[-1]["session_seconds"]) - float(timeline_rows[0]["session_seconds"])
|
||||
) / float(realtime["speed"])
|
||||
wall_seconds = finished - replay_started
|
||||
effective_fps = int(queue_state["consumed"]) / max(source_span, wall_seconds, 1e-9)
|
||||
drop_fraction = int(queue_state["dropped_overflow"]) / len(frame_paths)
|
||||
result_age = _percentiles(latency["result_age_ms"])
|
||||
acceptance = profile["acceptance"]
|
||||
expect_overload = bool(acceptance["expect_overload"])
|
||||
checks = {
|
||||
"queue_bounded": int(queue_state["maximum_depth"]) <= int(queue_state["capacity"]),
|
||||
"producer_accounting": int(queue_state["published"]) == len(frame_paths),
|
||||
"consumer_accounting": (
|
||||
int(queue_state["consumed"]) + int(queue_state["dropped_overflow"]) == len(frame_paths)
|
||||
),
|
||||
"zero_failures": failures == 0,
|
||||
}
|
||||
if expect_overload:
|
||||
checks["overload_drop_observed"] = int(queue_state["dropped_overflow"]) > 0
|
||||
checks["overload_degraded_or_stale_observed"] = (
|
||||
health_counts["stale"] + health_counts["unavailable"] > 0
|
||||
)
|
||||
else:
|
||||
checks.update(
|
||||
{
|
||||
"minimum_effective_fps": effective_fps
|
||||
>= float(acceptance["minimum_effective_fps"]),
|
||||
"maximum_drop_fraction": drop_fraction
|
||||
<= float(acceptance["maximum_drop_fraction"]),
|
||||
"maximum_p95_result_age_ms": float(result_age["p95"])
|
||||
<= float(acceptance["maximum_p95_result_age_ms"]),
|
||||
}
|
||||
)
|
||||
accepted = all(checks.values())
|
||||
|
||||
identity = {
|
||||
"schema_version": IDENTITY_SCHEMA,
|
||||
"job_id": job["job_id"],
|
||||
"input_sha256": job["input_sha256"],
|
||||
"session_id": job["input"]["session_id"],
|
||||
"source_id": job["input"]["source_id"],
|
||||
"selection": {
|
||||
"frame_count": len(frame_paths),
|
||||
"source_start_frame_index": timeline_rows[0]["source_frame_index"],
|
||||
"source_end_frame_index": timeline_rows[-1]["source_frame_index"],
|
||||
"timeline_start_seconds": timeline_rows[0]["session_seconds"],
|
||||
"timeline_end_seconds": timeline_rows[-1]["session_seconds"],
|
||||
"timeline_sha256": _sha256(timeline_path),
|
||||
},
|
||||
"configuration": {
|
||||
"pipeline": PIPELINE_ID,
|
||||
"profile_sha256": profile_sha256,
|
||||
"profile": profile,
|
||||
"runner_sha256": _sha256(Path(__file__).resolve(strict=True)),
|
||||
"orchestrator_sha256": args.orchestrator_sha256,
|
||||
"container_image": args.container_image,
|
||||
"valid_fov": valid_fov,
|
||||
},
|
||||
"models": {"detector": profile["model"], "files": model_files},
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
result_id = f"e8-realtime-tracking-{identity_sha256}"
|
||||
metrics = {
|
||||
"frames_expected": len(frame_paths),
|
||||
"frames_published": queue_state["published"],
|
||||
"frames_processed": queue_state["consumed"],
|
||||
"frames_dropped": queue_state["dropped_overflow"],
|
||||
"frames_failed": failures,
|
||||
"drop_fraction": round(drop_fraction, 9),
|
||||
"source_span_seconds": round(source_span, 6),
|
||||
"replay_wall_seconds": round(wall_seconds, 6),
|
||||
"effective_frames_per_second": round(effective_fps, 6),
|
||||
"queue": queue_state,
|
||||
"latency_ms": {name: _percentiles(values) for name, values in latency.items()},
|
||||
"health_counts": dict(sorted(health_counts.items())),
|
||||
"detections": int(sum(detection_labels.values())),
|
||||
"detections_by_label": dict(sorted(detection_labels.items())),
|
||||
"detection_rejections": dict(sorted(rejection_reasons.items())),
|
||||
"unique_confirmed_tracks": len(unique_track_ids),
|
||||
"track_observations": int(sum(track_labels.values())),
|
||||
"track_observations_by_label": dict(sorted(track_labels.items())),
|
||||
"same_class_duplicate_pairs_iou_ge_0_8": duplicate_pairs,
|
||||
"tracker_tracks_created": tracker.created,
|
||||
"tracker_tracks_retired": tracker.retired,
|
||||
"process_peak_rss_mib": round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 3),
|
||||
"gpu_telemetry": gpu_telemetry.summary(),
|
||||
"disk": {
|
||||
"free_bytes_before_replay": disk_before,
|
||||
"free_bytes_after_replay": shutil.disk_usage(output).free,
|
||||
"free_bytes_floor": args.free_bytes_floor,
|
||||
},
|
||||
}
|
||||
report = {
|
||||
"schema_version": REPORT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"state": "accepted" if accepted else "rejected",
|
||||
"ground_truth": False,
|
||||
"identity": identity,
|
||||
"runtime": {
|
||||
"hostname": platform.node(),
|
||||
"platform": platform.platform(),
|
||||
"python": platform.python_version(),
|
||||
"numpy": np.__version__,
|
||||
},
|
||||
"metrics": metrics,
|
||||
"acceptance": {
|
||||
"accepted": accepted,
|
||||
"checks": checks,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
"limitations": [
|
||||
"Recorded source-paced replay, not a live K1 transport.",
|
||||
"Generic COCO detector and IoU-only tracking are not safety validated.",
|
||||
"No semantic model, LiDAR fusion or navigation actuation is executed in this gate.",
|
||||
(
|
||||
"Decoded PNG input is temporary lab transport; camera ingest decode "
|
||||
"remains a separate gate."
|
||||
),
|
||||
],
|
||||
}
|
||||
report_path = output / "run-report.json"
|
||||
_write_json(report_path, report)
|
||||
artifacts = [
|
||||
_artifact(metadata_path, "realtime-tracking-frames", "application/x-ndjson", FRAME_SCHEMA),
|
||||
_artifact(
|
||||
telemetry_path, "realtime-tracking-telemetry", "application/x-ndjson", TELEMETRY_SCHEMA
|
||||
),
|
||||
_artifact(gpu_path, "worker-gpu-telemetry", "application/x-ndjson"),
|
||||
_artifact(report_path, "realtime-tracking-report", "application/json", REPORT_SCHEMA),
|
||||
]
|
||||
result = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": report["created_at_utc"],
|
||||
"acceptance_state": report["state"],
|
||||
"ground_truth": False,
|
||||
"publication_scope": "recorded-realtime-qualification-only",
|
||||
"frames_processed": queue_state["consumed"],
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
_write_json(output / "result.json", result)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result_id,
|
||||
"accepted": accepted,
|
||||
"frames_processed": queue_state["consumed"],
|
||||
"frames_dropped": queue_state["dropped_overflow"],
|
||||
"effective_fps": round(effective_fps, 6),
|
||||
"result_age_p95_ms": result_age["p95"],
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
return 0 if accepted else 2
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _arguments()
|
||||
if args.command == "preflight":
|
||||
return _preflight(args)
|
||||
return _run(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,933 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Qualify concurrent 10 Hz tracking and lower-rate semantic perception."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import resource
|
||||
import shutil
|
||||
import threading
|
||||
import time
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from run_e4_full_session_segmentation import (
|
||||
TARGET_CLASS_COUNT,
|
||||
_dependency_manifest,
|
||||
_load_model,
|
||||
)
|
||||
from run_e4_full_session_segmentation import (
|
||||
_profile as _read_semantic_profile,
|
||||
)
|
||||
from run_e4_full_session_segmentation import (
|
||||
_validate_source as _validate_semantic_source,
|
||||
)
|
||||
from run_e5_instance_tracking import (
|
||||
TwoStageTracker,
|
||||
_detections,
|
||||
_duplicate_pairs,
|
||||
_infer,
|
||||
_load_valid_fov,
|
||||
_preprocess,
|
||||
_read_timeline,
|
||||
_track_document,
|
||||
_validate_source,
|
||||
_verify_model,
|
||||
)
|
||||
from run_e8_realtime_tracking import (
|
||||
FrameEnvelope,
|
||||
LatestWinsQueue,
|
||||
_wait_until,
|
||||
)
|
||||
from run_e8_realtime_tracking import (
|
||||
_read_profile as _read_detector_profile,
|
||||
)
|
||||
from run_recorded_perception_epoch import (
|
||||
_artifact,
|
||||
_canonical_json,
|
||||
_GpuTelemetry,
|
||||
_percentiles,
|
||||
_sha256,
|
||||
_valid_sha256,
|
||||
_validate_job,
|
||||
_write_json,
|
||||
)
|
||||
|
||||
PROFILE_SCHEMA = "missioncore.e9-multirate-perception-profile/v1"
|
||||
DETECTOR_FRAME_SCHEMA = "missioncore.e9-multirate-detector-frame/v1"
|
||||
SEMANTIC_FRAME_SCHEMA = "missioncore.e9-multirate-semantic-frame/v1"
|
||||
MERGED_FRAME_SCHEMA = "missioncore.e9-multirate-merged-frame/v1"
|
||||
REPORT_SCHEMA = "missioncore.e9-multirate-perception-report/v1"
|
||||
RESULT_SCHEMA = "missioncore.e9-multirate-perception-result/v1"
|
||||
IDENTITY_SCHEMA = "missioncore.e9-multirate-perception-identity/v1"
|
||||
PIPELINE_ID = "concurrent-yolox-tracking-eomt-semantic-latest-wins/v1"
|
||||
|
||||
|
||||
def _arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
for name in ("preflight", "run"):
|
||||
command = commands.add_parser(name)
|
||||
command.add_argument("--job", type=Path, required=True)
|
||||
command.add_argument("--profile", type=Path, required=True)
|
||||
command.add_argument("--detector-profile", type=Path, required=True)
|
||||
command.add_argument("--semantic-profile", type=Path, required=True)
|
||||
command.add_argument("--valid-fov-root", type=Path, required=True)
|
||||
command.add_argument("--model-root", type=Path, required=True)
|
||||
command.add_argument("--cache", type=Path, required=True)
|
||||
command.add_argument("--environment", type=Path, required=True)
|
||||
if name == "run":
|
||||
command.add_argument("--frames", type=Path, required=True)
|
||||
command.add_argument("--timeline", type=Path, required=True)
|
||||
command.add_argument("--output", type=Path, required=True)
|
||||
command.add_argument("--triton-url", required=True)
|
||||
command.add_argument("--free-bytes-floor", type=int, default=0)
|
||||
command.add_argument("--orchestrator-sha256", required=True)
|
||||
command.add_argument("--container-image", required=True)
|
||||
command.add_argument("--telemetry-interval-seconds", type=float, default=1.0)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _read_object(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise RuntimeError(f"JSON root is not an object: {path}")
|
||||
return value
|
||||
|
||||
|
||||
def _read_profile(path: Path) -> tuple[dict[str, Any], str]:
|
||||
resolved = path.resolve(strict=True)
|
||||
profile = _read_object(resolved)
|
||||
replay = profile.get("replay")
|
||||
acceptance = profile.get("acceptance")
|
||||
source = profile.get("source")
|
||||
if (
|
||||
profile.get("schema_version") != PROFILE_SCHEMA
|
||||
or profile.get("mode") not in {"pilot", "qualification"}
|
||||
or not isinstance(replay, dict)
|
||||
or not isinstance(acceptance, dict)
|
||||
or not isinstance(source, dict)
|
||||
or source.get("source_id") != "sensor.camera.right"
|
||||
or source.get("resolution") != [800, 600]
|
||||
):
|
||||
raise RuntimeError("LAB E9 profile contract changed")
|
||||
speed = replay.get("speed")
|
||||
detector_capacity = replay.get("detector_queue_capacity")
|
||||
semantic_capacity = replay.get("semantic_queue_capacity")
|
||||
semantic_stride = replay.get("semantic_sample_every_frames")
|
||||
semantic_ttl = replay.get("semantic_ttl_ms")
|
||||
if (
|
||||
not _positive_number(speed)
|
||||
or not 0.1 <= float(speed) <= 10
|
||||
or not isinstance(detector_capacity, int)
|
||||
or isinstance(detector_capacity, bool)
|
||||
or not 1 <= detector_capacity <= 8
|
||||
or not isinstance(semantic_capacity, int)
|
||||
or isinstance(semantic_capacity, bool)
|
||||
or not 1 <= semantic_capacity <= 4
|
||||
or not isinstance(semantic_stride, int)
|
||||
or isinstance(semantic_stride, bool)
|
||||
or not 2 <= semantic_stride <= 30
|
||||
or not _positive_number(semantic_ttl)
|
||||
or not 100 <= float(semantic_ttl) <= 5000
|
||||
):
|
||||
raise RuntimeError("LAB E9 replay configuration is invalid")
|
||||
required = (
|
||||
"detector_minimum_effective_fps",
|
||||
"detector_maximum_drop_fraction",
|
||||
"detector_maximum_p95_result_age_ms",
|
||||
"semantic_minimum_effective_fps",
|
||||
"semantic_maximum_drop_fraction",
|
||||
"semantic_maximum_p95_completion_age_ms",
|
||||
"minimum_fresh_semantic_coverage",
|
||||
)
|
||||
if (
|
||||
any(not _nonnegative_number(acceptance.get(name)) for name in required)
|
||||
or float(acceptance["detector_maximum_drop_fraction"]) > 1
|
||||
or float(acceptance["semantic_maximum_drop_fraction"]) > 1
|
||||
or float(acceptance["minimum_fresh_semantic_coverage"]) > 1
|
||||
or not isinstance(acceptance.get("require_zero_failures"), bool)
|
||||
):
|
||||
raise RuntimeError("LAB E9 acceptance configuration is invalid")
|
||||
return profile, _sha256(resolved)
|
||||
|
||||
|
||||
def _positive_number(value: object) -> bool:
|
||||
return _nonnegative_number(value) and float(value) > 0
|
||||
|
||||
|
||||
def _nonnegative_number(value: object) -> bool:
|
||||
return (
|
||||
isinstance(value, int | float)
|
||||
and not isinstance(value, bool)
|
||||
and math.isfinite(float(value))
|
||||
and float(value) >= 0
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SemanticResult:
|
||||
frame_index: int
|
||||
source_frame_index: int
|
||||
session_seconds: float
|
||||
completed_monotonic: float
|
||||
completion_age_ms: float
|
||||
mask_sha256: str
|
||||
class_pixels: dict[str, int]
|
||||
class_fractions: dict[str, float]
|
||||
|
||||
|
||||
class LatestSemantic:
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._value: SemanticResult | None = None
|
||||
|
||||
def publish(self, value: SemanticResult) -> None:
|
||||
with self._lock:
|
||||
if self._value is not None and value.frame_index <= self._value.frame_index:
|
||||
raise RuntimeError("LAB E9 semantic results are not monotonic")
|
||||
self._value = value
|
||||
|
||||
def snapshot(self) -> SemanticResult | None:
|
||||
with self._lock:
|
||||
return self._value
|
||||
|
||||
|
||||
def _producer(
|
||||
*,
|
||||
detector_queue: LatestWinsQueue,
|
||||
semantic_queue: LatestWinsQueue,
|
||||
semantic_stride: int,
|
||||
frame_paths: list[Path],
|
||||
timeline_rows: list[dict[str, Any]],
|
||||
replay_started: float,
|
||||
speed: float,
|
||||
error: list[BaseException],
|
||||
) -> None:
|
||||
from PIL import Image
|
||||
|
||||
first_session = float(timeline_rows[0]["session_seconds"])
|
||||
try:
|
||||
for frame_index, (path, row) in enumerate(zip(frame_paths, timeline_rows, strict=True)):
|
||||
scheduled = replay_started + (float(row["session_seconds"]) - first_session) / speed
|
||||
_wait_until(scheduled)
|
||||
decode_started = time.perf_counter()
|
||||
with Image.open(path) as opened:
|
||||
image = np.asarray(opened.convert("RGB"), dtype=np.uint8)
|
||||
decoded = time.perf_counter()
|
||||
if image.shape != (600, 800, 3):
|
||||
raise RuntimeError("LAB E9 input resolution changed")
|
||||
envelope = FrameEnvelope(
|
||||
frame_index=frame_index,
|
||||
path=path,
|
||||
timeline=row,
|
||||
scheduled_monotonic=scheduled,
|
||||
decoded_monotonic=decoded,
|
||||
image=image,
|
||||
decode_ms=(decoded - decode_started) * 1000.0,
|
||||
source_release_lag_ms=max(0.0, (decode_started - scheduled) * 1000.0),
|
||||
)
|
||||
detector_queue.publish(envelope)
|
||||
if frame_index % semantic_stride == 0:
|
||||
semantic_queue.publish(envelope)
|
||||
except BaseException as exc:
|
||||
error.append(exc)
|
||||
finally:
|
||||
detector_queue.close()
|
||||
semantic_queue.close()
|
||||
|
||||
|
||||
def _semantic_worker(
|
||||
*,
|
||||
queue: LatestWinsQueue,
|
||||
latest: LatestSemantic,
|
||||
stream: Any,
|
||||
valid_mask: np.ndarray,
|
||||
target_lut: np.ndarray,
|
||||
target_names: dict[int, str],
|
||||
infer: Any,
|
||||
latency: dict[str, list[float]],
|
||||
failures: list[BaseException],
|
||||
) -> None:
|
||||
try:
|
||||
while (envelope := queue.take()) is not None:
|
||||
started = time.perf_counter()
|
||||
latency["image_decode_ms"].append(envelope.decode_ms)
|
||||
latency["queue_wait_ms"].append(
|
||||
max(0.0, (started - envelope.decoded_monotonic) * 1000.0)
|
||||
)
|
||||
fill_started = time.perf_counter()
|
||||
model_input = np.where(valid_mask[..., None], envelope.image, 0).astype(np.uint8)
|
||||
latency["valid_fov_fill_ms"].append((time.perf_counter() - fill_started) * 1000.0)
|
||||
semantic, measured = infer(model_input)
|
||||
if semantic.max() >= len(target_lut):
|
||||
raise RuntimeError("LAB E9 EoMT emitted an unknown category")
|
||||
target = target_lut[semantic]
|
||||
target = target.copy()
|
||||
target[~valid_mask] = 0
|
||||
for name, value in measured.items():
|
||||
latency[name].append(float(value))
|
||||
completed = time.perf_counter()
|
||||
completion_age_ms = max(0.0, (completed - envelope.scheduled_monotonic) * 1000.0)
|
||||
latency["completion_age_ms"].append(completion_age_ms)
|
||||
latency["processing_ms"].append((completed - started) * 1000.0)
|
||||
counts = np.bincount(target[valid_mask], minlength=TARGET_CLASS_COUNT)
|
||||
class_pixels = {
|
||||
target_names[index]: int(counts[index])
|
||||
for index in range(1, TARGET_CLASS_COUNT)
|
||||
if int(counts[index]) > 0
|
||||
}
|
||||
valid_pixels = int(valid_mask.sum())
|
||||
result = SemanticResult(
|
||||
frame_index=envelope.frame_index,
|
||||
source_frame_index=int(envelope.timeline["source_frame_index"]),
|
||||
session_seconds=float(envelope.timeline["session_seconds"]),
|
||||
completed_monotonic=completed,
|
||||
completion_age_ms=completion_age_ms,
|
||||
mask_sha256=hashlib.sha256(target.tobytes()).hexdigest(),
|
||||
class_pixels=class_pixels,
|
||||
class_fractions={
|
||||
name: round(count / valid_pixels, 9) for name, count in class_pixels.items()
|
||||
},
|
||||
)
|
||||
latest.publish(result)
|
||||
stream.write(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": SEMANTIC_FRAME_SCHEMA,
|
||||
"frame_index": result.frame_index,
|
||||
"source_frame_index": result.source_frame_index,
|
||||
"session_seconds": round(result.session_seconds, 9),
|
||||
"completion_age_ms": round(result.completion_age_ms, 6),
|
||||
"mask_sha256": result.mask_sha256,
|
||||
"class_pixels": result.class_pixels,
|
||||
"class_fractions": result.class_fractions,
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
except BaseException as exc:
|
||||
failures.append(exc)
|
||||
|
||||
|
||||
def _semantic_binding(
|
||||
semantic: SemanticResult | None,
|
||||
*,
|
||||
frame_session_seconds: float,
|
||||
ttl_ms: float,
|
||||
) -> dict[str, Any]:
|
||||
if semantic is None:
|
||||
return {
|
||||
"status": "unavailable",
|
||||
"source_frame_index": None,
|
||||
"source_age_ms": None,
|
||||
"completion_age_ms": None,
|
||||
"mask_sha256": None,
|
||||
}
|
||||
source_age_ms = max(0.0, (frame_session_seconds - semantic.session_seconds) * 1000.0)
|
||||
return {
|
||||
"status": "fresh" if source_age_ms <= ttl_ms else "stale",
|
||||
"source_frame_index": semantic.source_frame_index,
|
||||
"source_age_ms": round(source_age_ms, 6),
|
||||
"completion_age_ms": round(semantic.completion_age_ms, 6),
|
||||
"mask_sha256": semantic.mask_sha256,
|
||||
}
|
||||
|
||||
|
||||
def _semantic_infer_factory(
|
||||
processor: Any,
|
||||
model: Any,
|
||||
device: Any,
|
||||
) -> Any:
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
def infer(image: np.ndarray) -> tuple[np.ndarray, dict[str, float]]:
|
||||
processor_started = time.perf_counter()
|
||||
inputs = processor(images=Image.fromarray(image), return_tensors="pt")
|
||||
processor_ms = (time.perf_counter() - processor_started) * 1000.0
|
||||
torch.cuda.synchronize()
|
||||
transfer_started = time.perf_counter()
|
||||
inputs = {
|
||||
name: value.to(device) if isinstance(value, torch.Tensor) else value
|
||||
for name, value in inputs.items()
|
||||
}
|
||||
torch.cuda.synchronize()
|
||||
transfer_ms = (time.perf_counter() - transfer_started) * 1000.0
|
||||
forward_started = time.perf_counter()
|
||||
with torch.inference_mode(), torch.amp.autocast("cuda", dtype=torch.float16):
|
||||
outputs = model(**inputs)
|
||||
torch.cuda.synchronize()
|
||||
forward_ms = (time.perf_counter() - forward_started) * 1000.0
|
||||
post_started = time.perf_counter()
|
||||
semantic = processor.post_process_semantic_segmentation(
|
||||
outputs,
|
||||
target_sizes=[image.shape[:2]],
|
||||
)[0]
|
||||
semantic = semantic.detach().cpu().numpy().astype(np.uint8)
|
||||
post_ms = (time.perf_counter() - post_started) * 1000.0
|
||||
del inputs, outputs
|
||||
return semantic, {
|
||||
"processor_ms": processor_ms,
|
||||
"host_to_device_ms": transfer_ms,
|
||||
"forward_ms": forward_ms,
|
||||
"model_postprocess_ms": post_ms,
|
||||
}
|
||||
|
||||
return infer
|
||||
|
||||
|
||||
def _preflight(args: argparse.Namespace) -> int:
|
||||
import torch
|
||||
|
||||
job = _validate_job(args.job.resolve(strict=True))
|
||||
profile, profile_sha256 = _read_profile(args.profile)
|
||||
detector_profile, detector_profile_sha256 = _read_detector_profile(args.detector_profile)
|
||||
semantic_profile, semantic_profile_sha256 = _read_semantic_profile(args.semantic_profile)
|
||||
_validate_source(job, detector_profile)
|
||||
_validate_semantic_source(job, semantic_profile)
|
||||
if profile["source"] != detector_profile["source"]:
|
||||
raise RuntimeError("LAB E9 source and detector bindings differ")
|
||||
if profile["source"] != {
|
||||
key: semantic_profile["source"][key]
|
||||
for key in ("source_id", "resolution", "calibration_slot", "calibration_sha256")
|
||||
}:
|
||||
raise RuntimeError("LAB E9 source and semantic bindings differ")
|
||||
valid_mask, valid_fov = _load_valid_fov(args.valid_fov_root, job, detector_profile)
|
||||
detector_files = _verify_model(detector_profile, args.model_root)
|
||||
dependency = _dependency_manifest(args.environment.resolve(strict=True))
|
||||
if dependency["identity"]["profile_sha256"] != semantic_profile_sha256:
|
||||
raise RuntimeError("LAB E9 semantic dependencies belong to another profile")
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("CUDA is required for LAB E9")
|
||||
device = torch.device("cuda:0")
|
||||
processor, model, _labels, _lut, semantic_files = _load_model(
|
||||
semantic_profile,
|
||||
args.cache.resolve(strict=True),
|
||||
device,
|
||||
)
|
||||
del processor, model, _labels, _lut
|
||||
torch.cuda.empty_cache()
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"state": "preflight-ready",
|
||||
"job_id": job["job_id"],
|
||||
"profile_sha256": profile_sha256,
|
||||
"detector_profile_sha256": detector_profile_sha256,
|
||||
"semantic_profile_sha256": semantic_profile_sha256,
|
||||
"detector_files": detector_files,
|
||||
"semantic_files": semantic_files,
|
||||
"valid_fov": valid_fov,
|
||||
"cuda_device": torch.cuda.get_device_name(),
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def _run(args: argparse.Namespace) -> int:
|
||||
import torch
|
||||
import transformers
|
||||
from PIL import Image
|
||||
from scipy.optimize import linear_sum_assignment
|
||||
|
||||
if not _valid_sha256(args.orchestrator_sha256):
|
||||
raise RuntimeError("LAB E9 orchestrator SHA-256 is invalid")
|
||||
job = _validate_job(args.job.resolve(strict=True))
|
||||
profile, profile_sha256 = _read_profile(args.profile)
|
||||
detector_profile, detector_profile_sha256 = _read_detector_profile(args.detector_profile)
|
||||
semantic_profile, semantic_profile_sha256 = _read_semantic_profile(args.semantic_profile)
|
||||
_validate_source(job, detector_profile)
|
||||
_validate_semantic_source(job, semantic_profile)
|
||||
valid_mask, valid_fov = _load_valid_fov(args.valid_fov_root, job, detector_profile)
|
||||
detector_files = _verify_model(detector_profile, args.model_root)
|
||||
dependency = _dependency_manifest(args.environment.resolve(strict=True))
|
||||
if dependency["identity"]["profile_sha256"] != semantic_profile_sha256:
|
||||
raise RuntimeError("LAB E9 semantic dependency identity changed")
|
||||
frames_root = args.frames.resolve(strict=True)
|
||||
frame_paths = sorted(frames_root.glob("frame-*.png"))
|
||||
if not frame_paths or [path.name for path in frame_paths] != [
|
||||
f"frame-{index:06d}.png" for index in range(1, len(frame_paths) + 1)
|
||||
]:
|
||||
raise RuntimeError("LAB E9 decoded frame sequence changed")
|
||||
timeline_path = args.timeline.resolve(strict=True)
|
||||
timeline_rows = _read_timeline(timeline_path, len(frame_paths))
|
||||
|
||||
output = args.output.resolve()
|
||||
if output.exists():
|
||||
raise RuntimeError("LAB E9 output must be absent")
|
||||
output.mkdir(mode=0o700, parents=True, exist_ok=False)
|
||||
_assert_disk_floor(output, args.free_bytes_floor, 0)
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("CUDA is required for LAB E9")
|
||||
device = torch.device("cuda:0")
|
||||
processor, semantic_model, semantic_labels, target_lut, semantic_files = _load_model(
|
||||
semantic_profile,
|
||||
args.cache.resolve(strict=True),
|
||||
device,
|
||||
)
|
||||
target_names = {
|
||||
int(key): str(value) for key, value in semantic_profile["target_taxonomy"].items()
|
||||
}
|
||||
if set(target_names) != set(range(TARGET_CLASS_COUNT)):
|
||||
raise RuntimeError("LAB E9 target taxonomy changed")
|
||||
infer_semantic = _semantic_infer_factory(processor, semantic_model, device)
|
||||
|
||||
tracker = TwoStageTracker(detector_profile["tracking"])
|
||||
linear_sum_assignment(np.zeros((1, 1), dtype=np.float64))
|
||||
with Image.open(frame_paths[0]) as opened:
|
||||
warm_image = np.asarray(opened.convert("RGB"), dtype=np.uint8)
|
||||
detector_tensor = _preprocess(warm_image, valid_mask, detector_profile)
|
||||
_infer(args.triton_url, detector_profile["model"], detector_tensor)
|
||||
semantic_input = np.where(valid_mask[..., None], warm_image, 0).astype(np.uint8)
|
||||
warm_semantic, _warm_latency = infer_semantic(semantic_input)
|
||||
del detector_tensor, semantic_input, warm_semantic, _warm_latency, warm_image
|
||||
|
||||
replay = profile["replay"]
|
||||
detector_queue = LatestWinsQueue(int(replay["detector_queue_capacity"]))
|
||||
semantic_queue = LatestWinsQueue(int(replay["semantic_queue_capacity"]))
|
||||
latest_semantic = LatestSemantic()
|
||||
producer_errors: list[BaseException] = []
|
||||
semantic_errors: list[BaseException] = []
|
||||
detector_failures = 0
|
||||
detector_latency = {
|
||||
name: []
|
||||
for name in (
|
||||
"image_decode_ms",
|
||||
"source_release_lag_ms",
|
||||
"queue_wait_ms",
|
||||
"preprocess_ms",
|
||||
"triton_request_ms",
|
||||
"postprocess_ms",
|
||||
"tracking_ms",
|
||||
"processing_ms",
|
||||
"result_age_ms",
|
||||
)
|
||||
}
|
||||
semantic_latency = {
|
||||
name: []
|
||||
for name in (
|
||||
"image_decode_ms",
|
||||
"queue_wait_ms",
|
||||
"valid_fov_fill_ms",
|
||||
"processor_ms",
|
||||
"host_to_device_ms",
|
||||
"forward_ms",
|
||||
"model_postprocess_ms",
|
||||
"processing_ms",
|
||||
"completion_age_ms",
|
||||
)
|
||||
}
|
||||
detector_labels: Counter[str] = Counter()
|
||||
track_labels: Counter[str] = Counter()
|
||||
semantic_status: Counter[str] = Counter()
|
||||
unique_tracks: set[int] = set()
|
||||
duplicate_pairs = 0
|
||||
detector_path = output / "detector-frames.jsonl"
|
||||
semantic_path = output / "semantic-frames.jsonl"
|
||||
merged_path = output / "merged-frames.jsonl"
|
||||
gpu_path = output / "gpu-telemetry.jsonl"
|
||||
disk_before = shutil.disk_usage(output).free
|
||||
|
||||
with (
|
||||
detector_path.open("x", encoding="utf-8", newline="\n") as detector_stream,
|
||||
semantic_path.open("x", encoding="utf-8", newline="\n") as semantic_stream,
|
||||
merged_path.open("x", encoding="utf-8", newline="\n") as merged_stream,
|
||||
gpu_path.open("x", encoding="utf-8", newline="\n") as gpu_stream,
|
||||
_GpuTelemetry(gpu_stream, args.telemetry_interval_seconds) as gpu_telemetry,
|
||||
):
|
||||
semantic_thread = threading.Thread(
|
||||
target=_semantic_worker,
|
||||
kwargs={
|
||||
"queue": semantic_queue,
|
||||
"latest": latest_semantic,
|
||||
"stream": semantic_stream,
|
||||
"valid_mask": valid_mask,
|
||||
"target_lut": target_lut,
|
||||
"target_names": target_names,
|
||||
"infer": infer_semantic,
|
||||
"latency": semantic_latency,
|
||||
"failures": semantic_errors,
|
||||
},
|
||||
name="lab-e9-semantic-consumer",
|
||||
daemon=True,
|
||||
)
|
||||
semantic_thread.start()
|
||||
replay_started = time.perf_counter() + 0.25
|
||||
producer = threading.Thread(
|
||||
target=_producer,
|
||||
kwargs={
|
||||
"detector_queue": detector_queue,
|
||||
"semantic_queue": semantic_queue,
|
||||
"semantic_stride": int(replay["semantic_sample_every_frames"]),
|
||||
"frame_paths": frame_paths,
|
||||
"timeline_rows": timeline_rows,
|
||||
"replay_started": replay_started,
|
||||
"speed": float(replay["speed"]),
|
||||
"error": producer_errors,
|
||||
},
|
||||
name="lab-e9-source-producer",
|
||||
daemon=True,
|
||||
)
|
||||
producer.start()
|
||||
|
||||
while (envelope := detector_queue.take()) is not None:
|
||||
started = time.perf_counter()
|
||||
detector_latency["image_decode_ms"].append(envelope.decode_ms)
|
||||
detector_latency["source_release_lag_ms"].append(envelope.source_release_lag_ms)
|
||||
detector_latency["queue_wait_ms"].append(
|
||||
max(0.0, (started - envelope.decoded_monotonic) * 1000.0)
|
||||
)
|
||||
try:
|
||||
preprocess_started = time.perf_counter()
|
||||
tensor = _preprocess(envelope.image, valid_mask, detector_profile)
|
||||
detector_latency["preprocess_ms"].append(
|
||||
(time.perf_counter() - preprocess_started) * 1000.0
|
||||
)
|
||||
output_tensor, request_ms = _infer(
|
||||
args.triton_url, detector_profile["model"], tensor
|
||||
)
|
||||
detector_latency["triton_request_ms"].append(request_ms)
|
||||
post_started = time.perf_counter()
|
||||
detections, _rejected = _detections(output_tensor, detector_profile, valid_mask)
|
||||
detector_latency["postprocess_ms"].append(
|
||||
(time.perf_counter() - post_started) * 1000.0
|
||||
)
|
||||
tracking_started = time.perf_counter()
|
||||
tracks = tracker.update(detections, envelope.frame_index)
|
||||
detector_latency["tracking_ms"].append(
|
||||
(time.perf_counter() - tracking_started) * 1000.0
|
||||
)
|
||||
except Exception:
|
||||
detector_failures += 1
|
||||
raise
|
||||
completed = time.perf_counter()
|
||||
processing_ms = (completed - started) * 1000.0
|
||||
result_age_ms = max(0.0, (completed - envelope.scheduled_monotonic) * 1000.0)
|
||||
detector_latency["processing_ms"].append(processing_ms)
|
||||
detector_latency["result_age_ms"].append(result_age_ms)
|
||||
detector_labels.update(str(item["label"]) for item in detections)
|
||||
duplicate_pairs += _duplicate_pairs(tracks)
|
||||
for track in tracks:
|
||||
unique_tracks.add(track.track_id)
|
||||
track_labels[track.label] += 1
|
||||
semantic_binding = _semantic_binding(
|
||||
latest_semantic.snapshot(),
|
||||
frame_session_seconds=float(envelope.timeline["session_seconds"]),
|
||||
ttl_ms=float(replay["semantic_ttl_ms"]),
|
||||
)
|
||||
semantic_status[str(semantic_binding["status"])] += 1
|
||||
detector_document = {
|
||||
"schema_version": DETECTOR_FRAME_SCHEMA,
|
||||
"frame_index": envelope.frame_index,
|
||||
"source_frame_index": envelope.timeline["source_frame_index"],
|
||||
"session_seconds": round(float(envelope.timeline["session_seconds"]), 9),
|
||||
"result_age_ms": round(result_age_ms, 6),
|
||||
"detections": detections,
|
||||
"tracks": [_track_document(track) for track in tracks],
|
||||
}
|
||||
detector_stream.write(
|
||||
json.dumps(
|
||||
detector_document,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
merged_stream.write(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": MERGED_FRAME_SCHEMA,
|
||||
"frame_index": envelope.frame_index,
|
||||
"source_frame_index": envelope.timeline["source_frame_index"],
|
||||
"session_seconds": detector_document["session_seconds"],
|
||||
"detector_result_age_ms": round(result_age_ms, 6),
|
||||
"tracks": detector_document["tracks"],
|
||||
"semantic": semantic_binding,
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
consumed = int(detector_queue.snapshot()["consumed"])
|
||||
if consumed % 100 == 0:
|
||||
detector_stream.flush()
|
||||
merged_stream.flush()
|
||||
semantic_stream.flush()
|
||||
gpu_stream.flush()
|
||||
_assert_disk_floor(output, args.free_bytes_floor, consumed)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"phase": "multirate",
|
||||
"detector_consumed": consumed,
|
||||
"detector_dropped": detector_queue.snapshot()["dropped_overflow"],
|
||||
"semantic_consumed": semantic_queue.snapshot()["consumed"],
|
||||
"semantic_dropped": semantic_queue.snapshot()["dropped_overflow"],
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
producer.join(timeout=5)
|
||||
if producer.is_alive() or producer_errors:
|
||||
raise RuntimeError("LAB E9 producer failed")
|
||||
semantic_thread.join(timeout=30)
|
||||
if semantic_thread.is_alive():
|
||||
raise RuntimeError("LAB E9 semantic consumer did not terminate")
|
||||
if semantic_errors:
|
||||
raise RuntimeError("LAB E9 semantic consumer failed") from semantic_errors[0]
|
||||
detector_stream.flush()
|
||||
semantic_stream.flush()
|
||||
merged_stream.flush()
|
||||
os.fsync(detector_stream.fileno())
|
||||
os.fsync(semantic_stream.fileno())
|
||||
os.fsync(merged_stream.fileno())
|
||||
|
||||
replay_finished = time.perf_counter()
|
||||
detector_state = detector_queue.snapshot()
|
||||
semantic_state = semantic_queue.snapshot()
|
||||
source_span = (
|
||||
float(timeline_rows[-1]["session_seconds"]) - float(timeline_rows[0]["session_seconds"])
|
||||
) / float(replay["speed"])
|
||||
replay_wall = replay_finished - replay_started
|
||||
scheduled_semantic = ((len(frame_paths) - 1) // int(replay["semantic_sample_every_frames"])) + 1
|
||||
detector_fps = int(detector_state["consumed"]) / max(source_span, replay_wall, 1e-9)
|
||||
semantic_fps = int(semantic_state["consumed"]) / max(source_span, replay_wall, 1e-9)
|
||||
detector_drop_fraction = int(detector_state["dropped_overflow"]) / len(frame_paths)
|
||||
semantic_drop_fraction = int(semantic_state["dropped_overflow"]) / scheduled_semantic
|
||||
fresh_coverage = semantic_status["fresh"] / max(1, int(detector_state["consumed"]))
|
||||
detector_percentiles = {name: _percentiles(values) for name, values in detector_latency.items()}
|
||||
semantic_percentiles = {name: _percentiles(values) for name, values in semantic_latency.items()}
|
||||
acceptance = profile["acceptance"]
|
||||
checks = {
|
||||
"detector_queue_bounded": int(detector_state["maximum_depth"])
|
||||
<= int(detector_state["capacity"]),
|
||||
"detector_accounting": int(detector_state["consumed"])
|
||||
+ int(detector_state["dropped_overflow"])
|
||||
== len(frame_paths),
|
||||
"detector_minimum_effective_fps": detector_fps
|
||||
>= float(acceptance["detector_minimum_effective_fps"]),
|
||||
"detector_maximum_drop_fraction": detector_drop_fraction
|
||||
<= float(acceptance["detector_maximum_drop_fraction"]),
|
||||
"detector_maximum_p95_result_age_ms": float(detector_percentiles["result_age_ms"]["p95"])
|
||||
<= float(acceptance["detector_maximum_p95_result_age_ms"]),
|
||||
"semantic_queue_bounded": int(semantic_state["maximum_depth"])
|
||||
<= int(semantic_state["capacity"]),
|
||||
"semantic_accounting": int(semantic_state["consumed"])
|
||||
+ int(semantic_state["dropped_overflow"])
|
||||
== scheduled_semantic,
|
||||
"semantic_minimum_effective_fps": semantic_fps
|
||||
>= float(acceptance["semantic_minimum_effective_fps"]),
|
||||
"semantic_maximum_drop_fraction": semantic_drop_fraction
|
||||
<= float(acceptance["semantic_maximum_drop_fraction"]),
|
||||
"semantic_maximum_p95_completion_age_ms": float(
|
||||
semantic_percentiles["completion_age_ms"]["p95"]
|
||||
)
|
||||
<= float(acceptance["semantic_maximum_p95_completion_age_ms"]),
|
||||
"minimum_fresh_semantic_coverage": fresh_coverage
|
||||
>= float(acceptance["minimum_fresh_semantic_coverage"]),
|
||||
"zero_failures": detector_failures == 0 and not semantic_errors,
|
||||
}
|
||||
accepted = all(checks.values())
|
||||
|
||||
identity = {
|
||||
"schema_version": IDENTITY_SCHEMA,
|
||||
"job_id": job["job_id"],
|
||||
"input_sha256": job["input_sha256"],
|
||||
"session_id": job["input"]["session_id"],
|
||||
"source_id": job["input"]["source_id"],
|
||||
"selection": {
|
||||
"frame_count": len(frame_paths),
|
||||
"source_start_frame_index": timeline_rows[0]["source_frame_index"],
|
||||
"source_end_frame_index": timeline_rows[-1]["source_frame_index"],
|
||||
"timeline_start_seconds": timeline_rows[0]["session_seconds"],
|
||||
"timeline_end_seconds": timeline_rows[-1]["session_seconds"],
|
||||
"timeline_sha256": _sha256(timeline_path),
|
||||
},
|
||||
"configuration": {
|
||||
"pipeline": PIPELINE_ID,
|
||||
"profile": profile,
|
||||
"profile_sha256": profile_sha256,
|
||||
"detector_profile_sha256": detector_profile_sha256,
|
||||
"semantic_profile_sha256": semantic_profile_sha256,
|
||||
"runner_sha256": _sha256(Path(__file__).resolve(strict=True)),
|
||||
"orchestrator_sha256": args.orchestrator_sha256,
|
||||
"container_image": args.container_image,
|
||||
"valid_fov": valid_fov,
|
||||
},
|
||||
"models": {
|
||||
"detector": detector_profile["model"],
|
||||
"detector_files": detector_files,
|
||||
"semantic": semantic_profile["model"],
|
||||
"semantic_files": semantic_files,
|
||||
},
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
result_id = f"e9-multirate-perception-{identity_sha256}"
|
||||
metrics = {
|
||||
"source_span_seconds": round(source_span, 6),
|
||||
"replay_wall_seconds": round(replay_wall, 6),
|
||||
"detector": {
|
||||
"frames_expected": len(frame_paths),
|
||||
"frames_processed": detector_state["consumed"],
|
||||
"frames_dropped": detector_state["dropped_overflow"],
|
||||
"drop_fraction": round(detector_drop_fraction, 9),
|
||||
"effective_frames_per_second": round(detector_fps, 6),
|
||||
"queue": detector_state,
|
||||
"latency_ms": detector_percentiles,
|
||||
"detections": int(sum(detector_labels.values())),
|
||||
"unique_confirmed_tracks": len(unique_tracks),
|
||||
"track_observations": int(sum(track_labels.values())),
|
||||
"same_class_duplicate_pairs_iou_ge_0_8": duplicate_pairs,
|
||||
},
|
||||
"semantic": {
|
||||
"frames_scheduled": scheduled_semantic,
|
||||
"frames_processed": semantic_state["consumed"],
|
||||
"frames_dropped": semantic_state["dropped_overflow"],
|
||||
"drop_fraction": round(semantic_drop_fraction, 9),
|
||||
"effective_frames_per_second": round(semantic_fps, 6),
|
||||
"queue": semantic_state,
|
||||
"latency_ms": semantic_percentiles,
|
||||
"binding_status_counts": dict(sorted(semantic_status.items())),
|
||||
"fresh_coverage": round(fresh_coverage, 9),
|
||||
},
|
||||
"gpu_telemetry": gpu_telemetry.summary(),
|
||||
"process_peak_rss_mib": round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 3),
|
||||
"cuda_peak_memory_allocated_mib": round(torch.cuda.max_memory_allocated() / 2**20, 3),
|
||||
"cuda_peak_memory_reserved_mib": round(torch.cuda.max_memory_reserved() / 2**20, 3),
|
||||
"disk": {
|
||||
"free_bytes_before_replay": disk_before,
|
||||
"free_bytes_after_replay": shutil.disk_usage(output).free,
|
||||
"free_bytes_floor": args.free_bytes_floor,
|
||||
},
|
||||
}
|
||||
report = {
|
||||
"schema_version": REPORT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"state": "accepted" if accepted else "rejected",
|
||||
"ground_truth": False,
|
||||
"identity": identity,
|
||||
"runtime": {
|
||||
"hostname": platform.node(),
|
||||
"platform": platform.platform(),
|
||||
"python": platform.python_version(),
|
||||
"numpy": np.__version__,
|
||||
"torch": torch.__version__,
|
||||
"transformers": transformers.__version__,
|
||||
"scipy": importlib.metadata.version("scipy"),
|
||||
"gpu": torch.cuda.get_device_name(),
|
||||
},
|
||||
"metrics": metrics,
|
||||
"acceptance": {
|
||||
"accepted": accepted,
|
||||
"checks": checks,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
"limitations": [
|
||||
"Recorded source-paced replay, not live K1 transport.",
|
||||
(
|
||||
"Semantic masks are measured in memory; this gate stores hashes and "
|
||||
"class totals, not mask images."
|
||||
),
|
||||
"Generic Cityscapes and COCO models are not forest-domain or safety validated.",
|
||||
"LiDAR fusion and 3D world-state publication remain downstream gates.",
|
||||
],
|
||||
}
|
||||
report_path = output / "run-report.json"
|
||||
_write_json(report_path, report)
|
||||
artifacts = [
|
||||
_artifact(
|
||||
detector_path,
|
||||
"multirate-detector-frames",
|
||||
"application/x-ndjson",
|
||||
DETECTOR_FRAME_SCHEMA,
|
||||
),
|
||||
_artifact(
|
||||
semantic_path,
|
||||
"multirate-semantic-frames",
|
||||
"application/x-ndjson",
|
||||
SEMANTIC_FRAME_SCHEMA,
|
||||
),
|
||||
_artifact(
|
||||
merged_path, "multirate-merged-frames", "application/x-ndjson", MERGED_FRAME_SCHEMA
|
||||
),
|
||||
_artifact(gpu_path, "worker-gpu-telemetry", "application/x-ndjson"),
|
||||
_artifact(report_path, "multirate-run-report", "application/json", REPORT_SCHEMA),
|
||||
]
|
||||
result = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": report["created_at_utc"],
|
||||
"acceptance_state": report["state"],
|
||||
"ground_truth": False,
|
||||
"publication_scope": "recorded-multirate-qualification-only",
|
||||
"frames_processed": detector_state["consumed"],
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
_write_json(output / "result.json", result)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result_id,
|
||||
"accepted": accepted,
|
||||
"detector_fps": round(detector_fps, 6),
|
||||
"detector_dropped": detector_state["dropped_overflow"],
|
||||
"detector_result_age_p95_ms": detector_percentiles["result_age_ms"]["p95"],
|
||||
"semantic_fps": round(semantic_fps, 6),
|
||||
"semantic_dropped": semantic_state["dropped_overflow"],
|
||||
"semantic_completion_age_p95_ms": semantic_percentiles["completion_age_ms"]["p95"],
|
||||
"fresh_semantic_coverage": round(fresh_coverage, 6),
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
# A completed rejected qualification is still immutable diagnostic evidence.
|
||||
# The manifest state, not the process exit code, carries acceptance.
|
||||
return 0
|
||||
|
||||
|
||||
def _assert_disk_floor(path: Path, floor: int, frame: int) -> None:
|
||||
if floor < 0:
|
||||
raise RuntimeError("LAB E9 disk floor is invalid")
|
||||
free = shutil.disk_usage(path).free
|
||||
if floor and free < floor:
|
||||
raise RuntimeError(f"LAB E9 crossed the D-backed free-space floor at frame {frame}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _arguments()
|
||||
if args.command == "preflight":
|
||||
return _preflight(args)
|
||||
return _run(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,698 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate review-only E2 prelabels from the exact E0 model generations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import resource
|
||||
import time
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from run_recorded_perception_epoch import (
|
||||
INSTANCE_MASK_THRESHOLD,
|
||||
INSTANCE_SCORE_THRESHOLD,
|
||||
SEMANTIC_MODEL_ID,
|
||||
SEMANTIC_REVISION,
|
||||
_canonical_json,
|
||||
_GpuTelemetry,
|
||||
_model_files,
|
||||
_palette,
|
||||
_percentiles,
|
||||
_read_object,
|
||||
_sha256,
|
||||
_valid_sha256,
|
||||
_write_json,
|
||||
_write_png,
|
||||
)
|
||||
|
||||
PACK_SCHEMA = "missioncore.perception-evaluation-pack/v1"
|
||||
PACK_IDENTITY_SCHEMA = "missioncore.perception-evaluation-pack-identity/v1"
|
||||
CONTRACT_SCHEMA = "missioncore.perception-annotation-contract/v1"
|
||||
VALID_FOV_SCHEMA = "missioncore.k1-valid-fov-mask/v1"
|
||||
RESULT_SCHEMA = "missioncore.perception-evaluation-prelabels/v1"
|
||||
RESULT_IDENTITY_SCHEMA = "missioncore.perception-evaluation-prelabels-identity/v1"
|
||||
REPORT_SCHEMA = "missioncore.perception-evaluation-prelabels-report/v1"
|
||||
FRAME_SCHEMA = "missioncore.perception-evaluation-prelabel-frame/v1"
|
||||
MAPPING_PROFILE = "e0-coco-ade-to-e2-taxonomy/v1"
|
||||
PREVIEW_COUNT = 16
|
||||
|
||||
TARGET_NAMES = {
|
||||
0: "outside_valid_fov",
|
||||
1: "person",
|
||||
2: "bicycle",
|
||||
3: "motorcycle",
|
||||
4: "car",
|
||||
5: "heavy_vehicle",
|
||||
6: "building_structure",
|
||||
7: "paved_road",
|
||||
8: "sidewalk_curb",
|
||||
9: "ground_dirt",
|
||||
10: "grass_low_vegetation",
|
||||
11: "tree_woody_vegetation",
|
||||
12: "sky",
|
||||
13: "static_obstacle",
|
||||
14: "animal",
|
||||
15: "other_background",
|
||||
}
|
||||
|
||||
INSTANCE_MAPPING = {
|
||||
"person": 1,
|
||||
"bicycle": 2,
|
||||
"motorcycle": 3,
|
||||
"car": 4,
|
||||
"bus": 5,
|
||||
"truck": 5,
|
||||
"cat": 14,
|
||||
"dog": 14,
|
||||
"horse": 14,
|
||||
"sheep": 14,
|
||||
"cow": 14,
|
||||
"elephant": 14,
|
||||
"bear": 14,
|
||||
"zebra": 14,
|
||||
"giraffe": 14,
|
||||
"bench": 13,
|
||||
"fire hydrant": 13,
|
||||
"stop sign": 13,
|
||||
"parking meter": 13,
|
||||
"chair": 13,
|
||||
"potted plant": 13,
|
||||
}
|
||||
|
||||
SEMANTIC_MAPPING = {
|
||||
"person": 1,
|
||||
"bicycle": 2,
|
||||
"minibike": 3,
|
||||
"car": 4,
|
||||
"bus": 5,
|
||||
"truck": 5,
|
||||
"van": 5,
|
||||
"building": 6,
|
||||
"house": 6,
|
||||
"skyscraper": 6,
|
||||
"wall": 6,
|
||||
"door": 6,
|
||||
"windowpane": 6,
|
||||
"hovel": 6,
|
||||
"awning": 6,
|
||||
"road": 7,
|
||||
"runway": 7,
|
||||
"path": 7,
|
||||
"sidewalk": 8,
|
||||
"stairs": 8,
|
||||
"stairway": 8,
|
||||
"step": 8,
|
||||
"earth": 9,
|
||||
"sand": 9,
|
||||
"field": 9,
|
||||
"land": 9,
|
||||
"dirt track": 9,
|
||||
"grass": 10,
|
||||
"plant": 10,
|
||||
"flower": 10,
|
||||
"tree": 11,
|
||||
"palm": 11,
|
||||
"sky": 12,
|
||||
"animal": 14,
|
||||
}
|
||||
|
||||
|
||||
def _arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--evaluation-pack", type=Path, required=True)
|
||||
parser.add_argument("--valid-fov-root", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--cache", type=Path, required=True)
|
||||
parser.add_argument("--telemetry-interval-seconds", type=float, default=1.0)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _safe_artifact(root: Path, encoded: object) -> Path:
|
||||
if not isinstance(encoded, str):
|
||||
raise RuntimeError("evaluation artifact path is not a string")
|
||||
relative = PurePosixPath(encoded)
|
||||
if relative.is_absolute() or ".." in relative.parts or not relative.parts:
|
||||
raise RuntimeError("evaluation artifact path is unsafe")
|
||||
path = root.joinpath(*relative.parts).resolve(strict=True)
|
||||
if not path.is_file() or not path.is_relative_to(root):
|
||||
raise RuntimeError("evaluation artifact escapes the pack")
|
||||
return path
|
||||
|
||||
|
||||
def _validate_pack(root: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
resolved = root.resolve(strict=True)
|
||||
manifest = _read_object(resolved / "manifest.json")
|
||||
identity = manifest.get("identity")
|
||||
identity_sha256 = manifest.get("identity_sha256")
|
||||
if (
|
||||
manifest.get("schema_version") != PACK_SCHEMA
|
||||
or not isinstance(identity, dict)
|
||||
or identity.get("schema_version") != PACK_IDENTITY_SCHEMA
|
||||
or not _valid_sha256(identity_sha256)
|
||||
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
||||
or manifest.get("generation_id") != f"evaluation-pack-{identity_sha256}"
|
||||
or identity.get("preprocessing_profile") != "fixed-valid-fov-fill/v1"
|
||||
):
|
||||
raise RuntimeError("evaluation pack identity is invalid")
|
||||
artifacts = manifest.get("artifacts")
|
||||
if not isinstance(artifacts, list) or not artifacts:
|
||||
raise RuntimeError("evaluation pack artifacts are unavailable")
|
||||
seen: set[str] = set()
|
||||
for artifact in artifacts:
|
||||
if not isinstance(artifact, dict) or not isinstance(artifact.get("path"), str):
|
||||
raise RuntimeError("evaluation artifact descriptor is invalid")
|
||||
encoded = str(artifact["path"])
|
||||
if encoded in seen:
|
||||
raise RuntimeError("evaluation artifact descriptor is duplicated")
|
||||
path = _safe_artifact(resolved, encoded)
|
||||
if (
|
||||
artifact.get("byte_length") != path.stat().st_size
|
||||
or not _valid_sha256(artifact.get("sha256"))
|
||||
or _sha256(path) != artifact["sha256"]
|
||||
):
|
||||
raise RuntimeError(f"evaluation artifact changed: {encoded}")
|
||||
seen.add(encoded)
|
||||
contract = _read_object(resolved / "annotation-contract.json")
|
||||
if (
|
||||
contract.get("schema_version") != CONTRACT_SCHEMA
|
||||
or contract.get("evaluation_identity_sha256") != identity_sha256
|
||||
or [row.get("id") for row in contract.get("categories", [])]
|
||||
!= list(range(1, 16))
|
||||
):
|
||||
raise RuntimeError("evaluation annotation contract is invalid")
|
||||
frames = identity.get("frames")
|
||||
if not isinstance(frames, list) or len(frames) != 64:
|
||||
raise RuntimeError("evaluation frame set is invalid")
|
||||
for expected_image_id, frame in enumerate(frames, start=1):
|
||||
if not isinstance(frame, dict):
|
||||
raise RuntimeError("evaluation frame descriptor is invalid")
|
||||
image_id = frame.get("image_id")
|
||||
frame_index = frame.get("frame_index")
|
||||
if (
|
||||
image_id != expected_image_id
|
||||
or not isinstance(frame_index, int)
|
||||
or isinstance(frame_index, bool)
|
||||
):
|
||||
raise RuntimeError("evaluation frame order changed")
|
||||
name = f"image-{image_id:03d}-frame-{frame_index:06d}.png"
|
||||
expected_paths = {
|
||||
f"images/raw/{name}",
|
||||
f"images/valid-fov-fill/{name}",
|
||||
}
|
||||
if not expected_paths <= seen:
|
||||
raise RuntimeError("evaluation frame artifacts are incomplete")
|
||||
return manifest, frames
|
||||
|
||||
|
||||
def _load_valid_fov(root: Path, pack: dict[str, Any]) -> Any:
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
resolved = root.resolve(strict=True)
|
||||
manifest = _read_object(resolved / "manifest.json")
|
||||
identity = manifest.get("identity")
|
||||
identity_sha256 = manifest.get("identity_sha256")
|
||||
artifact = manifest.get("artifact")
|
||||
pack_identity = pack["identity"]
|
||||
if (
|
||||
manifest.get("schema_version") != VALID_FOV_SCHEMA
|
||||
or not isinstance(identity, dict)
|
||||
or not _valid_sha256(identity_sha256)
|
||||
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
||||
or manifest.get("generation_id") != f"valid-fov-mask-{identity_sha256}"
|
||||
or manifest.get("generation_id") != pack_identity["valid_fov_generation_id"]
|
||||
or identity.get("calibration_sha256") != pack_identity["calibration_sha256"]
|
||||
or identity.get("calibration_slot") != pack_identity["calibration_slot"]
|
||||
or identity.get("source_id") != pack_identity["source_id"]
|
||||
or identity.get("admitted_resolution") != [800, 600]
|
||||
or not isinstance(artifact, dict)
|
||||
or artifact.get("path") != "mask.png"
|
||||
or not _valid_sha256(artifact.get("sha256"))
|
||||
):
|
||||
raise RuntimeError("valid-FOV binding is invalid")
|
||||
mask_path = (resolved / "mask.png").resolve(strict=True)
|
||||
if mask_path.parent != resolved or _sha256(mask_path) != artifact["sha256"]:
|
||||
raise RuntimeError("valid-FOV artifact changed")
|
||||
with Image.open(mask_path) as opened:
|
||||
mask = np.asarray(opened, dtype=np.uint8)
|
||||
if mask.shape != (600, 800) or not np.isin(mask, (0, 255)).all():
|
||||
raise RuntimeError("valid-FOV mask pixels are invalid")
|
||||
return mask > 0
|
||||
|
||||
|
||||
def _resource_snapshot() -> dict[str, float]:
|
||||
usage = resource.getrusage(resource.RUSAGE_SELF)
|
||||
result = {
|
||||
"user_cpu_seconds": float(usage.ru_utime),
|
||||
"system_cpu_seconds": float(usage.ru_stime),
|
||||
"minor_page_faults": float(usage.ru_minflt),
|
||||
"major_page_faults": float(usage.ru_majflt),
|
||||
"voluntary_context_switches": float(usage.ru_nvcsw),
|
||||
"involuntary_context_switches": float(usage.ru_nivcsw),
|
||||
}
|
||||
try:
|
||||
for line in Path("/proc/self/io").read_text(encoding="ascii").splitlines():
|
||||
name, value = line.split(":", 1)
|
||||
if name in {"read_bytes", "write_bytes", "rchar", "wchar"}:
|
||||
result[f"proc_io_{name}"] = float(value.strip())
|
||||
except (OSError, UnicodeDecodeError, ValueError):
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def _resource_delta(before: dict[str, float], after: dict[str, float]) -> dict[str, float]:
|
||||
return {
|
||||
name: round(value - before[name], 6)
|
||||
for name, value in after.items()
|
||||
if name in before
|
||||
}
|
||||
|
||||
|
||||
def _semantic_target_map(labels: dict[int, str]) -> Any:
|
||||
import numpy as np
|
||||
|
||||
mapping = np.full(256, 15, dtype=np.uint8)
|
||||
for source_id, name in labels.items():
|
||||
mapping[source_id] = SEMANTIC_MAPPING.get(name.strip().lower(), 15)
|
||||
return mapping
|
||||
|
||||
|
||||
def _preview_indices(count: int) -> set[int]:
|
||||
admitted = min(PREVIEW_COUNT, count)
|
||||
if admitted == 1:
|
||||
return {0}
|
||||
return {
|
||||
(position * (count - 1) + (admitted - 1) // 2) // (admitted - 1)
|
||||
for position in range(admitted)
|
||||
}
|
||||
|
||||
|
||||
def _preview(image: Any, semantic: Any, instance_map: Any, instances: list[dict[str, Any]]) -> Any:
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
colors = np.zeros_like(image)
|
||||
for category_id in range(1, 16):
|
||||
colors[semantic == category_id] = _palette(category_id)
|
||||
blended = np.clip(
|
||||
image.astype(np.float32) * 0.52 + colors.astype(np.float32) * 0.48,
|
||||
0,
|
||||
255,
|
||||
).astype(np.uint8)
|
||||
canvas = Image.fromarray(blended)
|
||||
draw = ImageDraw.Draw(canvas)
|
||||
for instance in instances:
|
||||
instance_id = int(instance["instance_id"])
|
||||
color = _palette(500 + instance_id)
|
||||
x1, y1, x2, y2 = (int(round(value)) for value in instance["box_xyxy"])
|
||||
draw.rectangle((x1, y1, x2, y2), outline=color, width=2)
|
||||
label = f"{instance['draft_category']} {float(instance['score']):.0%}"
|
||||
draw.text((x1, max(0, y1 - 13)), label, fill=color)
|
||||
if not np.any(instance_map == instance_id):
|
||||
raise RuntimeError("preview instance lost its mask")
|
||||
return np.asarray(canvas, dtype=np.uint8)
|
||||
|
||||
|
||||
def _artifact(path: Path, root: Path) -> dict[str, Any]:
|
||||
return {
|
||||
"path": path.relative_to(root).as_posix(),
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
|
||||
|
||||
def _run(args: argparse.Namespace) -> int:
|
||||
pack_root = args.evaluation_pack.resolve(strict=True)
|
||||
output_root = args.output.resolve()
|
||||
cache_root = args.cache.resolve(strict=True)
|
||||
if output_root.exists():
|
||||
raise RuntimeError("prelabel output must be absent")
|
||||
pack, frames = _validate_pack(pack_root)
|
||||
valid_mask = _load_valid_fov(args.valid_fov_root, pack)
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as functional
|
||||
import torchvision
|
||||
import transformers
|
||||
from huggingface_hub import snapshot_download
|
||||
from PIL import Image
|
||||
from torchvision.models.detection import (
|
||||
MaskRCNN_ResNet50_FPN_V2_Weights,
|
||||
maskrcnn_resnet50_fpn_v2,
|
||||
)
|
||||
from transformers import AutoImageProcessor, BeitForSemanticSegmentation
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("CUDA is required for E2 prelabels")
|
||||
output_root.mkdir(mode=0o700, parents=True, exist_ok=False)
|
||||
instance_root = output_root / "instance-prelabels"
|
||||
semantic_root = output_root / "semantic-prelabels"
|
||||
preview_root = output_root / "previews"
|
||||
instance_root.mkdir(mode=0o700)
|
||||
semantic_root.mkdir(mode=0o700)
|
||||
preview_root.mkdir(mode=0o700)
|
||||
telemetry_path = output_root / "gpu-telemetry.jsonl"
|
||||
metadata_path = output_root / "frames.jsonl"
|
||||
device = torch.device("cuda:0")
|
||||
frame_paths = []
|
||||
for frame in frames:
|
||||
name = f"image-{int(frame['image_id']):03d}-frame-{int(frame['frame_index']):06d}.png"
|
||||
frame_paths.append(pack_root / "images" / "valid-fov-fill" / name)
|
||||
|
||||
resource_before = _resource_snapshot()
|
||||
wall_started = time.perf_counter()
|
||||
instance_latencies: list[float] = []
|
||||
semantic_latencies: list[float] = []
|
||||
instances_by_frame: list[list[dict[str, Any]]] = []
|
||||
instance_counts: Counter[str] = Counter()
|
||||
ignored_instance_counts: Counter[str] = Counter()
|
||||
preview_indices = _preview_indices(len(frames))
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
|
||||
with telemetry_path.open("x", encoding="utf-8", newline="\n") as telemetry_stream:
|
||||
with _GpuTelemetry(telemetry_stream, args.telemetry_interval_seconds) as telemetry:
|
||||
instance_weights = MaskRCNN_ResNet50_FPN_V2_Weights.DEFAULT
|
||||
instance_model = maskrcnn_resnet50_fpn_v2(weights=instance_weights).to(device).eval()
|
||||
instance_labels = list(instance_weights.meta["categories"])
|
||||
with Image.open(frame_paths[0]) as opened:
|
||||
warm_image = opened.convert("RGB")
|
||||
with torch.inference_mode():
|
||||
warm_tensor = instance_weights.transforms()(warm_image).to(device)
|
||||
_ = instance_model([warm_tensor])[0]
|
||||
torch.cuda.synchronize()
|
||||
del warm_tensor
|
||||
|
||||
with torch.inference_mode():
|
||||
for order, path in enumerate(frame_paths, start=1):
|
||||
with Image.open(path) as opened:
|
||||
image = np.asarray(opened.convert("RGB"), dtype=np.uint8)
|
||||
tensor = instance_weights.transforms()(Image.fromarray(image)).to(device)
|
||||
torch.cuda.synchronize()
|
||||
started = time.perf_counter()
|
||||
prediction = instance_model([tensor])[0]
|
||||
torch.cuda.synchronize()
|
||||
instance_latencies.append((time.perf_counter() - started) * 1000.0)
|
||||
scores = prediction["scores"].detach().cpu().numpy()
|
||||
keep = np.flatnonzero(scores >= INSTANCE_SCORE_THRESHOLD)
|
||||
index_map = np.zeros(valid_mask.shape, dtype=np.uint16)
|
||||
instances: list[dict[str, Any]] = []
|
||||
for output_index in keep:
|
||||
source_id = int(prediction["labels"][output_index].item())
|
||||
source_name = instance_labels[source_id].strip().lower()
|
||||
target_id = INSTANCE_MAPPING.get(source_name)
|
||||
if target_id is None:
|
||||
ignored_instance_counts[source_name] += 1
|
||||
continue
|
||||
mask = prediction["masks"][output_index, 0].detach().cpu().numpy()
|
||||
mask = np.logical_and(mask >= INSTANCE_MASK_THRESHOLD, valid_mask)
|
||||
admitted_mask = np.logical_and(mask, index_map == 0)
|
||||
mask_pixels = int(np.count_nonzero(admitted_mask))
|
||||
if mask_pixels < 8:
|
||||
continue
|
||||
instance_id = len(instances) + 1
|
||||
index_map[admitted_mask] = instance_id
|
||||
box = [
|
||||
round(float(value), 6)
|
||||
for value in prediction["boxes"][output_index]
|
||||
.detach()
|
||||
.cpu()
|
||||
.tolist()
|
||||
]
|
||||
instance = {
|
||||
"instance_id": instance_id,
|
||||
"draft_category_id": target_id,
|
||||
"draft_category": TARGET_NAMES[target_id],
|
||||
"source_model_category_id": source_id,
|
||||
"source_model_category": source_name,
|
||||
"score": round(float(scores[output_index]), 9),
|
||||
"box_xyxy": box,
|
||||
"mask_pixels": mask_pixels,
|
||||
"review_state": "unreviewed-model-draft",
|
||||
}
|
||||
instances.append(instance)
|
||||
instance_counts[TARGET_NAMES[target_id]] += 1
|
||||
_write_png(instance_root / f"image-{order:03d}.png", index_map)
|
||||
instances_by_frame.append(instances)
|
||||
del tensor, prediction
|
||||
if order % 16 == 0 or order == len(frames):
|
||||
print(
|
||||
json.dumps(
|
||||
{"phase": "instance", "processed": order, "total": len(frames)},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
checkpoint = Path(instance_weights.url).name
|
||||
checkpoint_path = Path(torch.hub.get_dir()) / "checkpoints" / checkpoint
|
||||
del instance_model
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
semantic_snapshot = Path(
|
||||
snapshot_download(
|
||||
repo_id=SEMANTIC_MODEL_ID,
|
||||
revision=SEMANTIC_REVISION,
|
||||
cache_dir=cache_root / "huggingface",
|
||||
allow_patterns=("config.json", "preprocessor_config.json", "pytorch_model.bin"),
|
||||
)
|
||||
)
|
||||
processor = AutoImageProcessor.from_pretrained(
|
||||
semantic_snapshot,
|
||||
local_files_only=True,
|
||||
use_fast=False,
|
||||
)
|
||||
semantic_model, loading = BeitForSemanticSegmentation.from_pretrained(
|
||||
semantic_snapshot,
|
||||
local_files_only=True,
|
||||
output_loading_info=True,
|
||||
)
|
||||
load_problems = {
|
||||
name: loading.get(name, [])
|
||||
for name in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs")
|
||||
if loading.get(name)
|
||||
}
|
||||
if load_problems:
|
||||
raise RuntimeError(
|
||||
"semantic checkpoint did not load exactly: " + json.dumps(load_problems)
|
||||
)
|
||||
semantic_model = semantic_model.to(device).eval()
|
||||
semantic_labels = {
|
||||
int(key): str(value) for key, value in semantic_model.config.id2label.items()
|
||||
}
|
||||
semantic_mapping = _semantic_target_map(semantic_labels)
|
||||
with Image.open(frame_paths[0]) as opened:
|
||||
warm_image = opened.convert("RGB")
|
||||
warm_inputs = processor(images=warm_image, return_tensors="pt")
|
||||
warm_inputs = {name: value.to(device) for name, value in warm_inputs.items()}
|
||||
with torch.inference_mode():
|
||||
_ = semantic_model(**warm_inputs).logits
|
||||
torch.cuda.synchronize()
|
||||
del warm_inputs
|
||||
|
||||
target_pixels: Counter[str] = Counter()
|
||||
with metadata_path.open("x", encoding="utf-8", newline="\n") as metadata_stream:
|
||||
with torch.inference_mode():
|
||||
for order, (frame, path, instances) in enumerate(
|
||||
zip(frames, frame_paths, instances_by_frame, strict=True),
|
||||
start=1,
|
||||
):
|
||||
with Image.open(path) as opened:
|
||||
image = np.asarray(opened.convert("RGB"), dtype=np.uint8)
|
||||
inputs = processor(images=Image.fromarray(image), return_tensors="pt")
|
||||
inputs = {name: value.to(device) for name, value in inputs.items()}
|
||||
torch.cuda.synchronize()
|
||||
started = time.perf_counter()
|
||||
logits = semantic_model(**inputs).logits
|
||||
resized = functional.interpolate(
|
||||
logits,
|
||||
size=image.shape[:2],
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
)
|
||||
source_semantic = (
|
||||
resized.argmax(dim=1)[0].detach().cpu().numpy().astype(np.uint8)
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
semantic_latencies.append((time.perf_counter() - started) * 1000.0)
|
||||
semantic = semantic_mapping[source_semantic]
|
||||
semantic = semantic.copy()
|
||||
semantic[~valid_mask] = 0
|
||||
instance_map = np.asarray(
|
||||
Image.open(instance_root / f"image-{order:03d}.png"),
|
||||
dtype=np.uint16,
|
||||
)
|
||||
for instance in instances:
|
||||
semantic[instance_map == int(instance["instance_id"])] = int(
|
||||
instance["draft_category_id"]
|
||||
)
|
||||
labels, counts = np.unique(semantic[valid_mask], return_counts=True)
|
||||
for label_id, count in zip(labels, counts, strict=True):
|
||||
target_pixels[TARGET_NAMES[int(label_id)]] += int(count)
|
||||
_write_png(semantic_root / f"image-{order:03d}.png", semantic)
|
||||
if order - 1 in preview_indices:
|
||||
_write_png(
|
||||
preview_root / f"image-{order:03d}.png",
|
||||
_preview(image, semantic, instance_map, instances),
|
||||
)
|
||||
metadata_stream.write(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": FRAME_SCHEMA,
|
||||
"image_id": frame["image_id"],
|
||||
"frame_index": frame["frame_index"],
|
||||
"session_seconds": frame["session_seconds"],
|
||||
"role": frame["role"],
|
||||
"group_id": frame["group_id"],
|
||||
"instances": instances,
|
||||
"review_state": "unreviewed-model-draft",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
del inputs, logits, resized
|
||||
if order % 16 == 0 or order == len(frames):
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"phase": "semantic",
|
||||
"processed": order,
|
||||
"total": len(frames),
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
metadata_stream.flush()
|
||||
os.fsync(metadata_stream.fileno())
|
||||
model_files = _model_files(semantic_snapshot, checkpoint_path)
|
||||
del semantic_model
|
||||
torch.cuda.empty_cache()
|
||||
telemetry_summary = telemetry.summary()
|
||||
|
||||
wall_seconds = time.perf_counter() - wall_started
|
||||
resource_after = _resource_snapshot()
|
||||
identity = {
|
||||
"schema_version": RESULT_IDENTITY_SCHEMA,
|
||||
"evaluation_pack_id": pack["generation_id"],
|
||||
"evaluation_identity_sha256": pack["identity_sha256"],
|
||||
"valid_fov_generation_id": pack["identity"]["valid_fov_generation_id"],
|
||||
"pipeline": "maskrcnn-beit-e2-prelabels/v1",
|
||||
"mapping_profile": MAPPING_PROFILE,
|
||||
# String keys make the content identity stable across a JSON round trip.
|
||||
"target_categories": {str(category_id): name for category_id, name in TARGET_NAMES.items()},
|
||||
"instance_mapping": INSTANCE_MAPPING,
|
||||
"semantic_mapping": SEMANTIC_MAPPING,
|
||||
"instance_score_threshold": INSTANCE_SCORE_THRESHOLD,
|
||||
"instance_mask_threshold": INSTANCE_MASK_THRESHOLD,
|
||||
"runner_sha256": _sha256(Path(__file__).resolve(strict=True)),
|
||||
"models": {
|
||||
"instance": {
|
||||
"id": "torchvision/maskrcnn_resnet50_fpn_v2",
|
||||
"weights": str(instance_weights),
|
||||
},
|
||||
"semantic": {"id": SEMANTIC_MODEL_ID, "revision": SEMANTIC_REVISION},
|
||||
"files": model_files,
|
||||
},
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
result_id = f"evaluation-prelabels-{identity_sha256}"
|
||||
report = {
|
||||
"schema_version": REPORT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"state": "completed-unreviewed-model-draft",
|
||||
"identity": identity,
|
||||
"warning": (
|
||||
"These outputs are model-assisted prelabels, not ground truth and not an accuracy "
|
||||
"measurement. Every accepted label requires human review."
|
||||
),
|
||||
"metrics": {
|
||||
"frames": len(frames),
|
||||
"instances": sum(instance_counts.values()),
|
||||
"instances_by_target": dict(instance_counts.most_common()),
|
||||
"ignored_instances_by_source": dict(ignored_instance_counts.most_common()),
|
||||
"semantic_pixels_by_target_inside_valid_fov": dict(target_pixels.most_common()),
|
||||
"instance_forward_ms": _percentiles(instance_latencies),
|
||||
"semantic_forward_ms": _percentiles(semantic_latencies),
|
||||
"combined_forward_mean_ms": round(
|
||||
_percentiles(instance_latencies)["mean"]
|
||||
+ _percentiles(semantic_latencies)["mean"],
|
||||
6,
|
||||
),
|
||||
"wall_seconds": round(wall_seconds, 6),
|
||||
},
|
||||
"runtime": {
|
||||
"hostname": platform.node(),
|
||||
"platform": platform.platform(),
|
||||
"python": platform.python_version(),
|
||||
"torch": torch.__version__,
|
||||
"torchvision": torchvision.__version__,
|
||||
"transformers": transformers.__version__,
|
||||
"cuda_runtime": torch.version.cuda,
|
||||
"gpu": torch.cuda.get_device_name(),
|
||||
"gpu_compute_capability": list(torch.cuda.get_device_capability()),
|
||||
"cuda_peak_memory_allocated_mib": round(
|
||||
torch.cuda.max_memory_allocated() / 2**20,
|
||||
3,
|
||||
),
|
||||
"cuda_peak_memory_reserved_mib": round(
|
||||
torch.cuda.max_memory_reserved() / 2**20,
|
||||
3,
|
||||
),
|
||||
"process_peak_rss_mib": round(
|
||||
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024,
|
||||
3,
|
||||
),
|
||||
"resource_delta": _resource_delta(resource_before, resource_after),
|
||||
"gpu_telemetry": telemetry_summary,
|
||||
},
|
||||
}
|
||||
_write_json(output_root / "run-report.json", report)
|
||||
artifacts = [
|
||||
_artifact(path, output_root)
|
||||
for path in sorted(output_root.rglob("*"))
|
||||
if path.is_file()
|
||||
]
|
||||
result = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"review_state": "unreviewed-model-draft",
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
_write_json(output_root / "result.json", result)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"state": "completed-unreviewed-model-draft",
|
||||
"result_id": result_id,
|
||||
"frames": len(frames),
|
||||
"wall_seconds": round(wall_seconds, 6),
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
return _run(_arguments())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,875 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Benchmark E1 valid-FOV preprocessing variants on one sealed frame slice."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import time
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from run_recorded_perception_epoch import (
|
||||
INSTANCE_MASK_THRESHOLD,
|
||||
INSTANCE_SCORE_THRESHOLD,
|
||||
SEMANTIC_MODEL_ID,
|
||||
SEMANTIC_REVISION,
|
||||
_canonical_json,
|
||||
_GpuTelemetry,
|
||||
_instance_overlay,
|
||||
_model_files,
|
||||
_percentiles,
|
||||
_read_object,
|
||||
_read_timeline,
|
||||
_semantic_overlay,
|
||||
_sha256,
|
||||
_valid_sha256,
|
||||
_validate_job,
|
||||
_write_json,
|
||||
_write_png,
|
||||
)
|
||||
|
||||
QUALIFICATION_SCHEMA = "missioncore.recorded-qualification-slice/v1"
|
||||
QUALIFICATION_IDENTITY_SCHEMA = "missioncore.recorded-qualification-slice-identity/v1"
|
||||
VALID_FOV_SCHEMA = "missioncore.k1-valid-fov-mask/v1"
|
||||
VALID_FOV_IDENTITY_SCHEMA = "missioncore.k1-valid-fov-mask-identity/v1"
|
||||
REPORT_SCHEMA = "missioncore.perception-preprocessing-qualification-report/v1"
|
||||
RESULT_SCHEMA = "missioncore.perception-preprocessing-qualification-result/v1"
|
||||
RESULT_IDENTITY_SCHEMA = "missioncore.perception-preprocessing-qualification-identity/v1"
|
||||
VARIANTS = ("baseline", "valid-fov-fill", "valid-fov-crop")
|
||||
PREVIEW_FRAME_COUNT = 12
|
||||
|
||||
|
||||
def _arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--job", type=Path, required=True)
|
||||
parser.add_argument("--frames", type=Path, required=True)
|
||||
parser.add_argument("--timeline", type=Path, required=True)
|
||||
parser.add_argument("--qualification", type=Path, required=True)
|
||||
parser.add_argument("--valid-fov-root", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--cache", type=Path, required=True)
|
||||
parser.add_argument("--calibration-sha256", required=True)
|
||||
parser.add_argument("--calibration-slot", required=True)
|
||||
parser.add_argument("--telemetry-interval-seconds", type=float, default=1.0)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _load_qualification(path: Path, job: dict[str, Any]) -> tuple[dict[str, Any], list[int]]:
|
||||
manifest = _read_object(path.resolve(strict=True))
|
||||
identity = manifest.get("identity")
|
||||
identity_sha256 = manifest.get("identity_sha256")
|
||||
input_document = job["input"]
|
||||
if (
|
||||
manifest.get("schema_version") != QUALIFICATION_SCHEMA
|
||||
or not isinstance(identity, dict)
|
||||
or identity.get("schema_version") != QUALIFICATION_IDENTITY_SCHEMA
|
||||
or not _valid_sha256(identity_sha256)
|
||||
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
||||
or manifest.get("generation_id") != f"qualification-slice-{identity_sha256}"
|
||||
or identity.get("job_id") != job["job_id"]
|
||||
or identity.get("input_sha256") != job["input_sha256"]
|
||||
or identity.get("source_id") != input_document["source_id"]
|
||||
or identity.get("codec_epoch") != input_document["codec_epoch"]
|
||||
or identity.get("source_frame_count") != input_document["segment_count"]
|
||||
or identity.get("policy") != "uniform-frame-index-full-epoch/v1"
|
||||
):
|
||||
raise RuntimeError("qualification slice identity is invalid")
|
||||
rows = manifest.get("frames")
|
||||
selected = identity.get("selected_frames")
|
||||
if not isinstance(rows, list) or not isinstance(selected, list) or len(rows) != len(selected):
|
||||
raise RuntimeError("qualification slice frame set is invalid")
|
||||
indices: list[int] = []
|
||||
previous = -1
|
||||
for row, selected_row in zip(rows, selected, strict=True):
|
||||
if not isinstance(row, dict) or not isinstance(selected_row, dict):
|
||||
raise RuntimeError("qualification frame descriptor is invalid")
|
||||
frame_index = row.get("frame_index")
|
||||
sequence = row.get("sequence")
|
||||
digest = row.get("segment_sha256")
|
||||
if (
|
||||
not isinstance(frame_index, int)
|
||||
or isinstance(frame_index, bool)
|
||||
or not previous < frame_index < int(input_document["segment_count"])
|
||||
or sequence != frame_index + 1
|
||||
or not _valid_sha256(digest)
|
||||
or selected_row
|
||||
!= {
|
||||
"frame_index": frame_index,
|
||||
"sequence": sequence,
|
||||
"segment_sha256": digest,
|
||||
}
|
||||
):
|
||||
raise RuntimeError("qualification frame binding is invalid")
|
||||
indices.append(frame_index)
|
||||
previous = frame_index
|
||||
if not indices:
|
||||
raise RuntimeError("qualification slice is empty")
|
||||
return manifest, indices
|
||||
|
||||
|
||||
def _load_valid_fov(
|
||||
root: Path,
|
||||
*,
|
||||
job: dict[str, Any],
|
||||
calibration_sha256: str,
|
||||
calibration_slot: str,
|
||||
) -> tuple[dict[str, Any], Any, tuple[int, int, int, int]]:
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
resolved = root.resolve(strict=True)
|
||||
manifest = _read_object(resolved / "manifest.json")
|
||||
identity = manifest.get("identity")
|
||||
identity_sha256 = manifest.get("identity_sha256")
|
||||
artifact = manifest.get("artifact")
|
||||
geometry = manifest.get("geometry")
|
||||
if (
|
||||
manifest.get("schema_version") != VALID_FOV_SCHEMA
|
||||
or not isinstance(identity, dict)
|
||||
or identity.get("schema_version") != VALID_FOV_IDENTITY_SCHEMA
|
||||
or not _valid_sha256(identity_sha256)
|
||||
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
||||
or manifest.get("generation_id") != f"valid-fov-mask-{identity_sha256}"
|
||||
or identity.get("calibration_sha256") != calibration_sha256
|
||||
or identity.get("calibration_slot") != calibration_slot
|
||||
or identity.get("source_id") != job["input"]["source_id"]
|
||||
or identity.get("admitted_resolution") != [800, 600]
|
||||
or not isinstance(artifact, dict)
|
||||
or artifact.get("path") != "mask.png"
|
||||
or artifact.get("media_type") != "image/png"
|
||||
or not _valid_sha256(artifact.get("sha256"))
|
||||
or not isinstance(geometry, dict)
|
||||
):
|
||||
raise RuntimeError("valid-FOV identity is invalid")
|
||||
mask_path = (resolved / "mask.png").resolve(strict=True)
|
||||
if mask_path.parent != resolved or _sha256(mask_path) != artifact["sha256"]:
|
||||
raise RuntimeError("valid-FOV mask artifact changed")
|
||||
with Image.open(mask_path) as opened:
|
||||
if opened.mode != "L" or opened.size != (800, 600):
|
||||
raise RuntimeError("valid-FOV mask format changed")
|
||||
mask = np.asarray(opened, dtype=np.uint8)
|
||||
if not np.isin(mask, (0, 255)).all():
|
||||
raise RuntimeError("valid-FOV mask is not binary")
|
||||
crop_value = geometry.get("crop_xyxy_exclusive")
|
||||
if (
|
||||
not isinstance(crop_value, list)
|
||||
or len(crop_value) != 4
|
||||
or not all(isinstance(value, int) and not isinstance(value, bool) for value in crop_value)
|
||||
):
|
||||
raise RuntimeError("valid-FOV crop is invalid")
|
||||
crop = tuple(int(value) for value in crop_value)
|
||||
left, top, right, bottom = crop
|
||||
if not 0 <= left < right <= 800 or not 0 <= top < bottom <= 600:
|
||||
raise RuntimeError("valid-FOV crop escapes the image")
|
||||
if int(np.count_nonzero(mask)) != geometry.get("valid_pixel_count"):
|
||||
raise RuntimeError("valid-FOV pixel count changed")
|
||||
return manifest, mask > 0, crop
|
||||
|
||||
|
||||
def _variant_image(
|
||||
image: Any,
|
||||
valid_mask: Any,
|
||||
crop: tuple[int, int, int, int],
|
||||
variant: str,
|
||||
) -> Any:
|
||||
import numpy as np
|
||||
|
||||
if variant == "baseline":
|
||||
return image
|
||||
masked = np.where(valid_mask[..., None], image, 0).astype(np.uint8, copy=False)
|
||||
if variant == "valid-fov-fill":
|
||||
return masked
|
||||
if variant == "valid-fov-crop":
|
||||
left, top, right, bottom = crop
|
||||
return masked[top:bottom, left:right]
|
||||
raise RuntimeError(f"unknown preprocessing variant: {variant}")
|
||||
|
||||
|
||||
def _full_mask(
|
||||
local_mask: Any,
|
||||
*,
|
||||
variant: str,
|
||||
valid_mask: Any,
|
||||
crop: tuple[int, int, int, int],
|
||||
) -> Any:
|
||||
import numpy as np
|
||||
|
||||
if variant != "valid-fov-crop":
|
||||
result = np.asarray(local_mask, dtype=bool)
|
||||
else:
|
||||
result = np.zeros(valid_mask.shape, dtype=bool)
|
||||
left, top, right, bottom = crop
|
||||
if local_mask.shape != (bottom - top, right - left):
|
||||
raise RuntimeError("cropped instance mask shape changed")
|
||||
result[top:bottom, left:right] = local_mask
|
||||
if variant != "baseline":
|
||||
result = np.logical_and(result, valid_mask)
|
||||
return result
|
||||
|
||||
|
||||
def _full_box(
|
||||
box_xyxy: list[float],
|
||||
*,
|
||||
variant: str,
|
||||
crop: tuple[int, int, int, int],
|
||||
) -> list[float]:
|
||||
if variant == "valid-fov-crop":
|
||||
left, top, _right, _bottom = crop
|
||||
return [
|
||||
box_xyxy[0] + left,
|
||||
box_xyxy[1] + top,
|
||||
box_xyxy[2] + left,
|
||||
box_xyxy[3] + top,
|
||||
]
|
||||
return box_xyxy
|
||||
|
||||
|
||||
def _full_semantic(
|
||||
local: Any,
|
||||
*,
|
||||
variant: str,
|
||||
valid_mask: Any,
|
||||
crop: tuple[int, int, int, int],
|
||||
) -> Any:
|
||||
import numpy as np
|
||||
|
||||
if variant != "valid-fov-crop":
|
||||
result = np.asarray(local, dtype=np.uint8)
|
||||
else:
|
||||
result = np.full(valid_mask.shape, 255, dtype=np.uint8)
|
||||
left, top, right, bottom = crop
|
||||
if local.shape != (bottom - top, right - left):
|
||||
raise RuntimeError("cropped semantic mask shape changed")
|
||||
result[top:bottom, left:right] = local
|
||||
if variant != "baseline":
|
||||
result = result.copy()
|
||||
result[~valid_mask] = 255
|
||||
return result
|
||||
|
||||
|
||||
def _resource_snapshot() -> dict[str, float]:
|
||||
usage = resource.getrusage(resource.RUSAGE_SELF)
|
||||
result = {
|
||||
"user_cpu_seconds": float(usage.ru_utime),
|
||||
"system_cpu_seconds": float(usage.ru_stime),
|
||||
"minor_page_faults": float(usage.ru_minflt),
|
||||
"major_page_faults": float(usage.ru_majflt),
|
||||
"voluntary_context_switches": float(usage.ru_nvcsw),
|
||||
"involuntary_context_switches": float(usage.ru_nivcsw),
|
||||
}
|
||||
try:
|
||||
for line in Path("/proc/self/io").read_text(encoding="ascii").splitlines():
|
||||
name, value = line.split(":", 1)
|
||||
if name in {"read_bytes", "write_bytes", "rchar", "wchar"}:
|
||||
result[f"proc_io_{name}"] = float(value.strip())
|
||||
except (OSError, UnicodeDecodeError, ValueError):
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def _resource_delta(before: dict[str, float], after: dict[str, float]) -> dict[str, float]:
|
||||
return {
|
||||
name: round(value - before.get(name, value), 6)
|
||||
for name, value in after.items()
|
||||
if name in before
|
||||
}
|
||||
|
||||
|
||||
def _metric_state() -> dict[str, Any]:
|
||||
return {
|
||||
"instance_source_decode_ms": [],
|
||||
"instance_preprocess_ms": [],
|
||||
"instance_host_to_device_ms": [],
|
||||
"instance_forward_ms": [],
|
||||
"instance_postprocess_ms": [],
|
||||
"semantic_source_decode_ms": [],
|
||||
"semantic_preprocess_ms": [],
|
||||
"semantic_host_to_device_ms": [],
|
||||
"semantic_forward_ms": [],
|
||||
"semantic_postprocess_ms": [],
|
||||
"instances": 0,
|
||||
"instance_scores": [],
|
||||
"instance_labels": Counter(),
|
||||
"huge_masks_over_half_valid_fov": 0,
|
||||
"huge_boxes_over_half_full_frame": 0,
|
||||
"predicted_mask_pixels": 0,
|
||||
"predicted_mask_pixels_outside_valid_fov": 0,
|
||||
"frames_with_outside_valid_fov_instance_pixels": 0,
|
||||
"semantic_labels": Counter(),
|
||||
"semantic_disagreement_with_baseline": [],
|
||||
}
|
||||
|
||||
|
||||
def _latency_document(state: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
branch: {
|
||||
name: _percentiles(
|
||||
[float(value) for value in state[f"{branch}_{name}"]]
|
||||
)
|
||||
for name in (
|
||||
"source_decode_ms",
|
||||
"preprocess_ms",
|
||||
"host_to_device_ms",
|
||||
"forward_ms",
|
||||
"postprocess_ms",
|
||||
)
|
||||
}
|
||||
for branch in ("instance", "semantic")
|
||||
}
|
||||
|
||||
|
||||
def _metric_document(state: dict[str, Any], frame_count: int) -> dict[str, Any]:
|
||||
scores = [float(value) for value in state["instance_scores"]]
|
||||
outside = int(state["predicted_mask_pixels_outside_valid_fov"])
|
||||
total = int(state["predicted_mask_pixels"])
|
||||
disagreements = [float(value) for value in state["semantic_disagreement_with_baseline"]]
|
||||
return {
|
||||
"frames": frame_count,
|
||||
"latency_ms": _latency_document(state),
|
||||
"instances": int(state["instances"]),
|
||||
"instances_per_frame": round(int(state["instances"]) / frame_count, 6),
|
||||
"instance_score": _percentiles(scores),
|
||||
"instance_labels": dict(state["instance_labels"].most_common()),
|
||||
"huge_masks_over_half_valid_fov": int(state["huge_masks_over_half_valid_fov"]),
|
||||
"huge_boxes_over_half_full_frame": int(state["huge_boxes_over_half_full_frame"]),
|
||||
"predicted_mask_pixels": total,
|
||||
"predicted_mask_pixels_outside_valid_fov": outside,
|
||||
"predicted_mask_outside_fraction": round(outside / total, 9) if total else 0.0,
|
||||
"frames_with_outside_valid_fov_instance_pixels": int(
|
||||
state["frames_with_outside_valid_fov_instance_pixels"]
|
||||
),
|
||||
"semantic_labels_inside_valid_fov": dict(state["semantic_labels"].most_common()),
|
||||
"semantic_disagreement_with_baseline_inside_valid_fov": _percentiles(disagreements),
|
||||
}
|
||||
|
||||
|
||||
def _preview_indices(indices: list[int]) -> set[int]:
|
||||
admitted = min(PREVIEW_FRAME_COUNT, len(indices))
|
||||
if admitted == 1:
|
||||
return {indices[len(indices) // 2]}
|
||||
positions = {
|
||||
(position * (len(indices) - 1) + (admitted - 1) // 2) // (admitted - 1)
|
||||
for position in range(admitted)
|
||||
}
|
||||
return {indices[position] for position in positions}
|
||||
|
||||
|
||||
def _run(args: argparse.Namespace) -> int:
|
||||
if not _valid_sha256(args.calibration_sha256):
|
||||
raise RuntimeError("calibration SHA-256 is invalid")
|
||||
job_root = args.job.resolve(strict=True)
|
||||
frames_root = args.frames.resolve(strict=True)
|
||||
timeline_path = args.timeline.resolve(strict=True)
|
||||
output_root = args.output.resolve()
|
||||
cache_root = args.cache.resolve(strict=True)
|
||||
if output_root.exists() or not frames_root.is_dir():
|
||||
raise RuntimeError("output must be absent and frames must be a directory")
|
||||
job = _validate_job(job_root)
|
||||
input_document = job["input"]
|
||||
frame_count = int(input_document["segment_count"])
|
||||
timeline = input_document["timeline"]
|
||||
timestamps = _read_timeline(
|
||||
timeline_path,
|
||||
frame_count,
|
||||
float(timeline["start_seconds"]),
|
||||
float(timeline["end_seconds"]),
|
||||
)
|
||||
qualification, selected_indices = _load_qualification(args.qualification, job)
|
||||
valid_fov, valid_mask, crop = _load_valid_fov(
|
||||
args.valid_fov_root,
|
||||
job=job,
|
||||
calibration_sha256=args.calibration_sha256,
|
||||
calibration_slot=args.calibration_slot,
|
||||
)
|
||||
frame_paths = [frames_root / f"frame-{index + 1:06d}.png" for index in selected_indices]
|
||||
if not all(path.is_file() for path in frame_paths):
|
||||
raise RuntimeError("qualification decoded frame set is incomplete")
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as functional
|
||||
import torchvision
|
||||
import transformers
|
||||
from huggingface_hub import snapshot_download
|
||||
from PIL import Image
|
||||
from torchvision.models.detection import (
|
||||
MaskRCNN_ResNet50_FPN_V2_Weights,
|
||||
maskrcnn_resnet50_fpn_v2,
|
||||
)
|
||||
from transformers import AutoImageProcessor, BeitForSemanticSegmentation
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("CUDA is required for preprocessing qualification")
|
||||
device = torch.device("cuda:0")
|
||||
output_root.mkdir(mode=0o700, parents=True, exist_ok=False)
|
||||
previews_root = output_root / "previews"
|
||||
previews_root.mkdir(mode=0o700)
|
||||
for variant in VARIANTS:
|
||||
(previews_root / variant).mkdir(mode=0o700)
|
||||
telemetry_path = output_root / "gpu-telemetry.jsonl"
|
||||
metrics = {variant: _metric_state() for variant in VARIANTS}
|
||||
preview_frame_indices = _preview_indices(selected_indices)
|
||||
preview_instances: dict[tuple[str, int], tuple[Any, list[dict[str, Any]]]] = {}
|
||||
valid_pixel_count = int(np.count_nonzero(valid_mask))
|
||||
total_pixel_count = int(valid_mask.size)
|
||||
resource_before = _resource_snapshot()
|
||||
wall_started = time.perf_counter()
|
||||
|
||||
with telemetry_path.open("x", encoding="utf-8", newline="\n") as telemetry_stream:
|
||||
with _GpuTelemetry(telemetry_stream, args.telemetry_interval_seconds) as telemetry:
|
||||
instance_weights = MaskRCNN_ResNet50_FPN_V2_Weights.DEFAULT
|
||||
instance_model = maskrcnn_resnet50_fpn_v2(weights=instance_weights).to(device).eval()
|
||||
instance_labels = list(instance_weights.meta["categories"])
|
||||
with Image.open(frame_paths[0]) as opened:
|
||||
warm_image = opened.convert("RGB")
|
||||
with torch.inference_mode():
|
||||
warm_tensor = instance_weights.transforms()(warm_image).to(device)
|
||||
_ = instance_model([warm_tensor])[0]
|
||||
torch.cuda.synchronize()
|
||||
del warm_tensor
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
|
||||
with torch.inference_mode():
|
||||
for order, (frame_index, path) in enumerate(
|
||||
zip(selected_indices, frame_paths, strict=True),
|
||||
start=1,
|
||||
):
|
||||
decode_started = time.perf_counter()
|
||||
with Image.open(path) as opened:
|
||||
image = np.asarray(opened.convert("RGB"), dtype=np.uint8)
|
||||
decode_ms = (time.perf_counter() - decode_started) * 1000.0
|
||||
for variant in VARIANTS:
|
||||
state = metrics[variant]
|
||||
state["instance_source_decode_ms"].append(decode_ms)
|
||||
preprocess_started = time.perf_counter()
|
||||
variant_image = _variant_image(image, valid_mask, crop, variant)
|
||||
tensor = instance_weights.transforms()(Image.fromarray(variant_image))
|
||||
state["instance_preprocess_ms"].append(
|
||||
(time.perf_counter() - preprocess_started) * 1000.0
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
transfer_started = time.perf_counter()
|
||||
tensor = tensor.to(device)
|
||||
torch.cuda.synchronize()
|
||||
state["instance_host_to_device_ms"].append(
|
||||
(time.perf_counter() - transfer_started) * 1000.0
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
forward_started = time.perf_counter()
|
||||
prediction = instance_model([tensor])[0]
|
||||
torch.cuda.synchronize()
|
||||
state["instance_forward_ms"].append(
|
||||
(time.perf_counter() - forward_started) * 1000.0
|
||||
)
|
||||
post_started = time.perf_counter()
|
||||
scores = prediction["scores"].detach().cpu().numpy()
|
||||
keep = np.flatnonzero(scores >= INSTANCE_SCORE_THRESHOLD)
|
||||
index_map = np.zeros(valid_mask.shape, dtype=np.uint16)
|
||||
instances: list[dict[str, Any]] = []
|
||||
frame_outside = 0
|
||||
for output_index in keep:
|
||||
local_mask = (
|
||||
prediction["masks"][output_index, 0].detach().cpu().numpy()
|
||||
>= INSTANCE_MASK_THRESHOLD
|
||||
)
|
||||
full_mask = _full_mask(
|
||||
local_mask,
|
||||
variant=variant,
|
||||
valid_mask=valid_mask,
|
||||
crop=crop,
|
||||
)
|
||||
mask_pixels = int(np.count_nonzero(full_mask))
|
||||
if mask_pixels < 8:
|
||||
continue
|
||||
original_full_mask = _full_mask(
|
||||
local_mask,
|
||||
variant="baseline" if variant != "valid-fov-crop" else variant,
|
||||
valid_mask=np.ones_like(valid_mask),
|
||||
crop=crop,
|
||||
)
|
||||
outside_pixels = int(
|
||||
np.count_nonzero(np.logical_and(original_full_mask, ~valid_mask))
|
||||
)
|
||||
frame_outside += outside_pixels
|
||||
label_id = int(prediction["labels"][output_index].item())
|
||||
score = float(scores[output_index])
|
||||
box_values = (
|
||||
prediction["boxes"][output_index].detach().cpu().tolist()
|
||||
)
|
||||
box = _full_box(
|
||||
[float(value) for value in box_values],
|
||||
variant=variant,
|
||||
crop=crop,
|
||||
)
|
||||
instance_id = len(instances) + 1
|
||||
index_map[np.logical_and(full_mask, index_map == 0)] = instance_id
|
||||
instances.append(
|
||||
{
|
||||
"instance_id": instance_id,
|
||||
"class_id": label_id,
|
||||
"label": instance_labels[label_id],
|
||||
"score": round(score, 9),
|
||||
"box_xyxy": [round(value, 6) for value in box],
|
||||
"mask_pixels": mask_pixels,
|
||||
"outside_valid_fov_pixels": outside_pixels,
|
||||
}
|
||||
)
|
||||
state["instances"] += 1
|
||||
state["instance_scores"].append(score)
|
||||
state["instance_labels"][instance_labels[label_id]] += 1
|
||||
state["predicted_mask_pixels"] += int(
|
||||
np.count_nonzero(original_full_mask)
|
||||
)
|
||||
state["predicted_mask_pixels_outside_valid_fov"] += outside_pixels
|
||||
if mask_pixels / valid_pixel_count > 0.5:
|
||||
state["huge_masks_over_half_valid_fov"] += 1
|
||||
box_area = max(0.0, box[2] - box[0]) * max(0.0, box[3] - box[1])
|
||||
if box_area / total_pixel_count > 0.5:
|
||||
state["huge_boxes_over_half_full_frame"] += 1
|
||||
if frame_outside:
|
||||
state["frames_with_outside_valid_fov_instance_pixels"] += 1
|
||||
if frame_index in preview_frame_indices:
|
||||
preview_instances[(variant, frame_index)] = (index_map, instances)
|
||||
state["instance_postprocess_ms"].append(
|
||||
(time.perf_counter() - post_started) * 1000.0
|
||||
)
|
||||
del tensor, prediction
|
||||
if order % 32 == 0 or order == len(selected_indices):
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"phase": "instance",
|
||||
"qualification_frames_processed": order,
|
||||
"qualification_frames_total": len(selected_indices),
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
checkpoint = Path(instance_weights.url).name
|
||||
checkpoint_path = Path(torch.hub.get_dir()) / "checkpoints" / checkpoint
|
||||
del instance_model
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
semantic_snapshot = Path(
|
||||
snapshot_download(
|
||||
repo_id=SEMANTIC_MODEL_ID,
|
||||
revision=SEMANTIC_REVISION,
|
||||
cache_dir=cache_root / "huggingface",
|
||||
allow_patterns=(
|
||||
"config.json",
|
||||
"preprocessor_config.json",
|
||||
"pytorch_model.bin",
|
||||
),
|
||||
)
|
||||
)
|
||||
processor = AutoImageProcessor.from_pretrained(
|
||||
semantic_snapshot,
|
||||
local_files_only=True,
|
||||
use_fast=False,
|
||||
)
|
||||
semantic_model, loading = BeitForSemanticSegmentation.from_pretrained(
|
||||
semantic_snapshot,
|
||||
local_files_only=True,
|
||||
output_loading_info=True,
|
||||
)
|
||||
load_problems = {
|
||||
name: loading.get(name, [])
|
||||
for name in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs")
|
||||
if loading.get(name)
|
||||
}
|
||||
if load_problems:
|
||||
raise RuntimeError(
|
||||
"semantic checkpoint did not load exactly: " + json.dumps(load_problems)
|
||||
)
|
||||
semantic_model = semantic_model.to(device).eval()
|
||||
semantic_labels = {
|
||||
int(key): str(value) for key, value in semantic_model.config.id2label.items()
|
||||
}
|
||||
with Image.open(frame_paths[0]) as opened:
|
||||
warm_image = opened.convert("RGB")
|
||||
warm_inputs = processor(images=warm_image, return_tensors="pt")
|
||||
warm_inputs = {name: value.to(device) for name, value in warm_inputs.items()}
|
||||
with torch.inference_mode():
|
||||
_ = semantic_model(**warm_inputs).logits
|
||||
torch.cuda.synchronize()
|
||||
del warm_inputs
|
||||
|
||||
preview_sequence = {
|
||||
frame_index: sequence
|
||||
for sequence, frame_index in enumerate(sorted(preview_frame_indices), start=1)
|
||||
}
|
||||
with torch.inference_mode():
|
||||
for order, (frame_index, path) in enumerate(
|
||||
zip(selected_indices, frame_paths, strict=True),
|
||||
start=1,
|
||||
):
|
||||
decode_started = time.perf_counter()
|
||||
with Image.open(path) as opened:
|
||||
image = np.asarray(opened.convert("RGB"), dtype=np.uint8)
|
||||
decode_ms = (time.perf_counter() - decode_started) * 1000.0
|
||||
semantic_by_variant: dict[str, Any] = {}
|
||||
for variant in VARIANTS:
|
||||
state = metrics[variant]
|
||||
state["semantic_source_decode_ms"].append(decode_ms)
|
||||
preprocess_started = time.perf_counter()
|
||||
variant_image = _variant_image(image, valid_mask, crop, variant)
|
||||
inputs = processor(
|
||||
images=Image.fromarray(variant_image),
|
||||
return_tensors="pt",
|
||||
)
|
||||
state["semantic_preprocess_ms"].append(
|
||||
(time.perf_counter() - preprocess_started) * 1000.0
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
transfer_started = time.perf_counter()
|
||||
inputs = {name: value.to(device) for name, value in inputs.items()}
|
||||
torch.cuda.synchronize()
|
||||
state["semantic_host_to_device_ms"].append(
|
||||
(time.perf_counter() - transfer_started) * 1000.0
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
forward_started = time.perf_counter()
|
||||
logits = semantic_model(**inputs).logits
|
||||
resized = functional.interpolate(
|
||||
logits,
|
||||
size=variant_image.shape[:2],
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
)
|
||||
local_semantic = (
|
||||
resized.argmax(dim=1)[0].detach().cpu().numpy().astype(np.uint8)
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
state["semantic_forward_ms"].append(
|
||||
(time.perf_counter() - forward_started) * 1000.0
|
||||
)
|
||||
post_started = time.perf_counter()
|
||||
semantic = _full_semantic(
|
||||
local_semantic,
|
||||
variant=variant,
|
||||
valid_mask=valid_mask,
|
||||
crop=crop,
|
||||
)
|
||||
semantic_by_variant[variant] = semantic
|
||||
labels, counts = np.unique(semantic[valid_mask], return_counts=True)
|
||||
for label_id, count in zip(labels, counts, strict=True):
|
||||
state["semantic_labels"][
|
||||
semantic_labels.get(int(label_id), f"class-{int(label_id)}")
|
||||
] += int(count)
|
||||
if frame_index in preview_frame_indices:
|
||||
display_image = _variant_image(image, valid_mask, crop, variant)
|
||||
if variant == "valid-fov-crop":
|
||||
full_display = np.zeros_like(image)
|
||||
left, top, right, bottom = crop
|
||||
full_display[top:bottom, left:right] = display_image
|
||||
display_image = full_display
|
||||
display_semantic = semantic.copy()
|
||||
display_semantic[display_semantic == 255] = 0
|
||||
semantic_overlay, _classes = _semantic_overlay(
|
||||
display_image,
|
||||
display_semantic,
|
||||
semantic_labels,
|
||||
)
|
||||
instance_map, instances = preview_instances[(variant, frame_index)]
|
||||
overlay = _instance_overlay(
|
||||
semantic_overlay,
|
||||
instance_map,
|
||||
instances,
|
||||
)
|
||||
sequence = preview_sequence[frame_index]
|
||||
_write_png(
|
||||
previews_root / variant / f"frame-{sequence:03d}.png",
|
||||
overlay,
|
||||
)
|
||||
state["semantic_postprocess_ms"].append(
|
||||
(time.perf_counter() - post_started) * 1000.0
|
||||
)
|
||||
del inputs, logits, resized
|
||||
baseline_semantic = semantic_by_variant["baseline"]
|
||||
for variant in VARIANTS:
|
||||
selected_semantic = semantic_by_variant[variant][valid_mask]
|
||||
disagreement = float(
|
||||
np.mean(selected_semantic != baseline_semantic[valid_mask])
|
||||
)
|
||||
metrics[variant]["semantic_disagreement_with_baseline"].append(
|
||||
disagreement
|
||||
)
|
||||
if order % 32 == 0 or order == len(selected_indices):
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"phase": "semantic",
|
||||
"qualification_frames_processed": order,
|
||||
"qualification_frames_total": len(selected_indices),
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
del semantic_model
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
wall_seconds = time.perf_counter() - wall_started
|
||||
resource_after = _resource_snapshot()
|
||||
model_files = _model_files(semantic_snapshot, checkpoint_path)
|
||||
metric_documents = {
|
||||
variant: _metric_document(metrics[variant], len(selected_indices))
|
||||
for variant in VARIANTS
|
||||
}
|
||||
identity = {
|
||||
"schema_version": RESULT_IDENTITY_SCHEMA,
|
||||
"job_id": job["job_id"],
|
||||
"input_sha256": job["input_sha256"],
|
||||
"qualification_generation_id": qualification["generation_id"],
|
||||
"qualification_identity_sha256": qualification["identity_sha256"],
|
||||
"valid_fov_generation_id": valid_fov["generation_id"],
|
||||
"valid_fov_identity_sha256": valid_fov["identity_sha256"],
|
||||
"calibration_sha256": args.calibration_sha256,
|
||||
"calibration_slot": args.calibration_slot,
|
||||
"configuration": {
|
||||
"pipeline": "recorded-preprocessing-ab-mask-crop/v1",
|
||||
"variants": list(VARIANTS),
|
||||
"precision": "fp32",
|
||||
"batch_size": 1,
|
||||
"execution": "same-model-load-interleaved-variants-per-frame",
|
||||
"instance_score_threshold": INSTANCE_SCORE_THRESHOLD,
|
||||
"instance_mask_threshold": INSTANCE_MASK_THRESHOLD,
|
||||
"runner_sha256": _sha256(Path(__file__).resolve(strict=True)),
|
||||
"preview_frame_count": len(preview_frame_indices),
|
||||
},
|
||||
"models": {
|
||||
"instance": {
|
||||
"id": "torchvision/maskrcnn_resnet50_fpn_v2",
|
||||
"weights": str(instance_weights),
|
||||
},
|
||||
"semantic": {
|
||||
"id": SEMANTIC_MODEL_ID,
|
||||
"revision": SEMANTIC_REVISION,
|
||||
"checkpoint_load": "exact-no-missing-unexpected-or-mismatched-keys",
|
||||
},
|
||||
"files": model_files,
|
||||
},
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
result_id = f"qualification-result-{identity_sha256}"
|
||||
report = {
|
||||
"schema_version": REPORT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"state": "completed",
|
||||
"identity": identity,
|
||||
"input": {
|
||||
"job_id": job["job_id"],
|
||||
"input_sha256": job["input_sha256"],
|
||||
"session_id": input_document["session_id"],
|
||||
"source_id": input_document["source_id"],
|
||||
"codec_epoch": input_document["codec_epoch"],
|
||||
"source_frame_count": frame_count,
|
||||
"qualification_frame_count": len(selected_indices),
|
||||
"selected_frame_indices": selected_indices,
|
||||
"selected_session_seconds": [timestamps[index] for index in selected_indices],
|
||||
},
|
||||
"valid_fov": {
|
||||
"generation_id": valid_fov["generation_id"],
|
||||
"calibration_sha256": args.calibration_sha256,
|
||||
"calibration_slot": args.calibration_slot,
|
||||
"geometry": valid_fov["geometry"],
|
||||
},
|
||||
"metrics": metric_documents,
|
||||
"runtime": {
|
||||
"wall_seconds": round(wall_seconds, 6),
|
||||
"variant_model_evaluations": len(selected_indices) * len(VARIANTS) * 2,
|
||||
"gpu": torch.cuda.get_device_name(),
|
||||
"torch": torch.__version__,
|
||||
"torchvision": torchvision.__version__,
|
||||
"transformers": transformers.__version__,
|
||||
"cuda_runtime": torch.version.cuda,
|
||||
"cuda_peak_memory_allocated_mib": round(
|
||||
torch.cuda.max_memory_allocated() / 2**20,
|
||||
3,
|
||||
),
|
||||
"cuda_peak_memory_reserved_mib": round(
|
||||
torch.cuda.max_memory_reserved() / 2**20,
|
||||
3,
|
||||
),
|
||||
"process_peak_rss_mib": round(
|
||||
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024,
|
||||
3,
|
||||
),
|
||||
"resource_delta": _resource_delta(resource_before, resource_after),
|
||||
"system_load_average": [round(value, 6) for value in os.getloadavg()],
|
||||
"gpu_telemetry": telemetry.summary(),
|
||||
},
|
||||
"quality_status": {
|
||||
"ground_truth": "absent",
|
||||
"decision_scope": "preprocessing proxy comparison only",
|
||||
"not_accepted": [
|
||||
"2D mIoU/AP",
|
||||
"3D geometry",
|
||||
"distance accuracy",
|
||||
"tracking",
|
||||
"safety",
|
||||
],
|
||||
},
|
||||
}
|
||||
_write_json(output_root / "run-report.json", report)
|
||||
preview_artifacts = []
|
||||
for path in sorted(previews_root.glob("*/*.png")):
|
||||
preview_artifacts.append(
|
||||
{
|
||||
"path": path.relative_to(output_root).as_posix(),
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
)
|
||||
result = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"artifacts": {
|
||||
"run_report": {
|
||||
"path": "run-report.json",
|
||||
"byte_length": (output_root / "run-report.json").stat().st_size,
|
||||
"sha256": _sha256(output_root / "run-report.json"),
|
||||
},
|
||||
"gpu_telemetry": {
|
||||
"path": "gpu-telemetry.jsonl",
|
||||
"byte_length": telemetry_path.stat().st_size,
|
||||
"sha256": _sha256(telemetry_path),
|
||||
},
|
||||
"previews": preview_artifacts,
|
||||
},
|
||||
}
|
||||
_write_json(output_root / "result.json", result)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"state": "completed",
|
||||
"result_id": result_id,
|
||||
"qualification_frames": len(selected_indices),
|
||||
"wall_seconds": round(wall_seconds, 6),
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
return _run(_arguments())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,824 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run and seal a complete recorded panoptic-perception camera epoch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import resource
|
||||
import statistics
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, TextIO
|
||||
|
||||
INSTANCE_SCORE_THRESHOLD = 0.5
|
||||
INSTANCE_MASK_THRESHOLD = 0.5
|
||||
SEMANTIC_ALPHA = 0.46
|
||||
INSTANCE_ALPHA = 0.58
|
||||
SEMANTIC_MODEL_ID = "microsoft/beit-base-finetuned-ade-640-640"
|
||||
SEMANTIC_REVISION = "a8b6f5ef4acb2ea55d882989deaa02d39401e2b2"
|
||||
RESULT_SCHEMA = "missioncore.recorded-perception-result/v2"
|
||||
REPORT_SCHEMA = "missioncore.perception-run-report/v1"
|
||||
FRAME_SCHEMA = "missioncore.panoptic-frame/v1"
|
||||
SAFE_SHA256 = set("0123456789abcdef")
|
||||
|
||||
|
||||
def _arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
preflight = subparsers.add_parser("preflight")
|
||||
preflight.add_argument("--job", type=Path, required=True)
|
||||
preflight.add_argument("--cache", type=Path, required=True)
|
||||
|
||||
run = subparsers.add_parser("run")
|
||||
run.add_argument("--job", type=Path, required=True)
|
||||
run.add_argument("--frames", type=Path, required=True)
|
||||
run.add_argument("--timeline", type=Path, required=True)
|
||||
run.add_argument("--output", type=Path, required=True)
|
||||
run.add_argument("--cache", type=Path, required=True)
|
||||
run.add_argument("--calibration-sha256", required=True)
|
||||
run.add_argument("--calibration-slot", required=True)
|
||||
run.add_argument("--telemetry-interval-seconds", type=float, default=1.0)
|
||||
|
||||
finalize = subparsers.add_parser("finalize")
|
||||
finalize.add_argument("--output", type=Path, required=True)
|
||||
finalize.add_argument("--video", type=Path, required=True)
|
||||
finalize.add_argument("--masks", type=Path, required=True)
|
||||
finalize.add_argument("--extract-seconds", type=float, required=True)
|
||||
finalize.add_argument("--encode-seconds", type=float, required=True)
|
||||
finalize.add_argument("--wall-seconds", type=float, required=True)
|
||||
finalize.add_argument("--encoder", required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _read_object(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise RuntimeError(f"{path.name} is not a JSON object")
|
||||
return value
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
with temporary.open("x", encoding="utf-8", newline="\n") as stream:
|
||||
json.dump(value, stream, ensure_ascii=False, sort_keys=True, indent=2, allow_nan=False)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def _valid_sha256(value: object) -> bool:
|
||||
return isinstance(value, str) and len(value) == 64 and set(value) <= SAFE_SHA256
|
||||
|
||||
|
||||
def _safe_job_path(root: Path, encoded: object) -> Path:
|
||||
if not isinstance(encoded, str):
|
||||
raise RuntimeError("job artifact path is not a string")
|
||||
relative = PurePosixPath(encoded)
|
||||
if relative.is_absolute() or ".." in relative.parts or not relative.parts:
|
||||
raise RuntimeError("job artifact path is unsafe")
|
||||
path = root.joinpath(*relative.parts).resolve(strict=True)
|
||||
if not path.is_file() or not path.is_relative_to(root):
|
||||
raise RuntimeError("job artifact escapes the job root")
|
||||
return path
|
||||
|
||||
|
||||
def _validate_job(job_root: Path) -> dict[str, Any]:
|
||||
root = job_root.resolve(strict=True)
|
||||
job = _read_object(root / "job.json")
|
||||
input_document = job.get("input")
|
||||
if (
|
||||
job.get("schema_version") != "missioncore.compute-job/v1"
|
||||
or not isinstance(input_document, dict)
|
||||
or hashlib.sha256(_canonical_json(input_document)).hexdigest() != job.get("input_sha256")
|
||||
or job.get("job_id") != f"recorded-camera-{str(job.get('input_sha256'))[:24]}"
|
||||
):
|
||||
raise RuntimeError("compute job identity is invalid")
|
||||
files = input_document.get("files")
|
||||
if not isinstance(files, list) or not files:
|
||||
raise RuntimeError("compute job has no files")
|
||||
seen: set[str] = set()
|
||||
total_bytes = 0
|
||||
for artifact in files:
|
||||
if not isinstance(artifact, dict):
|
||||
raise RuntimeError("compute job artifact descriptor is invalid")
|
||||
encoded = artifact.get("path")
|
||||
if not isinstance(encoded, str) or encoded in seen:
|
||||
raise RuntimeError("compute job artifact descriptor is duplicated")
|
||||
seen.add(encoded)
|
||||
path = _safe_job_path(root, encoded)
|
||||
byte_length = artifact.get("byte_length")
|
||||
digest = artifact.get("sha256")
|
||||
if (
|
||||
not isinstance(byte_length, int)
|
||||
or byte_length < 1
|
||||
or path.stat().st_size != byte_length
|
||||
or not _valid_sha256(digest)
|
||||
or _sha256(path) != digest
|
||||
):
|
||||
raise RuntimeError(f"compute job artifact changed: {encoded}")
|
||||
total_bytes += byte_length
|
||||
if total_bytes != input_document.get("byte_length"):
|
||||
raise RuntimeError("compute job byte length changed")
|
||||
return job
|
||||
|
||||
|
||||
def _read_timeline(path: Path, expected_count: int, start: float, end: float) -> list[float]:
|
||||
timestamps: list[float] = []
|
||||
with path.open("r", encoding="utf-8") as stream:
|
||||
for expected_index, line in enumerate(stream):
|
||||
value = json.loads(line)
|
||||
if not isinstance(value, dict) or value.get("frame_index") != expected_index:
|
||||
raise RuntimeError("decoded frame timeline index changed")
|
||||
session_seconds = value.get("session_seconds")
|
||||
if not isinstance(session_seconds, (int, float)):
|
||||
raise RuntimeError("decoded frame timestamp is invalid")
|
||||
timestamp = float(session_seconds)
|
||||
if timestamps and timestamp <= timestamps[-1]:
|
||||
raise RuntimeError("decoded frame timestamps are not increasing")
|
||||
timestamps.append(timestamp)
|
||||
if len(timestamps) != expected_count:
|
||||
raise RuntimeError("decoded frame count differs from the camera job")
|
||||
if timestamps[0] < start - 0.001 or timestamps[-1] > end + 0.001:
|
||||
raise RuntimeError("decoded frame timeline escapes the camera epoch")
|
||||
return timestamps
|
||||
|
||||
|
||||
def _palette(index: int) -> tuple[int, int, int]:
|
||||
digest = hashlib.sha256(f"mission-core-segment-{index}".encode()).digest()
|
||||
return (64 + digest[0] % 176, 64 + digest[1] % 176, 64 + digest[2] % 176)
|
||||
|
||||
|
||||
def _blend(image: Any, colors: Any, alpha: float) -> Any:
|
||||
import numpy as np
|
||||
|
||||
return np.clip(
|
||||
image.astype(np.float32) * (1.0 - alpha) + colors.astype(np.float32) * alpha,
|
||||
0,
|
||||
255,
|
||||
).astype(np.uint8)
|
||||
|
||||
|
||||
def _semantic_overlay(
|
||||
image: Any,
|
||||
index_map: Any,
|
||||
labels: dict[int, str],
|
||||
) -> tuple[Any, list[dict[str, Any]]]:
|
||||
import numpy as np
|
||||
|
||||
colors = np.zeros_like(image)
|
||||
counts = np.bincount(index_map.reshape(-1), minlength=max(labels) + 1)
|
||||
present = np.flatnonzero(counts)
|
||||
for index in present:
|
||||
colors[index_map == index] = _palette(int(index))
|
||||
total = int(index_map.size)
|
||||
classes = [
|
||||
{
|
||||
"id": int(index),
|
||||
"label": labels.get(int(index), f"class-{int(index)}"),
|
||||
"pixels": int(counts[index]),
|
||||
"fraction": round(float(counts[index]) / total, 9),
|
||||
}
|
||||
for index in sorted(present, key=lambda value: int(counts[value]), reverse=True)
|
||||
]
|
||||
return _blend(image, colors, SEMANTIC_ALPHA), classes
|
||||
|
||||
|
||||
def _instance_overlay(image: Any, index_map: Any, instances: list[dict[str, Any]]) -> Any:
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
result = image.copy()
|
||||
for item in instances:
|
||||
instance_id = int(item["instance_id"])
|
||||
mask = index_map == instance_id
|
||||
color = np.asarray(_palette(500 + instance_id), dtype=np.uint8)
|
||||
if bool(mask.any()):
|
||||
result[mask] = _blend(
|
||||
result[mask],
|
||||
np.broadcast_to(color, result[mask].shape),
|
||||
INSTANCE_ALPHA,
|
||||
)
|
||||
canvas = Image.fromarray(result)
|
||||
draw = ImageDraw.Draw(canvas)
|
||||
for item in instances:
|
||||
color = _palette(500 + int(item["instance_id"]))
|
||||
x1, y1, x2, y2 = (int(round(value)) for value in item["box_xyxy"])
|
||||
draw.rectangle((x1, y1, x2, y2), outline=color, width=2)
|
||||
label = f"{item['label']} {float(item['score']):.0%}"
|
||||
text_box = draw.textbbox((x1, max(0, y1 - 13)), label)
|
||||
draw.rectangle(text_box, fill=(7, 8, 10))
|
||||
draw.text((x1, max(0, y1 - 13)), label, fill=color)
|
||||
return np.asarray(canvas, dtype=np.uint8)
|
||||
|
||||
|
||||
def _write_png(path: Path, array: Any) -> None:
|
||||
from PIL import Image
|
||||
|
||||
Image.fromarray(array).save(path, format="PNG", optimize=False)
|
||||
|
||||
|
||||
def _percentiles(values: list[float]) -> dict[str, float]:
|
||||
if not values:
|
||||
return {"mean": 0.0, "p50": 0.0, "p95": 0.0, "max": 0.0}
|
||||
ordered = sorted(values)
|
||||
|
||||
def percentile(fraction: float) -> float:
|
||||
index = min(len(ordered) - 1, max(0, round((len(ordered) - 1) * fraction)))
|
||||
return round(ordered[index], 6)
|
||||
|
||||
return {
|
||||
"mean": round(statistics.fmean(values), 6),
|
||||
"p50": percentile(0.5),
|
||||
"p95": percentile(0.95),
|
||||
"max": round(max(values), 6),
|
||||
}
|
||||
|
||||
|
||||
class _GpuTelemetry:
|
||||
def __init__(self, stream: TextIO, interval_seconds: float) -> None:
|
||||
if not 0.25 <= interval_seconds <= 60:
|
||||
raise RuntimeError("telemetry interval is outside bounds")
|
||||
self._stream = stream
|
||||
self._interval = interval_seconds
|
||||
self._stop = threading.Event()
|
||||
self._thread = threading.Thread(target=self._run, name="gpu-telemetry", daemon=True)
|
||||
self.samples: list[dict[str, float]] = []
|
||||
|
||||
def __enter__(self) -> _GpuTelemetry:
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_: object) -> None:
|
||||
self._stop.set()
|
||||
self._thread.join(timeout=self._interval + 5)
|
||||
|
||||
def _run(self) -> None:
|
||||
query = (
|
||||
"timestamp,utilization.gpu,utilization.memory,memory.used,memory.total,"
|
||||
"temperature.gpu,power.draw"
|
||||
)
|
||||
while not self._stop.is_set():
|
||||
before = time.time()
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
[
|
||||
"nvidia-smi",
|
||||
f"--query-gpu={query}",
|
||||
"--format=csv,noheader,nounits",
|
||||
],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
values = [item.strip() for item in completed.stdout.splitlines()[0].split(",")]
|
||||
sample = {
|
||||
"epoch_seconds": round(before, 6),
|
||||
"gpu_utilization_percent": float(values[1]),
|
||||
"gpu_memory_utilization_percent": float(values[2]),
|
||||
"gpu_memory_used_mib": float(values[3]),
|
||||
"gpu_memory_total_mib": float(values[4]),
|
||||
"gpu_temperature_c": float(values[5]),
|
||||
"gpu_power_w": float(values[6]),
|
||||
}
|
||||
self.samples.append(sample)
|
||||
self._stream.write(json.dumps(sample, sort_keys=True) + "\n")
|
||||
self._stream.flush()
|
||||
except (OSError, subprocess.SubprocessError, ValueError, IndexError):
|
||||
pass
|
||||
self._stop.wait(max(0.0, self._interval - (time.time() - before)))
|
||||
|
||||
def summary(self) -> dict[str, object]:
|
||||
fields = (
|
||||
"gpu_utilization_percent",
|
||||
"gpu_memory_utilization_percent",
|
||||
"gpu_memory_used_mib",
|
||||
"gpu_temperature_c",
|
||||
"gpu_power_w",
|
||||
)
|
||||
return {
|
||||
"sample_count": len(self.samples),
|
||||
"interval_seconds": self._interval,
|
||||
**{
|
||||
field: _percentiles([float(sample[field]) for sample in self.samples])
|
||||
for field in fields
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _model_files(semantic_snapshot: Path, checkpoint_path: Path) -> list[dict[str, object]]:
|
||||
files = [
|
||||
{"name": f"beit/{path.name}", "bytes": path.stat().st_size, "sha256": _sha256(path)}
|
||||
for path in sorted(semantic_snapshot.iterdir())
|
||||
if path.is_file()
|
||||
]
|
||||
files.append(
|
||||
{
|
||||
"name": f"torchvision/{checkpoint_path.name}",
|
||||
"bytes": checkpoint_path.stat().st_size,
|
||||
"sha256": _sha256(checkpoint_path),
|
||||
}
|
||||
)
|
||||
return files
|
||||
|
||||
|
||||
def _preflight(args: argparse.Namespace) -> int:
|
||||
job = _validate_job(args.job.resolve(strict=True))
|
||||
cache_root = args.cache.resolve(strict=True)
|
||||
|
||||
import torch
|
||||
import torchvision # noqa: F401
|
||||
import transformers # noqa: F401
|
||||
from huggingface_hub import snapshot_download
|
||||
from torchvision.models.detection import MaskRCNN_ResNet50_FPN_V2_Weights
|
||||
|
||||
if not torch.cuda.is_available() or torch.cuda.device_count() < 1:
|
||||
raise RuntimeError("CUDA device 0 is unavailable")
|
||||
cuda_probe = torch.zeros(1, device="cuda:0")
|
||||
torch.cuda.synchronize()
|
||||
del cuda_probe
|
||||
checkpoint = Path(MaskRCNN_ResNet50_FPN_V2_Weights.DEFAULT.url).name
|
||||
checkpoint_path = Path(torch.hub.get_dir()) / "checkpoints" / checkpoint
|
||||
if not checkpoint_path.is_file() or checkpoint_path.stat().st_size < 1:
|
||||
raise RuntimeError("cached instance checkpoint is unavailable")
|
||||
semantic_snapshot = Path(
|
||||
snapshot_download(
|
||||
repo_id=SEMANTIC_MODEL_ID,
|
||||
revision=SEMANTIC_REVISION,
|
||||
cache_dir=cache_root / "huggingface",
|
||||
allow_patterns=("config.json", "preprocessor_config.json", "pytorch_model.bin"),
|
||||
)
|
||||
)
|
||||
required = ("config.json", "preprocessor_config.json", "pytorch_model.bin")
|
||||
if any(not (semantic_snapshot / name).is_file() for name in required):
|
||||
raise RuntimeError("cached semantic checkpoint is incomplete")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"state": "preflight-ready",
|
||||
"job_id": job["job_id"],
|
||||
"cuda_device": torch.cuda.get_device_name(),
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def _run(args: argparse.Namespace) -> int:
|
||||
if not _valid_sha256(args.calibration_sha256):
|
||||
raise RuntimeError("calibration SHA-256 is invalid")
|
||||
job_root = args.job.resolve(strict=True)
|
||||
frames_root = args.frames.resolve(strict=True)
|
||||
timeline_path = args.timeline.resolve(strict=True)
|
||||
output_root = args.output.resolve()
|
||||
cache_root = args.cache.resolve()
|
||||
if output_root.exists() or not frames_root.is_dir():
|
||||
raise RuntimeError("output must be absent and frames must be a directory")
|
||||
job = _validate_job(job_root)
|
||||
input_document = job["input"]
|
||||
timeline = input_document["timeline"]
|
||||
segment_count = int(input_document["segment_count"])
|
||||
frame_paths = [frames_root / f"frame-{index:06d}.png" for index in range(1, segment_count + 1)]
|
||||
if not all(path.is_file() for path in frame_paths):
|
||||
raise RuntimeError("decoded frame set is incomplete")
|
||||
if len(list(frames_root.glob("frame-*.png"))) != segment_count:
|
||||
raise RuntimeError("decoded frame set contains unexpected files")
|
||||
timestamps = _read_timeline(
|
||||
timeline_path,
|
||||
segment_count,
|
||||
float(timeline["start_seconds"]),
|
||||
float(timeline["end_seconds"]),
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as functional
|
||||
import torchvision
|
||||
import transformers
|
||||
from huggingface_hub import snapshot_download
|
||||
from PIL import Image
|
||||
from torchvision.models.detection import (
|
||||
MaskRCNN_ResNet50_FPN_V2_Weights,
|
||||
maskrcnn_resnet50_fpn_v2,
|
||||
)
|
||||
from transformers import AutoImageProcessor, BeitForSemanticSegmentation
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("CUDA is required for a perception epoch run")
|
||||
cuda_index = 0
|
||||
device = torch.device(f"cuda:{cuda_index}")
|
||||
cache_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
output_root.mkdir(mode=0o700, parents=True, exist_ok=False)
|
||||
instance_root = output_root / "instance-masks"
|
||||
semantic_root = output_root / "semantic-masks"
|
||||
overlay_root = output_root / "overlay-frames"
|
||||
instance_root.mkdir(mode=0o700)
|
||||
semantic_root.mkdir(mode=0o700)
|
||||
overlay_root.mkdir(mode=0o700)
|
||||
telemetry_path = output_root / "gpu-telemetry.jsonl"
|
||||
started = time.perf_counter()
|
||||
instance_latencies: list[float] = []
|
||||
semantic_latencies: list[float] = []
|
||||
instances_by_frame: list[list[dict[str, Any]]] = []
|
||||
total_instances = 0
|
||||
|
||||
with telemetry_path.open("x", encoding="utf-8", newline="\n") as telemetry_stream:
|
||||
with _GpuTelemetry(telemetry_stream, args.telemetry_interval_seconds) as telemetry:
|
||||
instance_weights = MaskRCNN_ResNet50_FPN_V2_Weights.DEFAULT
|
||||
instance_model = maskrcnn_resnet50_fpn_v2(weights=instance_weights).to(device).eval()
|
||||
instance_labels = list(instance_weights.meta["categories"])
|
||||
with torch.inference_mode():
|
||||
for frame_index, path in enumerate(frame_paths):
|
||||
with Image.open(path) as opened:
|
||||
image = np.asarray(opened.convert("RGB"), dtype=np.uint8)
|
||||
tensor = instance_weights.transforms()(Image.fromarray(image)).to(device)
|
||||
torch.cuda.synchronize()
|
||||
before = time.perf_counter()
|
||||
prediction = instance_model([tensor])[0]
|
||||
torch.cuda.synchronize()
|
||||
instance_latencies.append((time.perf_counter() - before) * 1000.0)
|
||||
scores = prediction["scores"].detach().cpu().numpy()
|
||||
keep = np.flatnonzero(scores >= INSTANCE_SCORE_THRESHOLD)
|
||||
index_map = np.zeros(image.shape[:2], dtype=np.uint16)
|
||||
instances: list[dict[str, Any]] = []
|
||||
for output_index in keep:
|
||||
mask = (
|
||||
prediction["masks"][output_index, 0].detach().cpu().numpy()
|
||||
>= INSTANCE_MASK_THRESHOLD
|
||||
)
|
||||
mask_pixels = int(mask.sum())
|
||||
if mask_pixels < 8:
|
||||
continue
|
||||
label_id = int(prediction["labels"][output_index].item())
|
||||
instance_id = len(instances) + 1
|
||||
index_map[np.logical_and(mask, index_map == 0)] = instance_id
|
||||
instances.append(
|
||||
{
|
||||
"instance_id": instance_id,
|
||||
"class_id": label_id,
|
||||
"label": instance_labels[label_id],
|
||||
"score": round(float(scores[output_index]), 9),
|
||||
"box_xyxy": [
|
||||
round(float(value), 6)
|
||||
for value in prediction["boxes"][output_index]
|
||||
.detach()
|
||||
.cpu()
|
||||
.tolist()
|
||||
],
|
||||
"mask_pixels": mask_pixels,
|
||||
}
|
||||
)
|
||||
_write_png(instance_root / f"frame-{frame_index + 1:06d}.png", index_map)
|
||||
instances_by_frame.append(instances)
|
||||
total_instances += len(instances)
|
||||
if (frame_index + 1) % 100 == 0 or frame_index + 1 == segment_count:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"phase": "instance",
|
||||
"frames_processed": frame_index + 1,
|
||||
"frames_total": segment_count,
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
checkpoint = Path(instance_weights.url).name
|
||||
checkpoint_path = Path(torch.hub.get_dir()) / "checkpoints" / checkpoint
|
||||
del instance_model
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
semantic_snapshot = Path(
|
||||
snapshot_download(
|
||||
repo_id=SEMANTIC_MODEL_ID,
|
||||
revision=SEMANTIC_REVISION,
|
||||
cache_dir=cache_root / "huggingface",
|
||||
allow_patterns=("config.json", "preprocessor_config.json", "pytorch_model.bin"),
|
||||
)
|
||||
)
|
||||
processor = AutoImageProcessor.from_pretrained(
|
||||
semantic_snapshot,
|
||||
local_files_only=True,
|
||||
use_fast=False,
|
||||
)
|
||||
semantic_model, loading = BeitForSemanticSegmentation.from_pretrained(
|
||||
semantic_snapshot,
|
||||
local_files_only=True,
|
||||
output_loading_info=True,
|
||||
)
|
||||
load_problems = {
|
||||
name: loading.get(name, [])
|
||||
for name in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs")
|
||||
if loading.get(name)
|
||||
}
|
||||
if load_problems:
|
||||
raise RuntimeError(
|
||||
"semantic checkpoint did not load exactly: "
|
||||
+ json.dumps(load_problems)
|
||||
)
|
||||
semantic_model = semantic_model.to(device).eval()
|
||||
semantic_labels = {
|
||||
int(key): str(value)
|
||||
for key, value in semantic_model.config.id2label.items()
|
||||
}
|
||||
frame_metadata = output_root / "frames.jsonl"
|
||||
with frame_metadata.open("x", encoding="utf-8", newline="\n") as metadata_stream:
|
||||
with torch.inference_mode():
|
||||
for frame_index, (path, session_seconds) in enumerate(
|
||||
zip(frame_paths, timestamps, strict=True)
|
||||
):
|
||||
with Image.open(path) as opened:
|
||||
image = np.asarray(opened.convert("RGB"), dtype=np.uint8)
|
||||
inputs = processor(images=Image.fromarray(image), return_tensors="pt")
|
||||
inputs = {name: value.to(device) for name, value in inputs.items()}
|
||||
torch.cuda.synchronize()
|
||||
before = time.perf_counter()
|
||||
logits = semantic_model(**inputs).logits
|
||||
resized = functional.interpolate(
|
||||
logits,
|
||||
size=image.shape[:2],
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
)
|
||||
semantic = resized.argmax(dim=1)[0].detach().cpu().numpy().astype(np.uint8)
|
||||
torch.cuda.synchronize()
|
||||
semantic_latencies.append((time.perf_counter() - before) * 1000.0)
|
||||
semantic_overlay, classes = _semantic_overlay(
|
||||
image,
|
||||
semantic,
|
||||
semantic_labels,
|
||||
)
|
||||
instance_map = np.asarray(
|
||||
Image.open(instance_root / f"frame-{frame_index + 1:06d}.png"),
|
||||
dtype=np.uint16,
|
||||
)
|
||||
overlay = _instance_overlay(
|
||||
semantic_overlay,
|
||||
instance_map,
|
||||
instances_by_frame[frame_index],
|
||||
)
|
||||
_write_png(semantic_root / f"frame-{frame_index + 1:06d}.png", semantic)
|
||||
_write_png(overlay_root / f"frame-{frame_index + 1:06d}.png", overlay)
|
||||
metadata_stream.write(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": FRAME_SCHEMA,
|
||||
"frame_index": frame_index,
|
||||
"sequence": frame_index + 1,
|
||||
"session_seconds": round(session_seconds, 9),
|
||||
"instances": instances_by_frame[frame_index],
|
||||
"semantic_classes": classes,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
if (frame_index + 1) % 100 == 0 or frame_index + 1 == segment_count:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"phase": "semantic",
|
||||
"frames_processed": frame_index + 1,
|
||||
"frames_total": segment_count,
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
metadata_stream.flush()
|
||||
os.fsync(metadata_stream.fileno())
|
||||
del semantic_model
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
inference_elapsed = time.perf_counter() - started
|
||||
model_files = _model_files(semantic_snapshot, checkpoint_path)
|
||||
report = {
|
||||
"schema_version": REPORT_SCHEMA,
|
||||
"run_id": f"{job['job_id']}-panoptic-v1",
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"state": "inference-complete-awaiting-publication",
|
||||
"input": {
|
||||
"job_id": job["job_id"],
|
||||
"input_sha256": job["input_sha256"],
|
||||
"session_id": input_document["session_id"],
|
||||
"source_id": input_document["source_id"],
|
||||
"codec_epoch": input_document["codec_epoch"],
|
||||
"segment_count": segment_count,
|
||||
"byte_length": input_document["byte_length"],
|
||||
"timeline_start_seconds": timeline["start_seconds"],
|
||||
"timeline_end_seconds": timeline["end_seconds"],
|
||||
"frame_timeline_sha256": _sha256(timeline_path),
|
||||
},
|
||||
"calibration": {
|
||||
"content_identity_sha256": args.calibration_sha256,
|
||||
"camera_slot": args.calibration_slot,
|
||||
},
|
||||
"configuration": {
|
||||
"pipeline": "recorded-panoptic-maskrcnn-beit/v1",
|
||||
"instance_score_threshold": INSTANCE_SCORE_THRESHOLD,
|
||||
"instance_mask_threshold": INSTANCE_MASK_THRESHOLD,
|
||||
"semantic_alpha": SEMANTIC_ALPHA,
|
||||
"instance_alpha": INSTANCE_ALPHA,
|
||||
"batch_size": 1,
|
||||
"frame_policy": "all-frames-no-sampling",
|
||||
"runner_sha256": _sha256(Path(__file__).resolve(strict=True)),
|
||||
},
|
||||
"models": {
|
||||
"instance": {
|
||||
"id": "torchvision/maskrcnn_resnet50_fpn_v2",
|
||||
"weights": str(instance_weights),
|
||||
"score_threshold": INSTANCE_SCORE_THRESHOLD,
|
||||
},
|
||||
"semantic": {
|
||||
"id": SEMANTIC_MODEL_ID,
|
||||
"revision": SEMANTIC_REVISION,
|
||||
"checkpoint_load": "exact-no-missing-unexpected-or-mismatched-keys",
|
||||
},
|
||||
"files": model_files,
|
||||
},
|
||||
"runtime": {
|
||||
"hostname": platform.node(),
|
||||
"platform": platform.platform(),
|
||||
"python": platform.python_version(),
|
||||
"torch": torch.__version__,
|
||||
"torchvision": torchvision.__version__,
|
||||
"transformers": transformers.__version__,
|
||||
"cuda_runtime": torch.version.cuda,
|
||||
"gpu": torch.cuda.get_device_name(),
|
||||
"gpu_compute_capability": list(torch.cuda.get_device_capability()),
|
||||
},
|
||||
"metrics": {
|
||||
"frames_expected": segment_count,
|
||||
"frames_processed": segment_count,
|
||||
"frames_failed": 0,
|
||||
"frames_skipped": 0,
|
||||
"instances": total_instances,
|
||||
"inference_wall_seconds": round(inference_elapsed, 6),
|
||||
"inference_frames_per_second": round(segment_count / inference_elapsed, 6),
|
||||
"instance_latency_ms": _percentiles(instance_latencies),
|
||||
"semantic_latency_ms": _percentiles(semantic_latencies),
|
||||
"cuda_peak_memory_allocated_mib": round(
|
||||
torch.cuda.max_memory_allocated() / 2**20,
|
||||
3,
|
||||
),
|
||||
"cuda_peak_memory_reserved_mib": round(
|
||||
torch.cuda.max_memory_reserved() / 2**20,
|
||||
3,
|
||||
),
|
||||
"process_peak_rss_mib": round(
|
||||
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024,
|
||||
3,
|
||||
),
|
||||
"system_load_average": [round(value, 6) for value in os.getloadavg()],
|
||||
"gpu_telemetry": telemetry.summary(),
|
||||
},
|
||||
"versions": {
|
||||
name: importlib.metadata.version(name)
|
||||
for name in ("numpy", "pillow", "torch", "torchvision", "transformers")
|
||||
},
|
||||
}
|
||||
_write_json(output_root / "run-report.partial.json", report)
|
||||
print(json.dumps({"state": "inference-complete", "frames": segment_count}, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def _artifact(
|
||||
path: Path,
|
||||
kind: str,
|
||||
media_type: str,
|
||||
schema_version: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
result: dict[str, object] = {
|
||||
"kind": kind,
|
||||
"path": path.name,
|
||||
"media_type": media_type,
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
if schema_version is not None:
|
||||
result["schema_version"] = schema_version
|
||||
return result
|
||||
|
||||
|
||||
def _finalize(args: argparse.Namespace) -> int:
|
||||
output_root = args.output.resolve(strict=True)
|
||||
video = args.video.resolve(strict=True)
|
||||
masks = args.masks.resolve(strict=True)
|
||||
if video.parent != output_root or masks.parent != output_root:
|
||||
raise RuntimeError("published artifacts must be direct result children")
|
||||
partial_path = output_root / "run-report.partial.json"
|
||||
frames_path = output_root / "frames.jsonl"
|
||||
telemetry_path = output_root / "gpu-telemetry.jsonl"
|
||||
partial = _read_object(partial_path)
|
||||
if partial.get("schema_version") != REPORT_SCHEMA:
|
||||
raise RuntimeError("partial run report is incompatible")
|
||||
input_document = partial["input"]
|
||||
configuration = partial["configuration"]
|
||||
models = partial["models"]
|
||||
identity = {
|
||||
"schema_version": "missioncore.recorded-perception-identity/v2",
|
||||
"job_id": input_document["job_id"],
|
||||
"input_sha256": input_document["input_sha256"],
|
||||
"calibration": partial["calibration"],
|
||||
"configuration": configuration,
|
||||
"models": models,
|
||||
"publication": {
|
||||
"video_encoder": args.encoder,
|
||||
"video_media_type": "video/mp4",
|
||||
"mask_archive_media_type": "application/gzip",
|
||||
},
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
result_id = f"result-{identity_sha256}"
|
||||
final_report = {
|
||||
**partial,
|
||||
"state": "published",
|
||||
"result_id": result_id,
|
||||
"publication": {
|
||||
"extract_seconds": round(args.extract_seconds, 6),
|
||||
"encode_seconds": round(args.encode_seconds, 6),
|
||||
"wall_seconds": round(args.wall_seconds, 6),
|
||||
"end_to_end_frames_per_second": round(
|
||||
int(partial["metrics"]["frames_processed"]) / args.wall_seconds,
|
||||
6,
|
||||
),
|
||||
"encoder": args.encoder,
|
||||
},
|
||||
}
|
||||
report_path = output_root / "run-report.json"
|
||||
_write_json(report_path, final_report)
|
||||
artifacts = [
|
||||
_artifact(video, "panoptic-overlay-video", "video/mp4"),
|
||||
_artifact(masks, "panoptic-mask-archive", "application/gzip"),
|
||||
_artifact(frames_path, "panoptic-frame-metadata", "application/x-ndjson", FRAME_SCHEMA),
|
||||
_artifact(telemetry_path, "worker-gpu-telemetry", "application/x-ndjson"),
|
||||
_artifact(report_path, "perception-run-report", "application/json", REPORT_SCHEMA),
|
||||
]
|
||||
result = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": final_report["created_at_utc"],
|
||||
"job_id": input_document["job_id"],
|
||||
"input_sha256": input_document["input_sha256"],
|
||||
"session_id": input_document["session_id"],
|
||||
"source_id": input_document["source_id"],
|
||||
"codec_epoch": input_document["codec_epoch"],
|
||||
"timestamp_basis": "session-time-seconds",
|
||||
"timeline_start_seconds": input_document["timeline_start_seconds"],
|
||||
"timeline_end_seconds": input_document["timeline_end_seconds"],
|
||||
"frames_processed": final_report["metrics"]["frames_processed"],
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
_write_json(output_root / "result.json", result)
|
||||
partial_path.unlink()
|
||||
print(json.dumps({"result_id": result_id, "identity_sha256": identity_sha256}, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _arguments()
|
||||
if args.command == "preflight":
|
||||
return _preflight(args)
|
||||
if args.command == "run":
|
||||
return _run(args)
|
||||
return _finalize(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,425 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a bounded, recorded-only segmentation probe on an external GPU worker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import statistics
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
MAX_INPUT_IMAGES = 16
|
||||
MAX_INPUT_BYTES = 32 * 1024 * 1024
|
||||
MAX_DIMENSION = 4096
|
||||
INSTANCE_SCORE_THRESHOLD = 0.5
|
||||
SEMANTIC_MODEL_ID = "microsoft/beit-base-finetuned-ade-640-640"
|
||||
SEMANTIC_REVISION = "a8b6f5ef4acb2ea55d882989deaa02d39401e2b2"
|
||||
RESULT_SCHEMA = "missioncore.recorded-segmentation-experiment/v1"
|
||||
|
||||
|
||||
def _arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--cache", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _palette(index: int) -> tuple[int, int, int]:
|
||||
digest = hashlib.sha256(f"mission-core-segment-{index}".encode()).digest()
|
||||
return (64 + digest[0] % 176, 64 + digest[1] % 176, 64 + digest[2] % 176)
|
||||
|
||||
|
||||
def _blend(image: Any, colors: Any, alpha: float) -> Any:
|
||||
import numpy as np
|
||||
|
||||
return np.clip(
|
||||
image.astype(np.float32) * (1.0 - alpha) + colors.astype(np.float32) * alpha,
|
||||
0,
|
||||
255,
|
||||
).astype(np.uint8)
|
||||
|
||||
|
||||
def _semantic_overlay(
|
||||
image: Any, index_map: Any, labels: dict[int, str]
|
||||
) -> tuple[Any, list[dict[str, Any]]]:
|
||||
import numpy as np
|
||||
|
||||
colors = np.zeros_like(image)
|
||||
counts = np.bincount(index_map.reshape(-1), minlength=max(labels) + 1)
|
||||
present = np.flatnonzero(counts)
|
||||
for index in present:
|
||||
colors[index_map == index] = _palette(int(index))
|
||||
total = int(index_map.size)
|
||||
classes = [
|
||||
{
|
||||
"id": int(index),
|
||||
"label": labels.get(int(index), f"class-{int(index)}"),
|
||||
"pixels": int(counts[index]),
|
||||
"fraction": round(float(counts[index]) / total, 9),
|
||||
}
|
||||
for index in sorted(present, key=lambda value: int(counts[value]), reverse=True)
|
||||
]
|
||||
return _blend(image, colors, 0.46), classes
|
||||
|
||||
|
||||
def _instance_overlay(image: Any, instances: list[dict[str, Any]], masks: list[Any]) -> Any:
|
||||
import numpy as np
|
||||
|
||||
result = image.copy()
|
||||
for index, (instance, mask) in enumerate(zip(instances, masks, strict=True), start=1):
|
||||
color = np.asarray(_palette(500 + index), dtype=np.uint8)
|
||||
result[mask] = _blend(result[mask], np.broadcast_to(color, result[mask].shape), 0.56)
|
||||
x1, y1, x2, y2 = (int(round(value)) for value in instance["box_xyxy"])
|
||||
result[max(0, y1) : min(result.shape[0], y1 + 2), max(0, x1) : min(result.shape[1], x2)] = (
|
||||
color
|
||||
)
|
||||
result[max(0, y2 - 2) : min(result.shape[0], y2), max(0, x1) : min(result.shape[1], x2)] = (
|
||||
color
|
||||
)
|
||||
result[max(0, y1) : min(result.shape[0], y2), max(0, x1) : min(result.shape[1], x1 + 2)] = (
|
||||
color
|
||||
)
|
||||
result[max(0, y1) : min(result.shape[0], y2), max(0, x2 - 2) : min(result.shape[1], x2)] = (
|
||||
color
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _write_png(path: Path, array: Any) -> None:
|
||||
from PIL import Image
|
||||
|
||||
Image.fromarray(array).save(path, format="PNG", optimize=False)
|
||||
|
||||
|
||||
def _artifact(path: Path) -> dict[str, object]:
|
||||
return {"name": path.name, "bytes": path.stat().st_size, "sha256": _sha256(path)}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _arguments()
|
||||
input_root = args.input.expanduser().resolve(strict=True)
|
||||
output_root = args.output.expanduser().resolve()
|
||||
cache_root = args.cache.expanduser().resolve()
|
||||
if not input_root.is_dir() or output_root.exists():
|
||||
raise RuntimeError("input must be a directory and output must not exist")
|
||||
images = sorted(input_root.glob("camera-*.png"))
|
||||
if not images or len(images) > MAX_INPUT_IMAGES:
|
||||
raise RuntimeError("input image count is outside the recorded probe bound")
|
||||
for path in images:
|
||||
if not path.is_file() or path.stat().st_size > MAX_INPUT_BYTES:
|
||||
raise RuntimeError("input image is missing or too large")
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as functional
|
||||
import torchvision
|
||||
import transformers
|
||||
from huggingface_hub import snapshot_download
|
||||
from PIL import Image
|
||||
from torchvision.models.detection import (
|
||||
MaskRCNN_ResNet50_FPN_V2_Weights,
|
||||
maskrcnn_resnet50_fpn_v2,
|
||||
)
|
||||
from transformers import AutoImageProcessor, BeitForSemanticSegmentation
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("CUDA is required for this worker experiment")
|
||||
device = torch.device("cuda:0")
|
||||
cache_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
staging = output_root.with_name(f".{output_root.name}.incomplete")
|
||||
staging.mkdir(mode=0o700, parents=True, exist_ok=False)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
decoded: list[tuple[Path, Any]] = []
|
||||
width: int | None = None
|
||||
height: int | None = None
|
||||
for path in images:
|
||||
with Image.open(path) as opened:
|
||||
opened.verify()
|
||||
with Image.open(path) as opened:
|
||||
rgb = opened.convert("RGB")
|
||||
if max(rgb.size) > MAX_DIMENSION:
|
||||
raise RuntimeError("input image dimension is outside the probe bound")
|
||||
if width is None:
|
||||
width, height = rgb.size
|
||||
elif rgb.size != (width, height):
|
||||
raise RuntimeError("input images do not share one physical epoch resolution")
|
||||
decoded.append((path, np.asarray(rgb, dtype=np.uint8)))
|
||||
|
||||
instance_weights = MaskRCNN_ResNet50_FPN_V2_Weights.DEFAULT
|
||||
instance_model = maskrcnn_resnet50_fpn_v2(weights=instance_weights).to(device).eval()
|
||||
instance_labels = list(instance_weights.meta["categories"])
|
||||
instance_results: dict[str, dict[str, Any]] = {}
|
||||
instance_latencies: list[float] = []
|
||||
with torch.inference_mode():
|
||||
for path, image in decoded:
|
||||
tensor = instance_weights.transforms()(Image.fromarray(image)).to(device)
|
||||
torch.cuda.synchronize()
|
||||
before = time.perf_counter()
|
||||
prediction = instance_model([tensor])[0]
|
||||
torch.cuda.synchronize()
|
||||
latency_ms = (time.perf_counter() - before) * 1000.0
|
||||
instance_latencies.append(latency_ms)
|
||||
scores = prediction["scores"].detach().cpu().numpy()
|
||||
keep = np.flatnonzero(scores >= INSTANCE_SCORE_THRESHOLD)
|
||||
masks: list[Any] = []
|
||||
instances: list[dict[str, Any]] = []
|
||||
index_map = np.zeros(image.shape[:2], dtype=np.uint16)
|
||||
for output_index in keep:
|
||||
mask = prediction["masks"][output_index, 0].detach().cpu().numpy() >= 0.5
|
||||
if int(mask.sum()) < 8:
|
||||
continue
|
||||
label_id = int(prediction["labels"][output_index].item())
|
||||
instance_id = len(instances) + 1
|
||||
index_map[np.logical_and(mask, index_map == 0)] = instance_id
|
||||
masks.append(mask)
|
||||
instances.append(
|
||||
{
|
||||
"instance_id": instance_id,
|
||||
"class_id": label_id,
|
||||
"label": instance_labels[label_id],
|
||||
"score": round(float(scores[output_index]), 9),
|
||||
"box_xyxy": [
|
||||
round(float(value), 6)
|
||||
for value in prediction["boxes"][output_index]
|
||||
.detach()
|
||||
.cpu()
|
||||
.tolist()
|
||||
],
|
||||
"mask_pixels": int(mask.sum()),
|
||||
}
|
||||
)
|
||||
stem = path.stem
|
||||
_write_png(staging / f"{stem}.instances.png", index_map)
|
||||
_write_png(
|
||||
staging / f"{stem}.instance-overlay.png",
|
||||
_instance_overlay(image, instances, masks),
|
||||
)
|
||||
instance_results[stem] = {
|
||||
"latency_ms": round(latency_ms, 6),
|
||||
"instances": instances,
|
||||
}
|
||||
del instance_model
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
semantic_snapshot = Path(
|
||||
snapshot_download(
|
||||
repo_id=SEMANTIC_MODEL_ID,
|
||||
revision=SEMANTIC_REVISION,
|
||||
cache_dir=cache_root / "huggingface",
|
||||
allow_patterns=(
|
||||
"config.json",
|
||||
"preprocessor_config.json",
|
||||
"pytorch_model.bin",
|
||||
),
|
||||
)
|
||||
)
|
||||
processor = AutoImageProcessor.from_pretrained(
|
||||
semantic_snapshot,
|
||||
local_files_only=True,
|
||||
use_fast=False,
|
||||
)
|
||||
semantic_model, semantic_loading = BeitForSemanticSegmentation.from_pretrained(
|
||||
semantic_snapshot,
|
||||
local_files_only=True,
|
||||
output_loading_info=True,
|
||||
)
|
||||
semantic_load_problems = {
|
||||
name: semantic_loading.get(name, [])
|
||||
for name in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs")
|
||||
if semantic_loading.get(name)
|
||||
}
|
||||
if semantic_load_problems:
|
||||
raise RuntimeError(
|
||||
"semantic checkpoint did not load exactly: "
|
||||
+ json.dumps(semantic_load_problems, sort_keys=True)
|
||||
)
|
||||
semantic_model = semantic_model.to(device).eval()
|
||||
semantic_labels = {
|
||||
int(key): str(value) for key, value in semantic_model.config.id2label.items()
|
||||
}
|
||||
semantic_results: dict[str, dict[str, Any]] = {}
|
||||
semantic_latencies: list[float] = []
|
||||
with torch.inference_mode():
|
||||
for path, image in decoded:
|
||||
inputs = processor(images=Image.fromarray(image), return_tensors="pt")
|
||||
inputs = {name: value.to(device) for name, value in inputs.items()}
|
||||
torch.cuda.synchronize()
|
||||
before = time.perf_counter()
|
||||
logits = semantic_model(**inputs).logits
|
||||
resized = functional.interpolate(
|
||||
logits,
|
||||
size=image.shape[:2],
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
)
|
||||
semantic = resized.argmax(dim=1)[0].detach().cpu().numpy().astype(np.uint8)
|
||||
torch.cuda.synchronize()
|
||||
latency_ms = (time.perf_counter() - before) * 1000.0
|
||||
semantic_latencies.append(latency_ms)
|
||||
overlay, classes = _semantic_overlay(image, semantic, semantic_labels)
|
||||
stem = path.stem
|
||||
_write_png(staging / f"{stem}.semantic.png", semantic)
|
||||
_write_png(staging / f"{stem}.semantic-overlay.png", overlay)
|
||||
semantic_results[stem] = {
|
||||
"latency_ms": round(latency_ms, 6),
|
||||
"classes": classes,
|
||||
}
|
||||
del semantic_model
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
rows = []
|
||||
for path, image in decoded:
|
||||
stem = path.stem
|
||||
instance_overlay = np.asarray(Image.open(staging / f"{stem}.instance-overlay.png"))
|
||||
semantic_overlay = np.asarray(Image.open(staging / f"{stem}.semantic-overlay.png"))
|
||||
rows.append(np.concatenate((image, instance_overlay, semantic_overlay), axis=1))
|
||||
_write_png(staging / "segmentation-mosaic.png", np.concatenate(rows, axis=0))
|
||||
|
||||
checkpoint = Path(instance_weights.url).name
|
||||
checkpoint_path = Path(torch.hub.get_dir()) / "checkpoints" / checkpoint
|
||||
model_files = [
|
||||
{
|
||||
"name": f"beit/{path.name}",
|
||||
"bytes": path.stat().st_size,
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
for path in sorted(semantic_snapshot.iterdir())
|
||||
if path.is_file()
|
||||
]
|
||||
model_files.append(
|
||||
{
|
||||
"name": f"torchvision/{checkpoint}",
|
||||
"bytes": checkpoint_path.stat().st_size,
|
||||
"sha256": _sha256(checkpoint_path),
|
||||
}
|
||||
)
|
||||
inputs_manifest = [
|
||||
{"name": path.name, "bytes": path.stat().st_size, "sha256": _sha256(path)}
|
||||
for path in images
|
||||
]
|
||||
identity = {
|
||||
"inputs": inputs_manifest,
|
||||
"instance_model": "torchvision/maskrcnn_resnet50_fpn_v2/default",
|
||||
"instance_threshold": INSTANCE_SCORE_THRESHOLD,
|
||||
"semantic_model": SEMANTIC_MODEL_ID,
|
||||
"semantic_revision": SEMANTIC_REVISION,
|
||||
"model_files": model_files,
|
||||
}
|
||||
manifest: dict[str, Any] = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"classification": "private-derived-recorded-perception-experiment",
|
||||
"generation_sha256": hashlib.sha256(_canonical_json(identity)).hexdigest(),
|
||||
"input": inputs_manifest,
|
||||
"models": {
|
||||
"instance": {
|
||||
"id": "torchvision/maskrcnn_resnet50_fpn_v2",
|
||||
"weights": str(instance_weights),
|
||||
"weights_url": instance_weights.url,
|
||||
"score_threshold": INSTANCE_SCORE_THRESHOLD,
|
||||
"license": "BSD-3-Clause (TorchVision code/weight distribution)",
|
||||
},
|
||||
"semantic": {
|
||||
"id": SEMANTIC_MODEL_ID,
|
||||
"revision": SEMANTIC_REVISION,
|
||||
"dataset": "ADE20K scene_parse_150",
|
||||
"license": "Apache-2.0 (model-card metadata)",
|
||||
"checkpoint_load": "exact-no-missing-unexpected-or-mismatched-keys",
|
||||
},
|
||||
"files": model_files,
|
||||
},
|
||||
"runtime": {
|
||||
"base_image": os.environ.get("MISSION_CORE_BASE_IMAGE", "unknown"),
|
||||
"python": platform.python_version(),
|
||||
"torch": torch.__version__,
|
||||
"torchvision": torchvision.__version__,
|
||||
"transformers": transformers.__version__,
|
||||
"cuda_runtime": torch.version.cuda,
|
||||
"cudnn": torch.backends.cudnn.version(),
|
||||
"gpu": torch.cuda.get_device_name(0),
|
||||
"packages": sorted(
|
||||
(
|
||||
{
|
||||
"name": distribution.metadata["Name"],
|
||||
"version": distribution.version,
|
||||
}
|
||||
for distribution in importlib.metadata.distributions()
|
||||
if distribution.metadata["Name"]
|
||||
),
|
||||
key=lambda item: str(item["name"]).lower(),
|
||||
),
|
||||
},
|
||||
"metrics": {
|
||||
"frame_count": len(decoded),
|
||||
"instance_latency_ms": {
|
||||
"mean": round(statistics.fmean(instance_latencies), 6),
|
||||
"max": round(max(instance_latencies), 6),
|
||||
},
|
||||
"semantic_latency_ms": {
|
||||
"mean": round(statistics.fmean(semantic_latencies), 6),
|
||||
"max": round(max(semantic_latencies), 6),
|
||||
},
|
||||
"elapsed_seconds": round(time.perf_counter() - started, 6),
|
||||
},
|
||||
"frames": {
|
||||
stem: {
|
||||
"instance": instance_results[stem],
|
||||
"semantic": semantic_results[stem],
|
||||
}
|
||||
for stem in sorted(instance_results)
|
||||
},
|
||||
"acceptance": {
|
||||
"recorded_segmentation_artifact": "generated",
|
||||
"quality": "operator-review-required",
|
||||
"live": "not-tested",
|
||||
"safety": "not-accepted",
|
||||
},
|
||||
}
|
||||
artifacts = sorted(staging.glob("*.png"))
|
||||
manifest["outputs"] = [_artifact(path) for path in artifacts]
|
||||
(staging / "manifest.redacted.json").write_bytes(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True).encode("utf-8")
|
||||
+ b"\n"
|
||||
)
|
||||
os.rename(staging, output_root)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
print(
|
||||
json.dumps({"output": output_root.name, "generation_sha256": manifest["generation_sha256"]})
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user