545 lines
21 KiB
PowerShell
545 lines
21 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$JobRoot,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$RunnerPath,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ProfilePath,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ValidFovRoot,
|
|
|
|
[ValidateRange(0, 1000000)]
|
|
[int]$StartFrame = 1000,
|
|
|
|
[ValidateRange(0, 1000000)]
|
|
[int]$EndFrame = 1600,
|
|
|
|
[ValidateRange(1, 1000)]
|
|
[int]$FreeGiBFloor = 360,
|
|
|
|
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
|
|
|
|
[string]$ModelRoot = "D:\NDC_MISSIONCORE\runtime\models\yolox_s",
|
|
|
|
[string]$TritonContainer = "mission-core-triton",
|
|
|
|
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$ProgressPreference = "SilentlyContinue"
|
|
|
|
function Assert-LastExitCode {
|
|
param([string]$Operation)
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "$Operation failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
function Assert-RegularFile {
|
|
param([string]$Path, [string]$Label)
|
|
$item = Get-Item -LiteralPath $Path -Force
|
|
if (-not $item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
|
return $item.FullName
|
|
}
|
|
throw "$Label is not a regular file"
|
|
}
|
|
|
|
function Assert-Directory {
|
|
param([string]$Path, [string]$Label)
|
|
$item = Get-Item -LiteralPath $Path -Force
|
|
if ($item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
|
return $item.FullName
|
|
}
|
|
throw "$Label is not a real directory"
|
|
}
|
|
|
|
function Assert-DDrivePath {
|
|
param([string]$Path, [string]$Label)
|
|
$fullPath = [IO.Path]::GetFullPath($Path)
|
|
if ([IO.Path]::GetPathRoot($fullPath).TrimEnd("\") -ine "D:") {
|
|
throw "$Label must be stored on D:"
|
|
}
|
|
return $fullPath
|
|
}
|
|
|
|
function Get-DFreeBytes {
|
|
return [int64](Get-PSDrive -Name D).Free
|
|
}
|
|
|
|
function Assert-FreeSpace {
|
|
param(
|
|
[string]$Phase,
|
|
[int64]$RequiredAdditionalBytes = 0
|
|
)
|
|
$freeBytes = Get-DFreeBytes
|
|
$floorBytes = [int64]$FreeGiBFloor * 1GB
|
|
Write-Host (
|
|
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3} REQUIRED_ADDITIONAL_GIB={4}" -f
|
|
$Phase,
|
|
$freeBytes,
|
|
[math]::Round($freeBytes / 1GB, 3),
|
|
$FreeGiBFloor,
|
|
[math]::Round($RequiredAdditionalBytes / 1GB, 3)
|
|
)
|
|
if ($freeBytes -lt ($floorBytes + $RequiredAdditionalBytes)) {
|
|
throw "D: does not have the guarded LAB E5 working-set reserve during $Phase"
|
|
}
|
|
return $freeBytes
|
|
}
|
|
|
|
function Convert-ToDockerPath {
|
|
param([string]$Path)
|
|
return $Path.Replace("\", "/")
|
|
}
|
|
|
|
function Test-TritonModelReady {
|
|
param([string]$ModelName)
|
|
try {
|
|
$response = Invoke-WebRequest `
|
|
-Uri ("http://127.0.0.1:8000/v2/models/{0}/ready" -f $ModelName) `
|
|
-Method Get `
|
|
-UseBasicParsing `
|
|
-TimeoutSec 10
|
|
return $response.StatusCode -eq 200
|
|
}
|
|
catch {
|
|
return $false
|
|
}
|
|
}
|
|
|
|
$jobDirectory = Assert-DDrivePath (
|
|
Assert-Directory (Resolve-Path -LiteralPath $JobRoot).Path "Job root"
|
|
) "Job root"
|
|
$runner = Assert-DDrivePath (
|
|
Assert-RegularFile (Resolve-Path -LiteralPath $RunnerPath).Path "LAB E5 runner"
|
|
) "LAB E5 runner"
|
|
$profile = Assert-DDrivePath (
|
|
Assert-RegularFile (Resolve-Path -LiteralPath $ProfilePath).Path "LAB E5 profile"
|
|
) "LAB E5 profile"
|
|
$validFov = Assert-DDrivePath (
|
|
Assert-Directory (Resolve-Path -LiteralPath $ValidFovRoot).Path "Valid-FOV root"
|
|
) "Valid-FOV root"
|
|
$runtime = Assert-DDrivePath (
|
|
Assert-Directory (Resolve-Path -LiteralPath $RuntimeRoot).Path "Runtime root"
|
|
) "Runtime root"
|
|
$model = Assert-DDrivePath (
|
|
Assert-Directory (Resolve-Path -LiteralPath $ModelRoot).Path "YOLOX model root"
|
|
) "YOLOX model root"
|
|
$runnerRoot = Split-Path $runner -Parent
|
|
if ((Split-Path $profile -Parent) -ne $runnerRoot) {
|
|
throw "LAB E5 runner and profile must share one read-only mount"
|
|
}
|
|
foreach ($dependency in @(
|
|
"run_recorded_perception_epoch.py",
|
|
"run_e3_rectified_segmentation.py",
|
|
"run_evaluation_prelabels.py"
|
|
)) {
|
|
$null = Assert-RegularFile (Join-Path $runnerRoot $dependency) "LAB E5 runner dependency"
|
|
}
|
|
|
|
$job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json
|
|
if ($job.schema_version -ne "missioncore.compute-job/v1" -or $job.job_id -ne (Split-Path $jobDirectory -Leaf)) {
|
|
throw "Compute job manifest is incompatible"
|
|
}
|
|
$sourceId = [string]$job.input.source_id
|
|
$epoch = [int]$job.input.codec_epoch
|
|
$fullFrameCount = [int]$job.input.segment_count
|
|
$clipFrameCount = $EndFrame - $StartFrame + 1
|
|
$timelineStart = [double]$job.input.timeline.start_seconds
|
|
if (
|
|
$sourceId -ne "sensor.camera.right" -or
|
|
$StartFrame -lt 0 -or
|
|
$EndFrame -lt $StartFrame -or
|
|
$EndFrame -ge $fullFrameCount -or
|
|
$clipFrameCount -lt 2
|
|
) {
|
|
throw "LAB E5 clip escapes the camera job"
|
|
}
|
|
Write-Output (
|
|
"PHASE=job-manifest-validated JOB={0} CLIP={1}-{2} FRAMES={3}" -f
|
|
$job.job_id, $StartFrame, $EndFrame, $clipFrameCount
|
|
)
|
|
|
|
$epochRoot = Join-Path $jobDirectory ("input\camera\{0}\epoch-{1}" -f $sourceId, $epoch)
|
|
$epochRoot = Assert-DDrivePath (Assert-Directory $epochRoot "Camera epoch") "Camera epoch"
|
|
$initPath = Assert-RegularFile (Join-Path $epochRoot "init.mp4") "Camera init"
|
|
$segmentsRoot = Assert-Directory (Join-Path $epochRoot "segments") "Camera segments"
|
|
$derivedRoot = Join-Path $runtime "derived"
|
|
$tmpRoot = Join-Path $runtime "tmp"
|
|
$pythonEnvironment = Assert-DDrivePath (
|
|
Assert-Directory (Join-Path $derivedRoot "perception-p0-env-v1") "Python environment"
|
|
) "Python environment"
|
|
$null = New-Item -ItemType Directory -Path $derivedRoot -Force
|
|
$null = New-Item -ItemType Directory -Path $tmpRoot -Force
|
|
|
|
# Conservative bound: partial reconstructed stream, decoded RGB PNG payload,
|
|
# RGB overlays, encoded result and 2 GiB of filesystem/codec overhead.
|
|
$pixelWorkingSet = [int64]$clipFrameCount * 800 * 600 * 6
|
|
$partialStreamReserve = [int64][math]::Ceiling(
|
|
([int64]$job.input.byte_length * ($EndFrame + 1) / $fullFrameCount) * 1.2
|
|
)
|
|
$workingSetReserve = [int64][math]::Ceiling($pixelWorkingSet * 1.4) + $partialStreamReserve + 2GB
|
|
$freeBytesBefore = Assert-FreeSpace "preflight" $workingSetReserve
|
|
|
|
& docker image inspect $ContainerImage *> $null
|
|
Assert-LastExitCode "Existing container image inspection"
|
|
$tritonState = docker inspect --format "{{.State.Running}}" $TritonContainer
|
|
Assert-LastExitCode "Triton container inspection"
|
|
if ($tritonState.Trim().ToLowerInvariant() -ne "true") {
|
|
throw "LAB E5 requires the existing Triton container"
|
|
}
|
|
|
|
$runnerName = Split-Path $runner -Leaf
|
|
$profileName = Split-Path $profile -Leaf
|
|
$orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$preflightArgs = @(
|
|
"run", "--rm", "--network", "none", "--read-only",
|
|
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
|
"--pids-limit", "128", "--tmpfs", "/tmp:rw,noexec,nosuid,size=256m",
|
|
"-e", "PYTHONPATH=/runner:/opt/env",
|
|
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
|
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
|
"-v", ((Convert-ToDockerPath $model) + ":/model:ro"),
|
|
"-v", ((Convert-ToDockerPath $pythonEnvironment) + ":/opt/env:ro"),
|
|
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
|
"--entrypoint", "python3",
|
|
$ContainerImage,
|
|
("/runner/{0}" -f $runnerName), "preflight",
|
|
"--job", "/job",
|
|
"--profile", ("/runner/{0}" -f $profileName),
|
|
"--valid-fov-root", "/valid-fov",
|
|
"--model-root", "/model"
|
|
)
|
|
Write-Output "PHASE=e5-preflight-start"
|
|
& docker @preflightArgs
|
|
Assert-LastExitCode "LAB E5 preflight"
|
|
Write-Output "PHASE=e5-preflight-complete"
|
|
|
|
$modelWasReady = Test-TritonModelReady "yolox_s"
|
|
$loadedByRun = $false
|
|
if (-not $modelWasReady) {
|
|
Invoke-WebRequest `
|
|
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/load" `
|
|
-Method Post `
|
|
-ContentType "application/json" `
|
|
-Body "{}" `
|
|
-UseBasicParsing `
|
|
-TimeoutSec 60 *> $null
|
|
$loadedByRun = $true
|
|
$deadline = [DateTime]::UtcNow.AddSeconds(60)
|
|
while (-not (Test-TritonModelReady "yolox_s")) {
|
|
if ([DateTime]::UtcNow -ge $deadline) {
|
|
throw "YOLOX-S did not become ready in Triton"
|
|
}
|
|
Start-Sleep -Milliseconds 500
|
|
}
|
|
}
|
|
Write-Output ("PHASE=e5-model-ready LOADED_BY_RUN={0}" -f $loadedByRun)
|
|
|
|
$runToken = [Guid]::NewGuid().ToString("N")
|
|
$workRoot = Join-Path $tmpRoot ("{0}-e5-{1}" -f $job.job_id, $runToken)
|
|
$framesRoot = Join-Path $workRoot "frames"
|
|
$streamPath = Join-Path $workRoot "camera.mp4"
|
|
$ptsPath = Join-Path $workRoot "pts.json"
|
|
$timelinePath = Join-Path $workRoot "timeline.jsonl"
|
|
$publishRoot = Join-Path $derivedRoot (".{0}-e5-{1}.publish" -f $job.job_id, $runToken)
|
|
$stagingRoot = Join-Path $publishRoot "output"
|
|
$null = New-Item -ItemType Directory -Path $framesRoot
|
|
$null = New-Item -ItemType Directory -Path $publishRoot
|
|
$totalWatch = [Diagnostics.Stopwatch]::StartNew()
|
|
$completed = $false
|
|
Write-Output "PHASE=e5-private-staging-created"
|
|
|
|
try {
|
|
$stream = [IO.File]::Open(
|
|
$streamPath,
|
|
[IO.FileMode]::CreateNew,
|
|
[IO.FileAccess]::Write,
|
|
[IO.FileShare]::None
|
|
)
|
|
try {
|
|
$input = [IO.File]::OpenRead($initPath)
|
|
try {
|
|
$input.CopyTo($stream)
|
|
}
|
|
finally {
|
|
$input.Dispose()
|
|
}
|
|
for ($sequence = 1; $sequence -le ($EndFrame + 1); $sequence++) {
|
|
$path = Assert-RegularFile (
|
|
Join-Path $segmentsRoot ("{0}.m4s" -f $sequence)
|
|
) "Camera segment"
|
|
$input = [IO.File]::OpenRead($path)
|
|
try {
|
|
$input.CopyTo($stream)
|
|
}
|
|
finally {
|
|
$input.Dispose()
|
|
}
|
|
if ($sequence % 500 -eq 0 -or $sequence -eq ($EndFrame + 1)) {
|
|
Write-Output (
|
|
"PHASE=e5-stream-reconstruction SEGMENTS={0}/{1}" -f
|
|
$sequence, ($EndFrame + 1)
|
|
)
|
|
}
|
|
}
|
|
$stream.Flush($true)
|
|
}
|
|
finally {
|
|
$stream.Dispose()
|
|
}
|
|
$null = Assert-FreeSpace "post-stream-reconstruction"
|
|
|
|
$extractWatch = [Diagnostics.Stopwatch]::StartNew()
|
|
Write-Output "PHASE=e5-frame-extraction-start"
|
|
$selectFilter = "select='between(n,{0},{1})'" -f $StartFrame, $EndFrame
|
|
& ffmpeg `
|
|
-hide_banner -loglevel fatal `
|
|
-i $streamPath `
|
|
-map 0:v:0 `
|
|
-vf $selectFilter `
|
|
-fps_mode passthrough `
|
|
(Join-Path $framesRoot "frame-%06d.png")
|
|
Assert-LastExitCode "LAB E5 camera clip extraction"
|
|
& ffprobe `
|
|
-v error `
|
|
-select_streams v:0 `
|
|
-show_entries frame=best_effort_timestamp_time `
|
|
-of json `
|
|
$streamPath | Set-Content -LiteralPath $ptsPath -Encoding utf8
|
|
Assert-LastExitCode "LAB E5 camera timestamp probe"
|
|
|
|
$decodedFrames = @(Get-ChildItem -LiteralPath $framesRoot -File -Filter "frame-*.png")
|
|
$ptsDocument = Get-Content -LiteralPath $ptsPath -Raw | ConvertFrom-Json
|
|
$pts = @($ptsDocument.frames)
|
|
if ($decodedFrames.Count -ne $clipFrameCount -or $pts.Count -ne ($EndFrame + 1)) {
|
|
throw "Decoded LAB E5 frame/timestamp count differs from the selected clip"
|
|
}
|
|
$firstEpochSeconds = [double]::Parse(
|
|
([string]$pts[0].best_effort_timestamp_time).Trim(),
|
|
[Globalization.CultureInfo]::InvariantCulture
|
|
)
|
|
$previousSessionSeconds = -1.0
|
|
$timelineWriter = [IO.StreamWriter]::new(
|
|
$timelinePath,
|
|
$false,
|
|
[Text.UTF8Encoding]::new($false)
|
|
)
|
|
try {
|
|
for ($localIndex = 0; $localIndex -lt $clipFrameCount; $localIndex++) {
|
|
$sourceIndex = $StartFrame + $localIndex
|
|
$epochSeconds = [double]::Parse(
|
|
([string]$pts[$sourceIndex].best_effort_timestamp_time).Trim(),
|
|
[Globalization.CultureInfo]::InvariantCulture
|
|
) - $firstEpochSeconds
|
|
$sessionSeconds = $timelineStart + $epochSeconds
|
|
if ($sessionSeconds -le $previousSessionSeconds) {
|
|
throw "Decoded LAB E5 clip timestamps are not strictly monotonic"
|
|
}
|
|
$row = [ordered]@{
|
|
frame_index = $localIndex
|
|
sequence = $localIndex + 1
|
|
source_frame_index = $sourceIndex
|
|
source_sequence = $sourceIndex + 1
|
|
epoch_seconds = $epochSeconds
|
|
session_seconds = $sessionSeconds
|
|
}
|
|
$timelineWriter.WriteLine(($row | ConvertTo-Json -Compress))
|
|
$previousSessionSeconds = $sessionSeconds
|
|
}
|
|
$timelineWriter.Flush()
|
|
}
|
|
finally {
|
|
$timelineWriter.Dispose()
|
|
}
|
|
$extractWatch.Stop()
|
|
$freeBytesPostExtract = Assert-FreeSpace "post-frame-extraction"
|
|
Write-Output ("PHASE=e5-frame-extraction-complete FRAMES={0}" -f $clipFrameCount)
|
|
|
|
$runArgs = @(
|
|
"run", "--rm", "--gpus", "all",
|
|
"--network", ("container:{0}" -f $TritonContainer),
|
|
"--read-only",
|
|
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
|
"--pids-limit", "256", "--shm-size", "1g",
|
|
"--tmpfs", "/tmp:rw,noexec,nosuid,size=512m",
|
|
"-e", "PYTHONPATH=/runner:/opt/env",
|
|
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
|
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
|
"-v", ((Convert-ToDockerPath $model) + ":/model:ro"),
|
|
"-v", ((Convert-ToDockerPath $pythonEnvironment) + ":/opt/env:ro"),
|
|
"-v", ((Convert-ToDockerPath $framesRoot) + ":/frames:ro"),
|
|
"-v", ((Convert-ToDockerPath $workRoot) + ":/work:ro"),
|
|
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
|
|
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
|
"--entrypoint", "python3",
|
|
$ContainerImage,
|
|
("/runner/{0}" -f $runnerName), "run",
|
|
"--job", "/job",
|
|
"--profile", ("/runner/{0}" -f $profileName),
|
|
"--valid-fov-root", "/valid-fov",
|
|
"--model-root", "/model",
|
|
"--frames", "/frames",
|
|
"--timeline", "/work/timeline.jsonl",
|
|
"--output", "/publish/output",
|
|
"--triton-url", "http://127.0.0.1:8000",
|
|
"--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB),
|
|
"--orchestrator-sha256", $orchestratorSha256,
|
|
"--container-image", $ContainerImage,
|
|
"--telemetry-interval-seconds", "1"
|
|
)
|
|
Write-Output ("PHASE=e5-inference-start FRAMES={0}" -f $clipFrameCount)
|
|
& docker @runArgs
|
|
Assert-LastExitCode "LAB E5 detector/tracker inference"
|
|
$freeBytesPostInference = Assert-FreeSpace "post-inference"
|
|
Write-Output "PHASE=e5-inference-complete"
|
|
|
|
$timelineRows = @(Get-Content -LiteralPath $timelinePath | ForEach-Object {
|
|
$_ | ConvertFrom-Json
|
|
})
|
|
$clipSpanSeconds = [double]$timelineRows[-1].session_seconds - [double]$timelineRows[0].session_seconds
|
|
if ($clipSpanSeconds -le 0) {
|
|
throw "LAB E5 clip duration is invalid"
|
|
}
|
|
$averageFps = ($clipFrameCount - 1) / $clipSpanSeconds
|
|
$fpsText = $averageFps.ToString("0.#########", [Globalization.CultureInfo]::InvariantCulture)
|
|
$videoPath = Join-Path $stagingRoot "tracking.mp4"
|
|
$encodeWatch = [Diagnostics.Stopwatch]::StartNew()
|
|
& ffmpeg `
|
|
-hide_banner -loglevel error `
|
|
-framerate $fpsText `
|
|
-i (Join-Path $stagingRoot "overlay-frames\frame-%06d.png") `
|
|
-c:v h264_nvenc `
|
|
-preset p4 `
|
|
-tune hq `
|
|
-rc vbr `
|
|
-cq 21 `
|
|
-b:v 0 `
|
|
-pix_fmt yuv420p `
|
|
-movflags +faststart `
|
|
$videoPath
|
|
Assert-LastExitCode "LAB E5 video encoding"
|
|
$encodeWatch.Stop()
|
|
Write-Output "PHASE=e5-video-encoding-complete"
|
|
|
|
$contactSheetPath = Join-Path $stagingRoot "contact-sheet.png"
|
|
$tileColumns = if ($clipFrameCount -le 60) { 4 } else { 3 }
|
|
$tileRows = 2
|
|
$sampleInterval = [math]::Max(0.25, $clipSpanSeconds / ($tileColumns * $tileRows))
|
|
$sampleText = $sampleInterval.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture)
|
|
$contactFilter = "fps=1/{0},scale=400:-1,tile={1}x{2}:padding=8:margin=8:color=black" -f `
|
|
$sampleText, $tileColumns, $tileRows
|
|
& ffmpeg `
|
|
-hide_banner -loglevel error `
|
|
-i $videoPath `
|
|
-vf $contactFilter `
|
|
-frames:v 1 `
|
|
$contactSheetPath
|
|
Assert-LastExitCode "LAB E5 contact-sheet generation"
|
|
|
|
$probe = & ffprobe `
|
|
-v error `
|
|
-select_streams v:0 `
|
|
-count_frames `
|
|
-show_entries stream=codec_name,width,height,nb_read_frames `
|
|
-of json `
|
|
$videoPath | ConvertFrom-Json
|
|
Assert-LastExitCode "LAB E5 result video probe"
|
|
$videoStream = @($probe.streams)[0]
|
|
if (
|
|
$videoStream.codec_name -ne "h264" -or
|
|
[int]$videoStream.width -ne 800 -or
|
|
[int]$videoStream.height -ne 600 -or
|
|
[int]$videoStream.nb_read_frames -ne $clipFrameCount
|
|
) {
|
|
throw "LAB E5 result video contract changed"
|
|
}
|
|
$freeBytesPostArtifacts = Assert-FreeSpace "post-artifacts"
|
|
|
|
$totalWatch.Stop()
|
|
$finalizeArgs = @(
|
|
"run", "--rm", "--network", "none", "--read-only",
|
|
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
|
"--pids-limit", "128", "--tmpfs", "/tmp:rw,noexec,nosuid,size=256m",
|
|
"-e", "PYTHONPATH=/runner",
|
|
"-v", ((Convert-ToDockerPath $stagingRoot) + ":/output:rw"),
|
|
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
|
"--entrypoint", "python3",
|
|
$ContainerImage,
|
|
("/runner/{0}" -f $runnerName), "finalize",
|
|
"--output", "/output",
|
|
"--video", "/output/tracking.mp4",
|
|
"--contact-sheet", "/output/contact-sheet.png",
|
|
"--extract-seconds", $extractWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
|
|
"--encode-seconds", $encodeWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
|
|
"--wall-seconds", $totalWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
|
|
"--encoder", "ffmpeg-h264_nvenc-p4-cq21-yuv420p-faststart",
|
|
"--disk-free-before-bytes", [string]$freeBytesBefore,
|
|
"--disk-free-post-extract-bytes", [string]$freeBytesPostExtract,
|
|
"--disk-free-post-inference-bytes", [string]$freeBytesPostInference,
|
|
"--disk-free-post-artifacts-bytes", [string]$freeBytesPostArtifacts,
|
|
"--disk-floor-bytes", [string]([int64]$FreeGiBFloor * 1GB),
|
|
"--working-set-reserve-bytes", [string]$workingSetReserve
|
|
)
|
|
& docker @finalizeArgs
|
|
Assert-LastExitCode "LAB E5 result finalization"
|
|
|
|
$result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json
|
|
if (
|
|
$result.schema_version -ne "missioncore.e5-tracking-result/v1" -or
|
|
$result.result_id -notmatch "^e5-tracking-[a-f0-9]{64}$" -or
|
|
[int]$result.frames_processed -ne $clipFrameCount -or
|
|
$result.ground_truth -ne $false -or
|
|
$result.publication_scope -ne "qualification-clip-only"
|
|
) {
|
|
throw "Final LAB E5 result manifest is incompatible"
|
|
}
|
|
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
|
if (Test-Path -LiteralPath $finalRoot) {
|
|
throw "An immutable result with the same LAB E5 identity already exists: $finalRoot"
|
|
}
|
|
$overlayFrames = Join-Path $stagingRoot "overlay-frames"
|
|
if (Test-Path -LiteralPath $overlayFrames) {
|
|
Remove-Item -LiteralPath $overlayFrames -Recurse -Force
|
|
}
|
|
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
|
|
Remove-Item -LiteralPath $publishRoot -Force
|
|
$completed = $true
|
|
$freeBytesFinal = Assert-FreeSpace "post-publication"
|
|
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
|
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
|
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
|
|
Write-Output ("DISK_FREE_BYTES_POST_EXTRACT={0}" -f $freeBytesPostExtract)
|
|
Write-Output ("DISK_FREE_BYTES_POST_INFERENCE={0}" -f $freeBytesPostInference)
|
|
Write-Output ("DISK_FREE_BYTES_POST_ARTIFACTS={0}" -f $freeBytesPostArtifacts)
|
|
Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeBytesFinal)
|
|
}
|
|
finally {
|
|
if (Test-Path -LiteralPath $workRoot) {
|
|
Remove-Item -LiteralPath $workRoot -Recurse -Force
|
|
}
|
|
if (-not $completed -and (Test-Path -LiteralPath $publishRoot)) {
|
|
Remove-Item -LiteralPath $publishRoot -Recurse -Force
|
|
}
|
|
if ($loadedByRun) {
|
|
try {
|
|
Invoke-WebRequest `
|
|
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/unload" `
|
|
-Method Post `
|
|
-ContentType "application/json" `
|
|
-Body "{}" `
|
|
-UseBasicParsing `
|
|
-TimeoutSec 60 *> $null
|
|
Write-Output "PHASE=e5-model-state-restored"
|
|
}
|
|
catch {
|
|
Write-Warning "LAB E5 could not restore the prior unloaded YOLOX-S state"
|
|
}
|
|
}
|
|
}
|