feat(perception): integrate calibrated operator pipeline

Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
2026-07-23 00:23:28 +03:00
parent ada2a55ee6
commit b53d6d5a45
221 changed files with 55923 additions and 1357 deletions
@@ -0,0 +1,544 @@
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$JobRoot,
+
+    [Parameter(Mandatory = $true)]
+    [string]$RunnerPath,
+
+    [Parameter(Mandatory = $true)]
+    [string]$ProfilePath,
+
+    [Parameter(Mandatory = $true)]
+    [string]$ValidFovRoot,
+
+    [ValidateRange(0, 1000000)]
+    [int]$StartFrame = 1000,
+
+    [ValidateRange(0, 1000000)]
+    [int]$EndFrame = 1600,
+
+    [ValidateRange(1, 1000)]
+    [int]$FreeGiBFloor = 360,
+
+    [string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
+
+    [string]$ModelRoot = "D:\NDC_MISSIONCORE\runtime\models\yolox_s",
+
+    [string]$TritonContainer = "mission-core-triton",
+
+    [string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3"
+)
+
+$ErrorActionPreference = "Stop"
+$ProgressPreference = "SilentlyContinue"
+
+function Assert-LastExitCode {
+    param([string]$Operation)
+    if ($LASTEXITCODE -ne 0) {
+        throw "$Operation failed with exit code $LASTEXITCODE"
+    }
+}
+
+function Assert-RegularFile {
+    param([string]$Path, [string]$Label)
+    $item = Get-Item -LiteralPath $Path -Force
+    if (-not $item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
+        return $item.FullName
+    }
+    throw "$Label is not a regular file"
+}
+
+function Assert-Directory {
+    param([string]$Path, [string]$Label)
+    $item = Get-Item -LiteralPath $Path -Force
+    if ($item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
+        return $item.FullName
+    }
+    throw "$Label is not a real directory"
+}
+
+function Assert-DDrivePath {
+    param([string]$Path, [string]$Label)
+    $fullPath = [IO.Path]::GetFullPath($Path)
+    if ([IO.Path]::GetPathRoot($fullPath).TrimEnd("\") -ine "D:") {
+        throw "$Label must be stored on D:"
+    }
+    return $fullPath
+}
+
+function Get-DFreeBytes {
+    return [int64](Get-PSDrive -Name D).Free
+}
+
+function Assert-FreeSpace {
+    param(
+        [string]$Phase,
+        [int64]$RequiredAdditionalBytes = 0
+    )
+    $freeBytes = Get-DFreeBytes
+    $floorBytes = [int64]$FreeGiBFloor * 1GB
+    Write-Host (
+        "DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3} REQUIRED_ADDITIONAL_GIB={4}" -f
+        $Phase,
+        $freeBytes,
+        [math]::Round($freeBytes / 1GB, 3),
+        $FreeGiBFloor,
+        [math]::Round($RequiredAdditionalBytes / 1GB, 3)
+    )
+    if ($freeBytes -lt ($floorBytes + $RequiredAdditionalBytes)) {
+        throw "D: does not have the guarded LAB E5 working-set reserve during $Phase"
+    }
+    return $freeBytes
+}
+
+function Convert-ToDockerPath {
+    param([string]$Path)
+    return $Path.Replace("\", "/")
+}
+
+function Test-TritonModelReady {
+    param([string]$ModelName)
+    try {
+        $response = Invoke-WebRequest `
+            -Uri ("http://127.0.0.1:8000/v2/models/{0}/ready" -f $ModelName) `
+            -Method Get `
+            -UseBasicParsing `
+            -TimeoutSec 10
+        return $response.StatusCode -eq 200
+    }
+    catch {
+        return $false
+    }
+}
+
+$jobDirectory = Assert-DDrivePath (
+    Assert-Directory (Resolve-Path -LiteralPath $JobRoot).Path "Job root"
+) "Job root"
+$runner = Assert-DDrivePath (
+    Assert-RegularFile (Resolve-Path -LiteralPath $RunnerPath).Path "LAB E5 runner"
+) "LAB E5 runner"
+$profile = Assert-DDrivePath (
+    Assert-RegularFile (Resolve-Path -LiteralPath $ProfilePath).Path "LAB E5 profile"
+) "LAB E5 profile"
+$validFov = Assert-DDrivePath (
+    Assert-Directory (Resolve-Path -LiteralPath $ValidFovRoot).Path "Valid-FOV root"
+) "Valid-FOV root"
+$runtime = Assert-DDrivePath (
+    Assert-Directory (Resolve-Path -LiteralPath $RuntimeRoot).Path "Runtime root"
+) "Runtime root"
+$model = Assert-DDrivePath (
+    Assert-Directory (Resolve-Path -LiteralPath $ModelRoot).Path "YOLOX model root"
+) "YOLOX model root"
+$runnerRoot = Split-Path $runner -Parent
+if ((Split-Path $profile -Parent) -ne $runnerRoot) {
+    throw "LAB E5 runner and profile must share one read-only mount"
+}
+foreach ($dependency in @(
+    "run_recorded_perception_epoch.py",
+    "run_e3_rectified_segmentation.py",
+    "run_evaluation_prelabels.py"
+)) {
+    $null = Assert-RegularFile (Join-Path $runnerRoot $dependency) "LAB E5 runner dependency"
+}
+
+$job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json
+if ($job.schema_version -ne "missioncore.compute-job/v1" -or $job.job_id -ne (Split-Path $jobDirectory -Leaf)) {
+    throw "Compute job manifest is incompatible"
+}
+$sourceId = [string]$job.input.source_id
+$epoch = [int]$job.input.codec_epoch
+$fullFrameCount = [int]$job.input.segment_count
+$clipFrameCount = $EndFrame - $StartFrame + 1
+$timelineStart = [double]$job.input.timeline.start_seconds
+if (
+    $sourceId -ne "sensor.camera.right" -or
+    $StartFrame -lt 0 -or
+    $EndFrame -lt $StartFrame -or
+    $EndFrame -ge $fullFrameCount -or
+    $clipFrameCount -lt 2
+) {
+    throw "LAB E5 clip escapes the camera job"
+}
+Write-Output (
+    "PHASE=job-manifest-validated JOB={0} CLIP={1}-{2} FRAMES={3}" -f
+    $job.job_id, $StartFrame, $EndFrame, $clipFrameCount
+)
+
+$epochRoot = Join-Path $jobDirectory ("input\camera\{0}\epoch-{1}" -f $sourceId, $epoch)
+$epochRoot = Assert-DDrivePath (Assert-Directory $epochRoot "Camera epoch") "Camera epoch"
+$initPath = Assert-RegularFile (Join-Path $epochRoot "init.mp4") "Camera init"
+$segmentsRoot = Assert-Directory (Join-Path $epochRoot "segments") "Camera segments"
+$derivedRoot = Join-Path $runtime "derived"
+$tmpRoot = Join-Path $runtime "tmp"
+$pythonEnvironment = Assert-DDrivePath (
+    Assert-Directory (Join-Path $derivedRoot "perception-p0-env-v1") "Python environment"
+) "Python environment"
+$null = New-Item -ItemType Directory -Path $derivedRoot -Force
+$null = New-Item -ItemType Directory -Path $tmpRoot -Force
+
+# Conservative bound: partial reconstructed stream, decoded RGB PNG payload,
+# RGB overlays, encoded result and 2 GiB of filesystem/codec overhead.
+$pixelWorkingSet = [int64]$clipFrameCount * 800 * 600 * 6
+$partialStreamReserve = [int64][math]::Ceiling(
+    ([int64]$job.input.byte_length * ($EndFrame + 1) / $fullFrameCount) * 1.2
+)
+$workingSetReserve = [int64][math]::Ceiling($pixelWorkingSet * 1.4) + $partialStreamReserve + 2GB
+$freeBytesBefore = Assert-FreeSpace "preflight" $workingSetReserve
+
+& docker image inspect $ContainerImage *> $null
+Assert-LastExitCode "Existing container image inspection"
+$tritonState = docker inspect --format "{{.State.Running}}" $TritonContainer
+Assert-LastExitCode "Triton container inspection"
+if ($tritonState.Trim().ToLowerInvariant() -ne "true") {
+    throw "LAB E5 requires the existing Triton container"
+}
+
+$runnerName = Split-Path $runner -Leaf
+$profileName = Split-Path $profile -Leaf
+$orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
+$preflightArgs = @(
+    "run", "--rm", "--network", "none", "--read-only",
+    "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
+    "--pids-limit", "128", "--tmpfs", "/tmp:rw,noexec,nosuid,size=256m",
+    "-e", "PYTHONPATH=/runner:/opt/env",
+    "-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
+    "-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
+    "-v", ((Convert-ToDockerPath $model) + ":/model:ro"),
+    "-v", ((Convert-ToDockerPath $pythonEnvironment) + ":/opt/env:ro"),
+    "-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
+    "--entrypoint", "python3",
+    $ContainerImage,
+    ("/runner/{0}" -f $runnerName), "preflight",
+    "--job", "/job",
+    "--profile", ("/runner/{0}" -f $profileName),
+    "--valid-fov-root", "/valid-fov",
+    "--model-root", "/model"
+)
+Write-Output "PHASE=e5-preflight-start"
+& docker @preflightArgs
+Assert-LastExitCode "LAB E5 preflight"
+Write-Output "PHASE=e5-preflight-complete"
+
+$modelWasReady = Test-TritonModelReady "yolox_s"
+$loadedByRun = $false
+if (-not $modelWasReady) {
+    Invoke-WebRequest `
+        -Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/load" `
+        -Method Post `
+        -ContentType "application/json" `
+        -Body "{}" `
+        -UseBasicParsing `
+        -TimeoutSec 60 *> $null
+    $loadedByRun = $true
+    $deadline = [DateTime]::UtcNow.AddSeconds(60)
+    while (-not (Test-TritonModelReady "yolox_s")) {
+        if ([DateTime]::UtcNow -ge $deadline) {
+            throw "YOLOX-S did not become ready in Triton"
+        }
+        Start-Sleep -Milliseconds 500
+    }
+}
+Write-Output ("PHASE=e5-model-ready LOADED_BY_RUN={0}" -f $loadedByRun)
+
+$runToken = [Guid]::NewGuid().ToString("N")
+$workRoot = Join-Path $tmpRoot ("{0}-e5-{1}" -f $job.job_id, $runToken)
+$framesRoot = Join-Path $workRoot "frames"
+$streamPath = Join-Path $workRoot "camera.mp4"
+$ptsPath = Join-Path $workRoot "pts.json"
+$timelinePath = Join-Path $workRoot "timeline.jsonl"
+$publishRoot = Join-Path $derivedRoot (".{0}-e5-{1}.publish" -f $job.job_id, $runToken)
+$stagingRoot = Join-Path $publishRoot "output"
+$null = New-Item -ItemType Directory -Path $framesRoot
+$null = New-Item -ItemType Directory -Path $publishRoot
+$totalWatch = [Diagnostics.Stopwatch]::StartNew()
+$completed = $false
+Write-Output "PHASE=e5-private-staging-created"
+
+try {
+    $stream = [IO.File]::Open(
+        $streamPath,
+        [IO.FileMode]::CreateNew,
+        [IO.FileAccess]::Write,
+        [IO.FileShare]::None
+    )
+    try {
+        $input = [IO.File]::OpenRead($initPath)
+        try {
+            $input.CopyTo($stream)
+        }
+        finally {
+            $input.Dispose()
+        }
+        for ($sequence = 1; $sequence -le ($EndFrame + 1); $sequence++) {
+            $path = Assert-RegularFile (
+                Join-Path $segmentsRoot ("{0}.m4s" -f $sequence)
+            ) "Camera segment"
+            $input = [IO.File]::OpenRead($path)
+            try {
+                $input.CopyTo($stream)
+            }
+            finally {
+                $input.Dispose()
+            }
+            if ($sequence % 500 -eq 0 -or $sequence -eq ($EndFrame + 1)) {
+                Write-Output (
+                    "PHASE=e5-stream-reconstruction SEGMENTS={0}/{1}" -f
+                    $sequence, ($EndFrame + 1)
+                )
+            }
+        }
+        $stream.Flush($true)
+    }
+    finally {
+        $stream.Dispose()
+    }
+    $null = Assert-FreeSpace "post-stream-reconstruction"
+
+    $extractWatch = [Diagnostics.Stopwatch]::StartNew()
+    Write-Output "PHASE=e5-frame-extraction-start"
+    $selectFilter = "select='between(n,{0},{1})'" -f $StartFrame, $EndFrame
+    & ffmpeg `
+        -hide_banner -loglevel fatal `
+        -i $streamPath `
+        -map 0:v:0 `
+        -vf $selectFilter `
+        -fps_mode passthrough `
+        (Join-Path $framesRoot "frame-%06d.png")
+    Assert-LastExitCode "LAB E5 camera clip extraction"
+    & ffprobe `
+        -v error `
+        -select_streams v:0 `
+        -show_entries frame=best_effort_timestamp_time `
+        -of json `
+        $streamPath | Set-Content -LiteralPath $ptsPath -Encoding utf8
+    Assert-LastExitCode "LAB E5 camera timestamp probe"
+
+    $decodedFrames = @(Get-ChildItem -LiteralPath $framesRoot -File -Filter "frame-*.png")
+    $ptsDocument = Get-Content -LiteralPath $ptsPath -Raw | ConvertFrom-Json
+    $pts = @($ptsDocument.frames)
+    if ($decodedFrames.Count -ne $clipFrameCount -or $pts.Count -ne ($EndFrame + 1)) {
+        throw "Decoded LAB E5 frame/timestamp count differs from the selected clip"
+    }
+    $firstEpochSeconds = [double]::Parse(
+        ([string]$pts[0].best_effort_timestamp_time).Trim(),
+        [Globalization.CultureInfo]::InvariantCulture
+    )
+    $previousSessionSeconds = -1.0
+    $timelineWriter = [IO.StreamWriter]::new(
+        $timelinePath,
+        $false,
+        [Text.UTF8Encoding]::new($false)
+    )
+    try {
+        for ($localIndex = 0; $localIndex -lt $clipFrameCount; $localIndex++) {
+            $sourceIndex = $StartFrame + $localIndex
+            $epochSeconds = [double]::Parse(
+                ([string]$pts[$sourceIndex].best_effort_timestamp_time).Trim(),
+                [Globalization.CultureInfo]::InvariantCulture
+            ) - $firstEpochSeconds
+            $sessionSeconds = $timelineStart + $epochSeconds
+            if ($sessionSeconds -le $previousSessionSeconds) {
+                throw "Decoded LAB E5 clip timestamps are not strictly monotonic"
+            }
+            $row = [ordered]@{
+                frame_index = $localIndex
+                sequence = $localIndex + 1
+                source_frame_index = $sourceIndex
+                source_sequence = $sourceIndex + 1
+                epoch_seconds = $epochSeconds
+                session_seconds = $sessionSeconds
+            }
+            $timelineWriter.WriteLine(($row | ConvertTo-Json -Compress))
+            $previousSessionSeconds = $sessionSeconds
+        }
+        $timelineWriter.Flush()
+    }
+    finally {
+        $timelineWriter.Dispose()
+    }
+    $extractWatch.Stop()
+    $freeBytesPostExtract = Assert-FreeSpace "post-frame-extraction"
+    Write-Output ("PHASE=e5-frame-extraction-complete FRAMES={0}" -f $clipFrameCount)
+
+    $runArgs = @(
+        "run", "--rm", "--gpus", "all",
+        "--network", ("container:{0}" -f $TritonContainer),
+        "--read-only",
+        "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
+        "--pids-limit", "256", "--shm-size", "1g",
+        "--tmpfs", "/tmp:rw,noexec,nosuid,size=512m",
+        "-e", "PYTHONPATH=/runner:/opt/env",
+        "-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
+        "-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
+        "-v", ((Convert-ToDockerPath $model) + ":/model:ro"),
+        "-v", ((Convert-ToDockerPath $pythonEnvironment) + ":/opt/env:ro"),
+        "-v", ((Convert-ToDockerPath $framesRoot) + ":/frames:ro"),
+        "-v", ((Convert-ToDockerPath $workRoot) + ":/work:ro"),
+        "-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
+        "-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
+        "--entrypoint", "python3",
+        $ContainerImage,
+        ("/runner/{0}" -f $runnerName), "run",
+        "--job", "/job",
+        "--profile", ("/runner/{0}" -f $profileName),
+        "--valid-fov-root", "/valid-fov",
+        "--model-root", "/model",
+        "--frames", "/frames",
+        "--timeline", "/work/timeline.jsonl",
+        "--output", "/publish/output",
+        "--triton-url", "http://127.0.0.1:8000",
+        "--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB),
+        "--orchestrator-sha256", $orchestratorSha256,
+        "--container-image", $ContainerImage,
+        "--telemetry-interval-seconds", "1"
+    )
+    Write-Output ("PHASE=e5-inference-start FRAMES={0}" -f $clipFrameCount)
+    & docker @runArgs
+    Assert-LastExitCode "LAB E5 detector/tracker inference"
+    $freeBytesPostInference = Assert-FreeSpace "post-inference"
+    Write-Output "PHASE=e5-inference-complete"
+
+    $timelineRows = @(Get-Content -LiteralPath $timelinePath | ForEach-Object {
+        $_ | ConvertFrom-Json
+    })
+    $clipSpanSeconds = [double]$timelineRows[-1].session_seconds - [double]$timelineRows[0].session_seconds
+    if ($clipSpanSeconds -le 0) {
+        throw "LAB E5 clip duration is invalid"
+    }
+    $averageFps = ($clipFrameCount - 1) / $clipSpanSeconds
+    $fpsText = $averageFps.ToString("0.#########", [Globalization.CultureInfo]::InvariantCulture)
+    $videoPath = Join-Path $stagingRoot "tracking.mp4"
+    $encodeWatch = [Diagnostics.Stopwatch]::StartNew()
+    & ffmpeg `
+        -hide_banner -loglevel error `
+        -framerate $fpsText `
+        -i (Join-Path $stagingRoot "overlay-frames\frame-%06d.png") `
+        -c:v h264_nvenc `
+        -preset p4 `
+        -tune hq `
+        -rc vbr `
+        -cq 21 `
+        -b:v 0 `
+        -pix_fmt yuv420p `
+        -movflags +faststart `
+        $videoPath
+    Assert-LastExitCode "LAB E5 video encoding"
+    $encodeWatch.Stop()
+    Write-Output "PHASE=e5-video-encoding-complete"
+
+    $contactSheetPath = Join-Path $stagingRoot "contact-sheet.png"
+    $tileColumns = if ($clipFrameCount -le 60) { 4 } else { 3 }
+    $tileRows = 2
+    $sampleInterval = [math]::Max(0.25, $clipSpanSeconds / ($tileColumns * $tileRows))
+    $sampleText = $sampleInterval.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture)
+    $contactFilter = "fps=1/{0},scale=400:-1,tile={1}x{2}:padding=8:margin=8:color=black" -f `
+        $sampleText, $tileColumns, $tileRows
+    & ffmpeg `
+        -hide_banner -loglevel error `
+        -i $videoPath `
+        -vf $contactFilter `
+        -frames:v 1 `
+        $contactSheetPath
+    Assert-LastExitCode "LAB E5 contact-sheet generation"
+
+    $probe = & ffprobe `
+        -v error `
+        -select_streams v:0 `
+        -count_frames `
+        -show_entries stream=codec_name,width,height,nb_read_frames `
+        -of json `
+        $videoPath | ConvertFrom-Json
+    Assert-LastExitCode "LAB E5 result video probe"
+    $videoStream = @($probe.streams)[0]
+    if (
+        $videoStream.codec_name -ne "h264" -or
+        [int]$videoStream.width -ne 800 -or
+        [int]$videoStream.height -ne 600 -or
+        [int]$videoStream.nb_read_frames -ne $clipFrameCount
+    ) {
+        throw "LAB E5 result video contract changed"
+    }
+    $freeBytesPostArtifacts = Assert-FreeSpace "post-artifacts"
+
+    $totalWatch.Stop()
+    $finalizeArgs = @(
+        "run", "--rm", "--network", "none", "--read-only",
+        "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
+        "--pids-limit", "128", "--tmpfs", "/tmp:rw,noexec,nosuid,size=256m",
+        "-e", "PYTHONPATH=/runner",
+        "-v", ((Convert-ToDockerPath $stagingRoot) + ":/output:rw"),
+        "-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
+        "--entrypoint", "python3",
+        $ContainerImage,
+        ("/runner/{0}" -f $runnerName), "finalize",
+        "--output", "/output",
+        "--video", "/output/tracking.mp4",
+        "--contact-sheet", "/output/contact-sheet.png",
+        "--extract-seconds", $extractWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
+        "--encode-seconds", $encodeWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
+        "--wall-seconds", $totalWatch.Elapsed.TotalSeconds.ToString("0.######", [Globalization.CultureInfo]::InvariantCulture),
+        "--encoder", "ffmpeg-h264_nvenc-p4-cq21-yuv420p-faststart",
+        "--disk-free-before-bytes", [string]$freeBytesBefore,
+        "--disk-free-post-extract-bytes", [string]$freeBytesPostExtract,
+        "--disk-free-post-inference-bytes", [string]$freeBytesPostInference,
+        "--disk-free-post-artifacts-bytes", [string]$freeBytesPostArtifacts,
+        "--disk-floor-bytes", [string]([int64]$FreeGiBFloor * 1GB),
+        "--working-set-reserve-bytes", [string]$workingSetReserve
+    )
+    & docker @finalizeArgs
+    Assert-LastExitCode "LAB E5 result finalization"
+
+    $result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json
+    if (
+        $result.schema_version -ne "missioncore.e5-tracking-result/v1" -or
+        $result.result_id -notmatch "^e5-tracking-[a-f0-9]{64}$" -or
+        [int]$result.frames_processed -ne $clipFrameCount -or
+        $result.ground_truth -ne $false -or
+        $result.publication_scope -ne "qualification-clip-only"
+    ) {
+        throw "Final LAB E5 result manifest is incompatible"
+    }
+    $finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
+    if (Test-Path -LiteralPath $finalRoot) {
+        throw "An immutable result with the same LAB E5 identity already exists: $finalRoot"
+    }
+    $overlayFrames = Join-Path $stagingRoot "overlay-frames"
+    if (Test-Path -LiteralPath $overlayFrames) {
+        Remove-Item -LiteralPath $overlayFrames -Recurse -Force
+    }
+    Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
+    Remove-Item -LiteralPath $publishRoot -Force
+    $completed = $true
+    $freeBytesFinal = Assert-FreeSpace "post-publication"
+    Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
+    Write-Output ("RESULT_ID={0}" -f $result.result_id)
+    Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
+    Write-Output ("DISK_FREE_BYTES_POST_EXTRACT={0}" -f $freeBytesPostExtract)
+    Write-Output ("DISK_FREE_BYTES_POST_INFERENCE={0}" -f $freeBytesPostInference)
+    Write-Output ("DISK_FREE_BYTES_POST_ARTIFACTS={0}" -f $freeBytesPostArtifacts)
+    Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeBytesFinal)
+}
+finally {
+    if (Test-Path -LiteralPath $workRoot) {
+        Remove-Item -LiteralPath $workRoot -Recurse -Force
+    }
+    if (-not $completed -and (Test-Path -LiteralPath $publishRoot)) {
+        Remove-Item -LiteralPath $publishRoot -Recurse -Force
+    }
+    if ($loadedByRun) {
+        try {
+            Invoke-WebRequest `
+                -Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/unload" `
+                -Method Post `
+                -ContentType "application/json" `
+                -Body "{}" `
+                -UseBasicParsing `
+                -TimeoutSec 60 *> $null
+            Write-Output "PHASE=e5-model-state-restored"
+        }
+        catch {
+            Write-Warning "LAB E5 could not restore the prior unloaded YOLOX-S state"
+        }
+    }
+}