[CmdletBinding()] param( [Parameter(Mandatory = $true)] [string]$PackageRoot, [string]$SourceJobRoot = 'D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d', [string]$RuntimeRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46h', [string]$LogPath = '' ) $ErrorActionPreference = 'Stop' $ProgressPreference = 'SilentlyContinue' function Get-Sha256([string]$Path) { return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() } function Assert-Sha256([string]$Path, [string]$Expected, [string]$Label) { if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { throw "$Label is missing: $Path" } $actual = Get-Sha256 $Path if ($actual -ne $Expected) { throw "$Label SHA-256 changed: expected $Expected, got $actual" } } function Invoke-Docker([string[]]$Arguments, [string]$Label) { & docker @Arguments if ($LASTEXITCODE -ne 0) { throw "$Label failed with exit code $LASTEXITCODE" } } function Get-VideoFrameCount([string]$Path) { $probe = & ffprobe -v error -select_streams v:0 -count_frames ` -show_entries stream=nb_read_frames -of json $Path | ConvertFrom-Json if ($LASTEXITCODE -ne 0) { throw "ffprobe failed: $Path" } return [int]@($probe.streams)[0].nb_read_frames } function Ensure-Model([pscustomobject]$Detector, [string]$ModelRoot) { New-Item -ItemType Directory -Force -Path $ModelRoot | Out-Null $modelPath = Join-Path $ModelRoot ([string]$Detector.model_file) if (Test-Path -LiteralPath $modelPath -PathType Leaf) { Assert-Sha256 $modelPath ([string]$Detector.model_sha256) 'TrafficCamNet model' return $modelPath } $temporary = "$modelPath.$([Guid]::NewGuid().ToString('N')).download" & curl.exe --fail --location --retry 3 --output $temporary ([string]$Detector.model_url) if ($LASTEXITCODE -ne 0) { throw "TrafficCamNet download failed with exit code $LASTEXITCODE" } Assert-Sha256 $temporary ([string]$Detector.model_sha256) 'downloaded TrafficCamNet model' Move-Item -LiteralPath $temporary -Destination $modelPath return $modelPath } $package = (Resolve-Path -LiteralPath $PackageRoot).Path $sourceJob = (Resolve-Path -LiteralPath $SourceJobRoot).Path $profilePath = Join-Path $package 'profile.json' $manifestPath = Join-Path $package 'manifest.json' if (-not (Test-Path -LiteralPath $profilePath -PathType Leaf) -or -not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) { throw 'E46H package is incomplete.' } $manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json if ($manifest.schema_version -ne 'missioncore.e46h-worker-package/v1' -or $manifest.package_id -ne (Split-Path -Leaf $package)) { throw 'E46H package identity is invalid.' } $expectedPaths = @($manifest.identity.artifact_paths) foreach ($artifact in @($manifest.artifacts)) { if ($expectedPaths -notcontains [string]$artifact.path) { throw "Unexpected E46H package artifact: $($artifact.path)" } $artifactPath = Join-Path $package ([string]$artifact.path) Assert-Sha256 $artifactPath ([string]$artifact.sha256) "package artifact $($artifact.path)" if ((Get-Item -LiteralPath $artifactPath).Length -ne [int64]$artifact.byte_length) { throw "Package artifact length changed: $($artifact.path)" } } $actualPaths = @(Get-ChildItem -LiteralPath $package -Recurse -File | ForEach-Object { $_.FullName.Substring($package.Length + 1).Replace('\', '/') }) if (@($actualPaths | Where-Object { $_ -ne 'manifest.json' -and $expectedPaths -notcontains $_ }).Count -ne 0 -or @($expectedPaths | Where-Object { $actualPaths -notcontains $_ }).Count -ne 0) { throw 'E46H package file set changed.' } $profile = Get-Content -LiteralPath $profilePath -Raw | ConvertFrom-Json if ($profile.schema_version -ne 'missioncore.e46h-full-rectified-front-replay-profile/v1' -or $profile.source.camera_source_id -ne 'sensor.camera.right' -or $profile.rectification.view -ne 'front' -or $profile.detector.name -ne 'NVIDIA TrafficCamNet Transformer Lite') { throw 'E46H profile is incompatible.' } $image = [string]$profile.runtime.container_image $imageDigestMatch = [regex]::Match($image, '@sha256:([0-9a-f]{64})$') if (-not $imageDigestMatch.Success) { throw 'E46H runtime image must be pinned by a full SHA-256 digest.' } $imageDigest = $imageDigestMatch.Groups[1].Value $streamSha = [string]$profile.source.stream_sha256 $frameCount = [int]$profile.selection.frame_count if ($frameCount -ne 4488 -or [int]$profile.selection.first_source_frame_index -ne 0 -or [int]$profile.selection.last_source_frame_index -ne 4487) { throw 'E46H retained route contract changed.' } $parserPath = Join-Path $package "runtime\$([string]$profile.parser.library_file)" Assert-Sha256 $parserPath ([string]$profile.parser.library_sha256) ` 'official NVIDIA DeepStream TAO parser' if (-not (Get-Command ffmpeg -ErrorAction SilentlyContinue) -or -not (Get-Command ffprobe -ErrorAction SilentlyContinue)) { throw 'E46H requires the existing Worker ffmpeg/ffprobe installation.' } New-Item -ItemType Directory -Force -Path $RuntimeRoot | Out-Null $inputsRoot = Join-Path $RuntimeRoot 'inputs' $runsRoot = Join-Path $RuntimeRoot 'runs' $resultsRoot = Join-Path $RuntimeRoot 'full-rectified-front-results' foreach ($path in @($inputsRoot, $runsRoot, $resultsRoot)) { New-Item -ItemType Directory -Force -Path $path | Out-Null } if ($LogPath) { $logParent = Split-Path -Parent $LogPath if ($logParent) { New-Item -ItemType Directory -Force -Path $logParent | Out-Null } Start-Transcript -LiteralPath $LogPath -Append | Out-Null } try { $jobPath = Join-Path $sourceJob 'job.json' $job = Get-Content -LiteralPath $jobPath -Raw | ConvertFrom-Json if ($job.job_id -ne $profile.source.job_id -or $job.input.archive_index_sha256 -ne $profile.source.archive_index_sha256 -or $job.input.archive_summary_sha256 -ne $profile.source.archive_summary_sha256) { throw 'Exact E46H source job binding changed.' } $inputPath = Join-Path $inputsRoot "right-$streamSha.mp4" $e46gInput = "D:\NDC_MISSIONCORE\runtime\experiments\e46g\inputs\right-$streamSha.mp4" $e46eInput = "D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs\right-$streamSha.mp4" if (Test-Path -LiteralPath $inputPath -PathType Leaf) { Assert-Sha256 $inputPath $streamSha 'E46H controlled RIGHT stream' } elseif (Test-Path -LiteralPath $e46gInput -PathType Leaf) { Assert-Sha256 $e46gInput $streamSha 'E46G controlled RIGHT stream' Copy-Item -LiteralPath $e46gInput -Destination $inputPath } elseif (Test-Path -LiteralPath $e46eInput -PathType Leaf) { Assert-Sha256 $e46eInput $streamSha 'E46E controlled RIGHT stream' Copy-Item -LiteralPath $e46eInput -Destination $inputPath } else { & (Join-Path $package 'runtime\Prepare-RectifiedCameraReplay.ps1') ` -JobRoot $sourceJob -OutputPath $inputPath } Assert-Sha256 $inputPath $streamSha 'E46H reconstructed RIGHT stream' $modelRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46e\models\trafficcamnet_transformer_lite\deployable_resnet50_v2.0' $modelPath = Ensure-Model $profile.detector $modelRoot $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = 'Continue' try { & docker image inspect $image *> $null $imageCached = $LASTEXITCODE -eq 0 } finally { $ErrorActionPreference = $previousErrorActionPreference } if (-not $imageCached) { Invoke-Docker @('pull', $image) 'DeepStream image pull' } $runId = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ') $runRoot = Join-Path $runsRoot $runId $rawRoot = Join-Path $runRoot 'raw' $sourceInputMount = Join-Path $runRoot 'source-input' $geometryRoot = Join-Path $rawRoot 'geometry' $runOutput = Join-Path $rawRoot 'run' $frontInputMount = Join-Path $runRoot 'front-input' foreach ($path in @( $rawRoot, $sourceInputMount, $geometryRoot, $runOutput, $frontInputMount, (Join-Path $runOutput 'detections'), (Join-Path $runOutput 'tracks') )) { New-Item -ItemType Directory -Force -Path $path | Out-Null } Copy-Item -LiteralPath $inputPath -Destination (Join-Path $sourceInputMount 'right.mp4') $workerLog = Join-Path $rawRoot 'worker.log' "E46H run $runId`nsource=$streamSha`nselection=0..4487`nview=front" | Set-Content -LiteralPath $workerLog -Encoding UTF8 $startedAt = (Get-Date).ToUniversalTime().ToString('o') $dewarperConfig = Join-Path $package "runtime\$([string]$profile.rectification.config_file)" Assert-Sha256 $dewarperConfig ([string]$profile.rectification.config_sha256) ` 'FRONT dewarper config' $frontPath = Join-Path $geometryRoot 'front.mp4' $dewarperLog = Join-Path $geometryRoot 'front.log' $dewarperCommand = @" set -euo pipefail gst-launch-1.0 -e filesrc location=/workspace/input/right.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=RGBA' ! nvdewarper config-file=/workspace/package/runtime/$([string]$profile.rectification.config_file) source-id=0 num-batch-buffers=1 ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=NV12' ! nvv4l2h264enc bitrate=6000000 ! h264parse ! qtmux ! filesink location=/workspace/output/front.mp4 "@ $dewarperArguments = @( 'run', '--rm', '--name', "ndc-mission-core-e46h-dewarper-$runId", '--gpus', 'all', '--network', 'none', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '--shm-size', '4g', '--label', 'com.nodedc.product=mission-core', '--label', 'com.nodedc.stack=perception', '--label', 'com.nodedc.role=nvdewarper-e46h-front', '--mount', "type=bind,src=$sourceInputMount,dst=/workspace/input,readonly", '--mount', "type=bind,src=$package,dst=/workspace/package,readonly", '--mount', "type=bind,src=$geometryRoot,dst=/workspace/output", '--entrypoint', '/bin/bash', $image, '-lc', $dewarperCommand ) $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = 'Continue' try { & docker @dewarperArguments 2>&1 | Tee-Object -LiteralPath $dewarperLog $dewarperExit = $LASTEXITCODE } finally { $ErrorActionPreference = $previousErrorActionPreference } if ($dewarperExit -ne 0 -or -not (Test-Path -LiteralPath $frontPath -PathType Leaf)) { throw "NVIDIA nvdewarper failed with exit code $dewarperExit" } if ((Get-VideoFrameCount $frontPath) -ne $frameCount) { throw 'E46H FRONT frame coverage changed.' } $normalizedFront = Join-Path $geometryRoot 'front-normalized.mp4' & ffmpeg -hide_banner -loglevel error -y -fflags +genpts -i $frontPath ` -vf 'setpts=N/(10*TB)' -an -c:v libx264 -preset fast -crf 18 ` -pix_fmt yuv420p -r 10 -movflags +faststart $normalizedFront if ($LASTEXITCODE -ne 0 -or (Get-VideoFrameCount $normalizedFront) -ne $frameCount) { throw 'E46H FRONT timestamp normalization failed.' } Move-Item -LiteralPath $normalizedFront -Destination $frontPath -Force Copy-Item -LiteralPath $frontPath -Destination (Join-Path $frontInputMount 'view.mp4') $appConfig = Join-Path $package "runtime\$([string]$profile.detector.deepstream_app_config)" $detectorConfig = Join-Path $package "runtime\$([string]$profile.detector.detector_config)" Assert-Sha256 $appConfig ([string]$profile.detector.deepstream_app_config_sha256) ` 'TrafficCamNet DeepStream app config' Assert-Sha256 $detectorConfig ([string]$profile.detector.detector_config_sha256) ` 'TrafficCamNet detector config' $deepstreamLog = Join-Path $runOutput 'deepstream.log' $deepstreamCommand = @" set -euo pipefail cp /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml /workspace/output/tracker-config.yml deepstream-app -c /workspace/package/runtime/$([string]$profile.detector.deepstream_app_config) "@ $deepstreamArguments = @( 'run', '--rm', '--name', "ndc-mission-core-e46h-front-$runId", '--gpus', 'all', '--network', 'none', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '--shm-size', '4g', '--label', 'com.nodedc.product=mission-core', '--label', 'com.nodedc.stack=perception', '--label', 'com.nodedc.role=e46h-front-trafficcamnet', '--mount', "type=bind,src=$frontInputMount,dst=/workspace/input,readonly", '--mount', "type=bind,src=$package,dst=/workspace/package,readonly", '--mount', "type=bind,src=$modelRoot,dst=/workspace/model", '--mount', "type=bind,src=$runOutput,dst=/workspace/output", '--entrypoint', '/bin/bash', $image, '-lc', $deepstreamCommand ) $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = 'Continue' try { & docker @deepstreamArguments 2>&1 | Tee-Object -LiteralPath $deepstreamLog $deepstreamExit = $LASTEXITCODE } finally { $ErrorActionPreference = $previousErrorActionPreference } if ($deepstreamExit -ne 0) { throw "DeepStream failed with exit code $deepstreamExit" } $overlayPath = Join-Path $runOutput 'overlay.mp4' $trackerPath = Join-Path $runOutput 'tracker-config.yml' $detections = @(Get-ChildItem -LiteralPath (Join-Path $runOutput 'detections') -File) $tracks = @(Get-ChildItem -LiteralPath (Join-Path $runOutput 'tracks') -File) if (-not (Test-Path -LiteralPath $overlayPath -PathType Leaf) -or $detections.Count -ne $frameCount -or $tracks.Count -ne $frameCount) { throw "DeepStream output coverage changed: detections=$($detections.Count), tracks=$($tracks.Count)" } $fastOverlay = Join-Path $runOutput 'overlay-faststart.mp4' & ffmpeg -hide_banner -loglevel error -y -i $overlayPath -c copy -movflags +faststart $fastOverlay if ($LASTEXITCODE -ne 0 -or (Get-VideoFrameCount $fastOverlay) -ne $frameCount) { throw 'E46H fast-start overlay normalization failed.' } Move-Item -LiteralPath $fastOverlay -Destination $overlayPath -Force $enginePath = "$modelPath`_b1_gpu0_fp16.engine" if (-not (Test-Path -LiteralPath $enginePath -PathType Leaf)) { throw 'TrafficCamNet TensorRT engine is missing.' } $runtime = [ordered]@{ schema_version = 'missioncore.e46h-full-rectified-front-runtime/v1' status = 'completed' worker_host = $env:COMPUTERNAME gpu_name = ((& nvidia-smi --query-gpu=name --format=csv,noheader | Select-Object -First 1).Trim()) started_at_utc = $startedAt completed_at_utc = (Get-Date).ToUniversalTime().ToString('o') container_image = $image container_image_digest = $imageDigest source_stream_sha256 = Get-Sha256 $inputPath frame_count = $frameCount retained_source_frame_index_range = @(0, 4487) geometry = [ordered]@{ video_path = 'geometry/front.mp4' video_sha256 = Get-Sha256 $frontPath log_path = 'geometry/front.log' log_sha256 = Get-Sha256 $dewarperLog config_sha256 = Get-Sha256 $dewarperConfig frame_count = $frameCount dewarper_exit_code = $dewarperExit } run = [ordered]@{ overlay_path = 'run/overlay.mp4' overlay_sha256 = Get-Sha256 $overlayPath deepstream_log_path = 'run/deepstream.log' deepstream_log_sha256 = Get-Sha256 $deepstreamLog model_sha256 = Get-Sha256 $modelPath model_engine_sha256 = Get-Sha256 $enginePath parser_library_sha256 = Get-Sha256 $parserPath deepstream_app_config_sha256 = Get-Sha256 $appConfig detector_config_sha256 = Get-Sha256 $detectorConfig tracker_config_sha256 = Get-Sha256 $trackerPath frame_count = $frameCount deepstream_exit_code = $deepstreamExit } } $runtime | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath (Join-Path $rawRoot 'runtime.json') -Encoding UTF8 Add-Content -LiteralPath $workerLog -Value "completed=$(Get-Date -Format o)" $consolidatorImage = 'nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794' $consolidatorArguments = @( 'run', '--rm', '--name', "ndc-mission-core-e46h-consolidator-$runId", '--network', 'none', '--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '--tmpfs', '/tmp:rw,noexec,nosuid,size=64m', '--mount', "type=bind,src=$package,dst=/workspace/package,readonly", '--mount', "type=bind,src=$sourceJob,dst=/workspace/source-job,readonly", '--mount', "type=bind,src=$rawRoot,dst=/workspace/raw,readonly", '--mount', "type=bind,src=$resultsRoot,dst=/workspace/results", '-e', 'PYTHONPATH=/workspace/package/runtime', '-e', 'PYTHONDONTWRITEBYTECODE=1', $consolidatorImage, 'python3', '/workspace/package/runtime/run_e46h_full_rectified_front_replay.py', '--source-job', '/workspace/source-job', '--raw-root', '/workspace/raw', '--profile', '/workspace/package/profile.json', '--output-root', '/workspace/results' ) Invoke-Docker $consolidatorArguments 'E46H consolidation' Write-Host "E46H_FULL_RECTIFIED_FRONT_COMPLETED run=$runId results=$resultsRoot" } finally { if ($LogPath) { Stop-Transcript | Out-Null } }