[CmdletBinding()] param( [Parameter(Mandatory = $true)] [string]$PackageRoot, [string]$SourceJobRoot = 'D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d', [string]$RuntimeRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46e', [string]$LogPath = '' ) $ErrorActionPreference = 'Stop' $ProgressPreference = 'SilentlyContinue' function Get-Sha256([string]$Path) { return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() } function Assert-Sha256([string]$Path, [string]$Expected, [string]$Label) { if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { throw "$Label is missing: $Path" } $actual = Get-Sha256 $Path if ($actual -ne $Expected) { throw "$Label SHA-256 changed: expected $Expected, got $actual" } } function Invoke-Docker([string[]]$Arguments, [string]$Label) { & docker @Arguments if ($LASTEXITCODE -ne 0) { throw "$Label failed with exit code $LASTEXITCODE" } } $package = (Resolve-Path -LiteralPath $PackageRoot).Path $sourceJob = (Resolve-Path -LiteralPath $SourceJobRoot).Path $profilePath = Join-Path $package 'profile.json' $manifestPath = Join-Path $package 'manifest.json' if (-not (Test-Path -LiteralPath $profilePath -PathType Leaf) -or -not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) { throw 'E46E package is incomplete.' } $manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json if ($manifest.schema_version -ne 'missioncore.e46e-worker-package/v1' -or $manifest.package_id -ne (Split-Path -Leaf $package)) { throw 'E46E package identity is invalid.' } $expectedPaths = @($manifest.identity.artifact_paths) foreach ($artifact in @($manifest.artifacts)) { if ($expectedPaths -notcontains [string]$artifact.path) { throw "Unexpected E46E package artifact: $($artifact.path)" } $artifactPath = Join-Path $package ([string]$artifact.path) Assert-Sha256 $artifactPath ([string]$artifact.sha256) "package artifact $($artifact.path)" if ((Get-Item -LiteralPath $artifactPath).Length -ne [int64]$artifact.byte_length) { throw "Package artifact length changed: $($artifact.path)" } } $actualPaths = @(Get-ChildItem -LiteralPath $package -Recurse -File | ForEach-Object { $_.FullName.Substring($package.Length + 1).Replace('\', '/') }) if (@($actualPaths | Where-Object { $_ -ne 'manifest.json' -and $expectedPaths -notcontains $_ }).Count -ne 0 -or @($expectedPaths | Where-Object { $actualPaths -notcontains $_ }).Count -ne 0) { throw 'E46E package file set changed.' } $profile = Get-Content -LiteralPath $profilePath -Raw | ConvertFrom-Json if ($profile.schema_version -ne 'missioncore.e46e-ready-stack-profile/v1') { throw 'E46E profile is incompatible.' } $image = [string]$profile.runtime.container_image $imageDigestMatch = [regex]::Match($image, '@sha256:([0-9a-f]{64})$') if (-not $imageDigestMatch.Success) { throw 'E46E runtime image must be pinned by a full SHA-256 digest.' } $imageDigest = $imageDigestMatch.Groups[1].Value $modelSha = [string]$profile.detector.model_sha256 $streamSha = [string]$profile.source.stream_sha256 $modelFile = [string]$profile.detector.model_file $modelUrl = [string]$profile.detector.model_url $parserFile = [string]$profile.parser.library_file $parserSha = [string]$profile.parser.library_sha256 $parserLibraryPath = Join-Path $package "runtime\$parserFile" New-Item -ItemType Directory -Force -Path $RuntimeRoot | Out-Null $logsRoot = Join-Path $RuntimeRoot 'logs' $modelsRoot = Join-Path $RuntimeRoot 'models\trafficcamnet_transformer_lite\deployable_resnet50_v2.0' $inputsRoot = Join-Path $RuntimeRoot 'inputs' $runsRoot = Join-Path $RuntimeRoot 'runs' $resultsRoot = Join-Path $RuntimeRoot 'ready-stack-results' foreach ($path in @($logsRoot, $modelsRoot, $inputsRoot, $runsRoot, $resultsRoot)) { New-Item -ItemType Directory -Force -Path $path | Out-Null } if ($LogPath) { $logParent = Split-Path -Parent $LogPath if ($logParent) { New-Item -ItemType Directory -Force -Path $logParent | Out-Null } Start-Transcript -LiteralPath $LogPath -Append | Out-Null } try { Write-Host "E46E package: $($manifest.package_id)" Write-Host "E46E source: $sourceJob" Write-Host "E46E image: $image" Assert-Sha256 $parserLibraryPath $parserSha 'official NVIDIA DeepStream TAO parser' $modelPath = Join-Path $modelsRoot $modelFile if (Test-Path -LiteralPath $modelPath -PathType Leaf) { Assert-Sha256 $modelPath $modelSha 'TrafficCamNet Transformer Lite model' } else { $modelTemp = "$modelPath.$([Guid]::NewGuid().ToString('N')).download" Write-Host 'Downloading exact NVIDIA TrafficCamNet Transformer Lite model...' & curl.exe --fail --location --retry 3 --output $modelTemp $modelUrl if ($LASTEXITCODE -ne 0) { throw "NVIDIA model download failed with exit code $LASTEXITCODE" } Assert-Sha256 $modelTemp $modelSha 'downloaded model' Move-Item -LiteralPath $modelTemp -Destination $modelPath } $jobPath = Join-Path $sourceJob 'job.json' $job = Get-Content -LiteralPath $jobPath -Raw | ConvertFrom-Json if ($job.job_id -ne $profile.source.job_id -or $job.input.archive_index_sha256 -ne $profile.source.archive_index_sha256 -or $job.input.archive_summary_sha256 -ne $profile.source.archive_summary_sha256) { throw 'Exact E46E source job binding changed.' } $cameraRoot = Join-Path $sourceJob 'input\camera\sensor.camera.right\epoch-1' $indexPath = Join-Path $cameraRoot 'index.jsonl' $summaryPath = Join-Path $cameraRoot 'summary.json' Assert-Sha256 $indexPath ([string]$profile.source.archive_index_sha256) 'source index' Assert-Sha256 $summaryPath ([string]$profile.source.archive_summary_sha256) 'source summary' $summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json if ($summary.stream_sha256 -ne $streamSha -or [int]$summary.segment_count -ne [int]$profile.source.segment_count) { throw 'Source stream identity changed.' } $inputPath = Join-Path $inputsRoot "right-$streamSha.mp4" if (Test-Path -LiteralPath $inputPath -PathType Leaf) { Assert-Sha256 $inputPath $streamSha 'reconstructed RIGHT stream' } else { $inputTemp = "$inputPath.$([Guid]::NewGuid().ToString('N')).tmp" $destinationStream = [System.IO.File]::Open( $inputTemp, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None ) $incremental = [System.Security.Cryptography.IncrementalHash]::CreateHash( [System.Security.Cryptography.HashAlgorithmName]::SHA256 ) try { $sourceParts = [System.Collections.Generic.List[string]]::new() $sourceParts.Add((Join-Path $cameraRoot 'init.mp4')) foreach ($line in [System.IO.File]::ReadLines($indexPath)) { $row = $line | ConvertFrom-Json $sourceParts.Add((Join-Path $cameraRoot ([string]$row.path))) } if ($sourceParts.Count -ne ([int]$profile.source.segment_count + 1)) { throw 'Source stream part count changed.' } $buffer = New-Object byte[] (4MB) foreach ($part in $sourceParts) { $inputStream = [System.IO.File]::OpenRead($part) try { while (($read = $inputStream.Read($buffer, 0, $buffer.Length)) -gt 0) { $destinationStream.Write($buffer, 0, $read) $incremental.AppendData($buffer, 0, $read) } } finally { $inputStream.Dispose() } } $destinationStream.Flush($true) $actualStreamSha = ([BitConverter]::ToString( $incremental.GetHashAndReset() )).Replace('-', '').ToLowerInvariant() } finally { $incremental.Dispose() $destinationStream.Dispose() } if ($actualStreamSha -ne $streamSha) { throw "Reconstructed stream SHA-256 changed: $actualStreamSha" } Move-Item -LiteralPath $inputTemp -Destination $inputPath } $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = 'Continue' try { & docker image inspect $image *> $null $imageCached = $LASTEXITCODE -eq 0 } finally { $ErrorActionPreference = $previousErrorActionPreference } if ($imageCached) { Write-Host 'Using the exact cached NVIDIA DeepStream image.' } else { Write-Host 'Pulling exact NVIDIA DeepStream image in the interactive user session...' Invoke-Docker @('pull', $image) 'DeepStream image pull' } $runId = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ') $runRoot = Join-Path $runsRoot $runId $rawRoot = Join-Path $runRoot 'raw' $inputMount = Join-Path $runRoot 'input' New-Item -ItemType Directory -Force -Path $rawRoot | Out-Null New-Item -ItemType Directory -Force -Path (Join-Path $rawRoot 'detections') | Out-Null New-Item -ItemType Directory -Force -Path (Join-Path $rawRoot 'tracks') | Out-Null New-Item -ItemType Directory -Force -Path $inputMount | Out-Null Copy-Item -LiteralPath $inputPath -Destination (Join-Path $inputMount 'right.mp4') $deepstreamLog = Join-Path $rawRoot 'deepstream.log' $trackerCopy = Join-Path $rawRoot 'tracker-config.yml' $startedAt = (Get-Date).ToUniversalTime().ToString('o') $containerCommand = @" set -euo pipefail cp /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml /workspace/output/tracker-config.yml deepstream-app -c /workspace/package/runtime/e46e_deepstream_app.txt "@ $dockerArguments = @( 'run', '--rm', '--name', "ndc-mission-core-deepstream-e46e-$runId", '--gpus', 'all', '--network', 'none', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '--shm-size', '4g', '--label', 'com.nodedc.product=mission-core', '--label', 'com.nodedc.stack=perception', '--label', 'com.nodedc.role=deepstream-ready-stack-e46e', '--label', 'com.nodedc.managed-by=mission-core-worker', '--mount', "type=bind,src=$inputMount,dst=/workspace/input,readonly", '--mount', "type=bind,src=$package,dst=/workspace/package,readonly", '--mount', "type=bind,src=$modelsRoot,dst=/workspace/model", '--mount', "type=bind,src=$rawRoot,dst=/workspace/output", '--entrypoint', '/bin/bash', $image, '-lc', $containerCommand ) Write-Host 'Running full 4489-frame NVIDIA detector + NvDCF replay...' $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = 'Continue' try { & docker @dockerArguments 2>&1 | Tee-Object -LiteralPath $deepstreamLog $deepstreamExit = $LASTEXITCODE } finally { $ErrorActionPreference = $previousErrorActionPreference } if ($deepstreamExit -ne 0) { throw "DeepStream replay failed with exit code $deepstreamExit" } $overlayPath = Join-Path $rawRoot 'overlay.mp4' $enginePath = Join-Path $modelsRoot "$modelFile`_b1_gpu0_fp16.engine" if (-not (Test-Path -LiteralPath $overlayPath -PathType Leaf) -or (Get-Item -LiteralPath $overlayPath).Length -eq 0) { throw 'DeepStream did not produce an overlay video.' } if (-not (Test-Path -LiteralPath $enginePath -PathType Leaf)) { throw 'DeepStream did not produce the exact TensorRT engine.' } $detectionFiles = @(Get-ChildItem -LiteralPath (Join-Path $rawRoot 'detections') -File) $trackFiles = @(Get-ChildItem -LiteralPath (Join-Path $rawRoot 'tracks') -File) if ($detectionFiles.Count -ne [int]$profile.source.segment_count -or $trackFiles.Count -ne [int]$profile.source.segment_count) { throw "DeepStream frame coverage changed: detections=$($detectionFiles.Count), tracks=$($trackFiles.Count)" } $runtime = [ordered]@{ schema_version = 'missioncore.e46e-deepstream-runtime/v1' status = 'completed' worker_host = $env:COMPUTERNAME gpu_name = ((& nvidia-smi --query-gpu=name --format=csv,noheader | Select-Object -First 1).Trim()) started_at_utc = $startedAt completed_at_utc = (Get-Date).ToUniversalTime().ToString('o') container_image = $image container_image_digest = $imageDigest model_sha256 = Get-Sha256 $modelPath model_engine_sha256 = Get-Sha256 $enginePath deepstream_config_sha256 = Get-Sha256 (Join-Path $package 'runtime\e46e_deepstream_app.txt') detector_config_sha256 = Get-Sha256 (Join-Path $package 'runtime\e46e_trafficcamnet_rtdetr.txt') parser_library_sha256 = Get-Sha256 $parserLibraryPath tracker_config_sha256 = Get-Sha256 $trackerCopy input_stream_sha256 = Get-Sha256 $inputPath overlay_sha256 = Get-Sha256 $overlayPath frame_count = [int]$profile.source.segment_count deepstream_exit_code = $deepstreamExit } $runtimePath = Join-Path $rawRoot 'runtime.json' $runtime | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $runtimePath -Encoding UTF8 $consolidatorImage = 'nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794' $consolidatorArguments = @( 'run', '--rm', '--name', "ndc-mission-core-e46e-consolidator-$runId", '--network', 'none', '--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '--tmpfs', '/tmp:rw,noexec,nosuid,size=64m', '--mount', "type=bind,src=$package,dst=/workspace/package,readonly", '--mount', "type=bind,src=$sourceJob,dst=/workspace/source-job,readonly", '--mount', "type=bind,src=$rawRoot,dst=/workspace/raw,readonly", '--mount', "type=bind,src=$resultsRoot,dst=/workspace/results", '-e', 'PYTHONPATH=/workspace/package/runtime', '-e', 'PYTHONDONTWRITEBYTECODE=1', $consolidatorImage, 'python3', '/workspace/package/runtime/run_e46e_ready_stack.py', '--source-job', '/workspace/source-job', '--raw-root', '/workspace/raw', '--profile', '/workspace/package/profile.json', '--output-root', '/workspace/results' ) Write-Host 'Freezing immutable E46E evidence...' Invoke-Docker $consolidatorArguments 'E46E consolidation' Write-Host "E46E_READY_STACK_COMPLETED run=$runId results=$resultsRoot" } finally { if ($LogPath) { Stop-Transcript | Out-Null } }