330 lines
15 KiB
PowerShell
330 lines
15 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$PackageRoot,
|
|
|
|
[string]$SourceJobRoot = 'D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d',
|
|
|
|
[string]$RuntimeRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46f',
|
|
|
|
[string]$LogPath = ''
|
|
)
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
|
|
function Get-Sha256([string]$Path) {
|
|
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
}
|
|
|
|
function Assert-Sha256([string]$Path, [string]$Expected, [string]$Label) {
|
|
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
|
throw "$Label is missing: $Path"
|
|
}
|
|
$actual = Get-Sha256 $Path
|
|
if ($actual -ne $Expected) {
|
|
throw "$Label SHA-256 changed: expected $Expected, got $actual"
|
|
}
|
|
}
|
|
|
|
function Invoke-Docker([string[]]$Arguments, [string]$Label) {
|
|
& docker @Arguments
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "$Label failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
$package = (Resolve-Path -LiteralPath $PackageRoot).Path
|
|
$sourceJob = (Resolve-Path -LiteralPath $SourceJobRoot).Path
|
|
$profilePath = Join-Path $package 'profile.json'
|
|
$manifestPath = Join-Path $package 'manifest.json'
|
|
if (-not (Test-Path -LiteralPath $profilePath -PathType Leaf) -or
|
|
-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) {
|
|
throw 'E46F package is incomplete.'
|
|
}
|
|
$manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
|
|
if ($manifest.schema_version -ne 'missioncore.e46f-worker-package/v1' -or
|
|
$manifest.package_id -ne (Split-Path -Leaf $package)) {
|
|
throw 'E46F package identity is invalid.'
|
|
}
|
|
$expectedPaths = @($manifest.identity.artifact_paths)
|
|
foreach ($artifact in @($manifest.artifacts)) {
|
|
if ($expectedPaths -notcontains [string]$artifact.path) {
|
|
throw "Unexpected E46F package artifact: $($artifact.path)"
|
|
}
|
|
$artifactPath = Join-Path $package ([string]$artifact.path)
|
|
Assert-Sha256 $artifactPath ([string]$artifact.sha256) "package artifact $($artifact.path)"
|
|
if ((Get-Item -LiteralPath $artifactPath).Length -ne [int64]$artifact.byte_length) {
|
|
throw "Package artifact length changed: $($artifact.path)"
|
|
}
|
|
}
|
|
$actualPaths = @(Get-ChildItem -LiteralPath $package -Recurse -File | ForEach-Object {
|
|
$_.FullName.Substring($package.Length + 1).Replace('\', '/')
|
|
})
|
|
if (@($actualPaths | Where-Object { $_ -ne 'manifest.json' -and $expectedPaths -notcontains $_ }).Count -ne 0 -or
|
|
@($expectedPaths | Where-Object { $actualPaths -notcontains $_ }).Count -ne 0) {
|
|
throw 'E46F package file set changed.'
|
|
}
|
|
|
|
$profile = Get-Content -LiteralPath $profilePath -Raw | ConvertFrom-Json
|
|
if ($profile.schema_version -ne 'missioncore.e46f-dashcam-bakeoff-profile/v1' -or
|
|
$profile.comparison_contract.controlled_change -ne 'detector-only') {
|
|
throw 'E46F profile is incompatible.'
|
|
}
|
|
$image = [string]$profile.runtime.container_image
|
|
$imageDigestMatch = [regex]::Match($image, '@sha256:([0-9a-f]{64})$')
|
|
if (-not $imageDigestMatch.Success) {
|
|
throw 'E46F runtime image must be pinned by a full SHA-256 digest.'
|
|
}
|
|
$imageDigest = $imageDigestMatch.Groups[1].Value
|
|
$modelSha = [string]$profile.detector.model_sha256
|
|
$streamSha = [string]$profile.source.stream_sha256
|
|
$modelFile = [string]$profile.detector.model_file
|
|
$modelUrl = [string]$profile.detector.model_url
|
|
|
|
New-Item -ItemType Directory -Force -Path $RuntimeRoot | Out-Null
|
|
$logsRoot = Join-Path $RuntimeRoot 'logs'
|
|
$modelsRoot = Join-Path $RuntimeRoot "models\dashcamnet\$([string]$profile.detector.version)"
|
|
$inputsRoot = Join-Path $RuntimeRoot 'inputs'
|
|
$runsRoot = Join-Path $RuntimeRoot 'runs'
|
|
$resultsRoot = Join-Path $RuntimeRoot 'dashcam-bakeoff-results'
|
|
foreach ($path in @($logsRoot, $modelsRoot, $inputsRoot, $runsRoot, $resultsRoot)) {
|
|
New-Item -ItemType Directory -Force -Path $path | Out-Null
|
|
}
|
|
if ($LogPath) {
|
|
$logParent = Split-Path -Parent $LogPath
|
|
if ($logParent) { New-Item -ItemType Directory -Force -Path $logParent | Out-Null }
|
|
Start-Transcript -LiteralPath $LogPath -Append | Out-Null
|
|
}
|
|
|
|
try {
|
|
Write-Host "E46F package: $($manifest.package_id)"
|
|
Write-Host "E46F source: $sourceJob"
|
|
Write-Host "E46F image: $image"
|
|
|
|
$modelPath = Join-Path $modelsRoot $modelFile
|
|
if (Test-Path -LiteralPath $modelPath -PathType Leaf) {
|
|
Assert-Sha256 $modelPath $modelSha 'DashCamNet model'
|
|
}
|
|
else {
|
|
$modelTemp = "$modelPath.$([Guid]::NewGuid().ToString('N')).download"
|
|
Write-Host 'Downloading exact NVIDIA DashCamNet model...'
|
|
& curl.exe --fail --location --retry 3 --output $modelTemp $modelUrl
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "NVIDIA model download failed with exit code $LASTEXITCODE"
|
|
}
|
|
Assert-Sha256 $modelTemp $modelSha 'downloaded DashCamNet model'
|
|
Move-Item -LiteralPath $modelTemp -Destination $modelPath
|
|
}
|
|
|
|
$jobPath = Join-Path $sourceJob 'job.json'
|
|
$job = Get-Content -LiteralPath $jobPath -Raw | ConvertFrom-Json
|
|
if ($job.job_id -ne $profile.source.job_id -or
|
|
$job.input.archive_index_sha256 -ne $profile.source.archive_index_sha256 -or
|
|
$job.input.archive_summary_sha256 -ne $profile.source.archive_summary_sha256) {
|
|
throw 'Exact E46F source job binding changed.'
|
|
}
|
|
$cameraRoot = Join-Path $sourceJob 'input\camera\sensor.camera.right\epoch-1'
|
|
$indexPath = Join-Path $cameraRoot 'index.jsonl'
|
|
$summaryPath = Join-Path $cameraRoot 'summary.json'
|
|
Assert-Sha256 $indexPath ([string]$profile.source.archive_index_sha256) 'source index'
|
|
Assert-Sha256 $summaryPath ([string]$profile.source.archive_summary_sha256) 'source summary'
|
|
$summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json
|
|
if ($summary.stream_sha256 -ne $streamSha -or
|
|
[int]$summary.segment_count -ne [int]$profile.source.segment_count) {
|
|
throw 'Source stream identity changed.'
|
|
}
|
|
|
|
$inputPath = Join-Path $inputsRoot "right-$streamSha.mp4"
|
|
$e46eInputPath = "D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs\right-$streamSha.mp4"
|
|
if (Test-Path -LiteralPath $inputPath -PathType Leaf) {
|
|
Assert-Sha256 $inputPath $streamSha 'E46F reconstructed RIGHT stream'
|
|
}
|
|
elseif (Test-Path -LiteralPath $e46eInputPath -PathType Leaf) {
|
|
Assert-Sha256 $e46eInputPath $streamSha 'E46E controlled RIGHT stream'
|
|
Copy-Item -LiteralPath $e46eInputPath -Destination $inputPath
|
|
Assert-Sha256 $inputPath $streamSha 'E46F copied RIGHT stream'
|
|
}
|
|
else {
|
|
$inputTemp = "$inputPath.$([Guid]::NewGuid().ToString('N')).tmp"
|
|
$destinationStream = [System.IO.File]::Open(
|
|
$inputTemp,
|
|
[System.IO.FileMode]::CreateNew,
|
|
[System.IO.FileAccess]::Write,
|
|
[System.IO.FileShare]::None
|
|
)
|
|
$incremental = [System.Security.Cryptography.IncrementalHash]::CreateHash(
|
|
[System.Security.Cryptography.HashAlgorithmName]::SHA256
|
|
)
|
|
try {
|
|
$sourceParts = [System.Collections.Generic.List[string]]::new()
|
|
$sourceParts.Add((Join-Path $cameraRoot 'init.mp4'))
|
|
foreach ($line in [System.IO.File]::ReadLines($indexPath)) {
|
|
$row = $line | ConvertFrom-Json
|
|
$sourceParts.Add((Join-Path $cameraRoot ([string]$row.path)))
|
|
}
|
|
if ($sourceParts.Count -ne ([int]$profile.source.segment_count + 1)) {
|
|
throw 'Source stream part count changed.'
|
|
}
|
|
$buffer = New-Object byte[] (4MB)
|
|
foreach ($part in $sourceParts) {
|
|
$inputStream = [System.IO.File]::OpenRead($part)
|
|
try {
|
|
while (($read = $inputStream.Read($buffer, 0, $buffer.Length)) -gt 0) {
|
|
$destinationStream.Write($buffer, 0, $read)
|
|
$incremental.AppendData($buffer, 0, $read)
|
|
}
|
|
}
|
|
finally {
|
|
$inputStream.Dispose()
|
|
}
|
|
}
|
|
$destinationStream.Flush($true)
|
|
$actualStreamSha = ([BitConverter]::ToString(
|
|
$incremental.GetHashAndReset()
|
|
)).Replace('-', '').ToLowerInvariant()
|
|
}
|
|
finally {
|
|
$incremental.Dispose()
|
|
$destinationStream.Dispose()
|
|
}
|
|
if ($actualStreamSha -ne $streamSha) {
|
|
throw "Reconstructed stream SHA-256 changed: $actualStreamSha"
|
|
}
|
|
Move-Item -LiteralPath $inputTemp -Destination $inputPath
|
|
}
|
|
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = 'Continue'
|
|
try {
|
|
& docker image inspect $image *> $null
|
|
$imageCached = $LASTEXITCODE -eq 0
|
|
}
|
|
finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
if ($imageCached) {
|
|
Write-Host 'Using the exact cached NVIDIA DeepStream image.'
|
|
}
|
|
else {
|
|
Write-Host 'Pulling exact NVIDIA DeepStream image in the interactive user session...'
|
|
Invoke-Docker @('pull', $image) 'DeepStream image pull'
|
|
}
|
|
|
|
$runId = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ')
|
|
$runRoot = Join-Path $runsRoot $runId
|
|
$rawRoot = Join-Path $runRoot 'raw'
|
|
$inputMount = Join-Path $runRoot 'input'
|
|
New-Item -ItemType Directory -Force -Path $rawRoot | Out-Null
|
|
New-Item -ItemType Directory -Force -Path (Join-Path $rawRoot 'detections') | Out-Null
|
|
New-Item -ItemType Directory -Force -Path (Join-Path $rawRoot 'tracks') | Out-Null
|
|
New-Item -ItemType Directory -Force -Path $inputMount | Out-Null
|
|
Copy-Item -LiteralPath $inputPath -Destination (Join-Path $inputMount 'right.mp4')
|
|
$deepstreamLog = Join-Path $rawRoot 'deepstream.log'
|
|
$trackerCopy = Join-Path $rawRoot 'tracker-config.yml'
|
|
$startedAt = (Get-Date).ToUniversalTime().ToString('o')
|
|
|
|
$containerCommand = @"
|
|
set -euo pipefail
|
|
cp /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml /workspace/output/tracker-config.yml
|
|
deepstream-app -c /workspace/package/runtime/e46f_deepstream_app.txt
|
|
"@
|
|
$dockerArguments = @(
|
|
'run', '--rm', '--name', "ndc-mission-core-deepstream-e46f-$runId",
|
|
'--gpus', 'all', '--network', 'none', '--cap-drop', 'ALL',
|
|
'--security-opt', 'no-new-privileges', '--shm-size', '4g',
|
|
'--label', 'com.nodedc.product=mission-core',
|
|
'--label', 'com.nodedc.stack=perception',
|
|
'--label', 'com.nodedc.role=deepstream-dashcam-bakeoff-e46f',
|
|
'--label', 'com.nodedc.managed-by=mission-core-worker',
|
|
'--mount', "type=bind,src=$inputMount,dst=/workspace/input,readonly",
|
|
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
|
'--mount', "type=bind,src=$modelsRoot,dst=/workspace/model",
|
|
'--mount', "type=bind,src=$rawRoot,dst=/workspace/output",
|
|
'--entrypoint', '/bin/bash', $image, '-lc', $containerCommand
|
|
)
|
|
Write-Host 'Running full 4489-frame NVIDIA DashCamNet + NvDCF bake-off...'
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = 'Continue'
|
|
try {
|
|
& docker @dockerArguments 2>&1 | Tee-Object -LiteralPath $deepstreamLog
|
|
$deepstreamExit = $LASTEXITCODE
|
|
}
|
|
finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
if ($deepstreamExit -ne 0) {
|
|
throw "DeepStream replay failed with exit code $deepstreamExit"
|
|
}
|
|
$invalidOutputBinding = Select-String `
|
|
-LiteralPath $deepstreamLog `
|
|
-Pattern 'Could not find output layer','Given invalid tensor name' `
|
|
-SimpleMatch `
|
|
-Quiet
|
|
if ($invalidOutputBinding) {
|
|
throw 'DeepStream accepted the process but rejected the configured detector output bindings.'
|
|
}
|
|
|
|
$overlayPath = Join-Path $rawRoot 'overlay.mp4'
|
|
$enginePath = Join-Path $modelsRoot "$modelFile`_b1_gpu0_fp16.engine"
|
|
if (-not (Test-Path -LiteralPath $overlayPath -PathType Leaf) -or
|
|
(Get-Item -LiteralPath $overlayPath).Length -eq 0) {
|
|
throw 'DeepStream did not produce an overlay video.'
|
|
}
|
|
if (-not (Test-Path -LiteralPath $enginePath -PathType Leaf)) {
|
|
throw 'DeepStream did not produce the exact TensorRT engine.'
|
|
}
|
|
$detectionFiles = @(Get-ChildItem -LiteralPath (Join-Path $rawRoot 'detections') -File)
|
|
$trackFiles = @(Get-ChildItem -LiteralPath (Join-Path $rawRoot 'tracks') -File)
|
|
if ($detectionFiles.Count -ne [int]$profile.source.segment_count -or
|
|
$trackFiles.Count -ne [int]$profile.source.segment_count) {
|
|
throw "DeepStream frame coverage changed: detections=$($detectionFiles.Count), tracks=$($trackFiles.Count)"
|
|
}
|
|
$runtime = [ordered]@{
|
|
schema_version = 'missioncore.e46f-dashcam-deepstream-runtime/v1'
|
|
status = 'completed'
|
|
worker_host = $env:COMPUTERNAME
|
|
gpu_name = ((& nvidia-smi --query-gpu=name --format=csv,noheader | Select-Object -First 1).Trim())
|
|
started_at_utc = $startedAt
|
|
completed_at_utc = (Get-Date).ToUniversalTime().ToString('o')
|
|
container_image = $image
|
|
container_image_digest = $imageDigest
|
|
model_sha256 = Get-Sha256 $modelPath
|
|
model_engine_sha256 = Get-Sha256 $enginePath
|
|
deepstream_config_sha256 = Get-Sha256 (Join-Path $package 'runtime\e46f_deepstream_app.txt')
|
|
detector_config_sha256 = Get-Sha256 (Join-Path $package 'runtime\e46f_dashcamnet_detectnet.txt')
|
|
tracker_config_sha256 = Get-Sha256 $trackerCopy
|
|
input_stream_sha256 = Get-Sha256 $inputPath
|
|
overlay_sha256 = Get-Sha256 $overlayPath
|
|
frame_count = [int]$profile.source.segment_count
|
|
deepstream_exit_code = $deepstreamExit
|
|
}
|
|
$runtimePath = Join-Path $rawRoot 'runtime.json'
|
|
$runtime | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $runtimePath -Encoding UTF8
|
|
|
|
$consolidatorImage = 'nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794'
|
|
$consolidatorArguments = @(
|
|
'run', '--rm', '--name', "ndc-mission-core-e46f-consolidator-$runId",
|
|
'--network', 'none', '--read-only', '--cap-drop', 'ALL',
|
|
'--security-opt', 'no-new-privileges', '--tmpfs', '/tmp:rw,noexec,nosuid,size=64m',
|
|
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
|
'--mount', "type=bind,src=$sourceJob,dst=/workspace/source-job,readonly",
|
|
'--mount', "type=bind,src=$rawRoot,dst=/workspace/raw,readonly",
|
|
'--mount', "type=bind,src=$resultsRoot,dst=/workspace/results",
|
|
'-e', 'PYTHONPATH=/workspace/package/runtime',
|
|
'-e', 'PYTHONDONTWRITEBYTECODE=1',
|
|
$consolidatorImage,
|
|
'python3', '/workspace/package/runtime/run_e46f_dashcam_bakeoff.py',
|
|
'--source-job', '/workspace/source-job',
|
|
'--raw-root', '/workspace/raw',
|
|
'--profile', '/workspace/package/profile.json',
|
|
'--output-root', '/workspace/results'
|
|
)
|
|
Write-Host 'Freezing immutable E46F evidence...'
|
|
Invoke-Docker $consolidatorArguments 'E46F consolidation'
|
|
Write-Host "E46F_DASHCAM_BAKEOFF_COMPLETED run=$runId results=$resultsRoot"
|
|
}
|
|
finally {
|
|
if ($LogPath) { Stop-Transcript | Out-Null }
|
|
}
|