370 lines
17 KiB
PowerShell
370 lines
17 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$PackageRoot,
|
|
|
|
[string]$SourceJobRoot = 'D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d',
|
|
|
|
[string]$RuntimeRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46h',
|
|
|
|
[string]$LogPath = ''
|
|
)
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
|
|
function Get-Sha256([string]$Path) {
|
|
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
}
|
|
|
|
function Assert-Sha256([string]$Path, [string]$Expected, [string]$Label) {
|
|
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
|
throw "$Label is missing: $Path"
|
|
}
|
|
$actual = Get-Sha256 $Path
|
|
if ($actual -ne $Expected) {
|
|
throw "$Label SHA-256 changed: expected $Expected, got $actual"
|
|
}
|
|
}
|
|
|
|
function Invoke-Docker([string[]]$Arguments, [string]$Label) {
|
|
& docker @Arguments
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "$Label failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
function Get-VideoFrameCount([string]$Path) {
|
|
$probe = & ffprobe -v error -select_streams v:0 -count_frames `
|
|
-show_entries stream=nb_read_frames -of json $Path | ConvertFrom-Json
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "ffprobe failed: $Path"
|
|
}
|
|
return [int]@($probe.streams)[0].nb_read_frames
|
|
}
|
|
|
|
function Ensure-Model([pscustomobject]$Detector, [string]$ModelRoot) {
|
|
New-Item -ItemType Directory -Force -Path $ModelRoot | Out-Null
|
|
$modelPath = Join-Path $ModelRoot ([string]$Detector.model_file)
|
|
if (Test-Path -LiteralPath $modelPath -PathType Leaf) {
|
|
Assert-Sha256 $modelPath ([string]$Detector.model_sha256) 'TrafficCamNet model'
|
|
return $modelPath
|
|
}
|
|
$temporary = "$modelPath.$([Guid]::NewGuid().ToString('N')).download"
|
|
& curl.exe --fail --location --retry 3 --output $temporary ([string]$Detector.model_url)
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "TrafficCamNet download failed with exit code $LASTEXITCODE"
|
|
}
|
|
Assert-Sha256 $temporary ([string]$Detector.model_sha256) 'downloaded TrafficCamNet model'
|
|
Move-Item -LiteralPath $temporary -Destination $modelPath
|
|
return $modelPath
|
|
}
|
|
|
|
$package = (Resolve-Path -LiteralPath $PackageRoot).Path
|
|
$sourceJob = (Resolve-Path -LiteralPath $SourceJobRoot).Path
|
|
$profilePath = Join-Path $package 'profile.json'
|
|
$manifestPath = Join-Path $package 'manifest.json'
|
|
if (-not (Test-Path -LiteralPath $profilePath -PathType Leaf) -or
|
|
-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) {
|
|
throw 'E46H package is incomplete.'
|
|
}
|
|
$manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
|
|
if ($manifest.schema_version -ne 'missioncore.e46h-worker-package/v1' -or
|
|
$manifest.package_id -ne (Split-Path -Leaf $package)) {
|
|
throw 'E46H package identity is invalid.'
|
|
}
|
|
$expectedPaths = @($manifest.identity.artifact_paths)
|
|
foreach ($artifact in @($manifest.artifacts)) {
|
|
if ($expectedPaths -notcontains [string]$artifact.path) {
|
|
throw "Unexpected E46H package artifact: $($artifact.path)"
|
|
}
|
|
$artifactPath = Join-Path $package ([string]$artifact.path)
|
|
Assert-Sha256 $artifactPath ([string]$artifact.sha256) "package artifact $($artifact.path)"
|
|
if ((Get-Item -LiteralPath $artifactPath).Length -ne [int64]$artifact.byte_length) {
|
|
throw "Package artifact length changed: $($artifact.path)"
|
|
}
|
|
}
|
|
$actualPaths = @(Get-ChildItem -LiteralPath $package -Recurse -File | ForEach-Object {
|
|
$_.FullName.Substring($package.Length + 1).Replace('\', '/')
|
|
})
|
|
if (@($actualPaths | Where-Object { $_ -ne 'manifest.json' -and $expectedPaths -notcontains $_ }).Count -ne 0 -or
|
|
@($expectedPaths | Where-Object { $actualPaths -notcontains $_ }).Count -ne 0) {
|
|
throw 'E46H package file set changed.'
|
|
}
|
|
|
|
$profile = Get-Content -LiteralPath $profilePath -Raw | ConvertFrom-Json
|
|
if ($profile.schema_version -ne 'missioncore.e46h-full-rectified-front-replay-profile/v1' -or
|
|
$profile.source.camera_source_id -ne 'sensor.camera.right' -or
|
|
$profile.rectification.view -ne 'front' -or
|
|
$profile.detector.name -ne 'NVIDIA TrafficCamNet Transformer Lite') {
|
|
throw 'E46H profile is incompatible.'
|
|
}
|
|
$image = [string]$profile.runtime.container_image
|
|
$imageDigestMatch = [regex]::Match($image, '@sha256:([0-9a-f]{64})$')
|
|
if (-not $imageDigestMatch.Success) {
|
|
throw 'E46H runtime image must be pinned by a full SHA-256 digest.'
|
|
}
|
|
$imageDigest = $imageDigestMatch.Groups[1].Value
|
|
$streamSha = [string]$profile.source.stream_sha256
|
|
$frameCount = [int]$profile.selection.frame_count
|
|
if ($frameCount -ne 4488 -or
|
|
[int]$profile.selection.first_source_frame_index -ne 0 -or
|
|
[int]$profile.selection.last_source_frame_index -ne 4487) {
|
|
throw 'E46H retained route contract changed.'
|
|
}
|
|
$parserPath = Join-Path $package "runtime\$([string]$profile.parser.library_file)"
|
|
Assert-Sha256 $parserPath ([string]$profile.parser.library_sha256) `
|
|
'official NVIDIA DeepStream TAO parser'
|
|
|
|
if (-not (Get-Command ffmpeg -ErrorAction SilentlyContinue) -or
|
|
-not (Get-Command ffprobe -ErrorAction SilentlyContinue)) {
|
|
throw 'E46H requires the existing Worker ffmpeg/ffprobe installation.'
|
|
}
|
|
|
|
New-Item -ItemType Directory -Force -Path $RuntimeRoot | Out-Null
|
|
$inputsRoot = Join-Path $RuntimeRoot 'inputs'
|
|
$runsRoot = Join-Path $RuntimeRoot 'runs'
|
|
$resultsRoot = Join-Path $RuntimeRoot 'full-rectified-front-results'
|
|
foreach ($path in @($inputsRoot, $runsRoot, $resultsRoot)) {
|
|
New-Item -ItemType Directory -Force -Path $path | Out-Null
|
|
}
|
|
if ($LogPath) {
|
|
$logParent = Split-Path -Parent $LogPath
|
|
if ($logParent) { New-Item -ItemType Directory -Force -Path $logParent | Out-Null }
|
|
Start-Transcript -LiteralPath $LogPath -Append | Out-Null
|
|
}
|
|
|
|
try {
|
|
$jobPath = Join-Path $sourceJob 'job.json'
|
|
$job = Get-Content -LiteralPath $jobPath -Raw | ConvertFrom-Json
|
|
if ($job.job_id -ne $profile.source.job_id -or
|
|
$job.input.archive_index_sha256 -ne $profile.source.archive_index_sha256 -or
|
|
$job.input.archive_summary_sha256 -ne $profile.source.archive_summary_sha256) {
|
|
throw 'Exact E46H source job binding changed.'
|
|
}
|
|
$inputPath = Join-Path $inputsRoot "right-$streamSha.mp4"
|
|
$e46gInput = "D:\NDC_MISSIONCORE\runtime\experiments\e46g\inputs\right-$streamSha.mp4"
|
|
$e46eInput = "D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs\right-$streamSha.mp4"
|
|
if (Test-Path -LiteralPath $inputPath -PathType Leaf) {
|
|
Assert-Sha256 $inputPath $streamSha 'E46H controlled RIGHT stream'
|
|
}
|
|
elseif (Test-Path -LiteralPath $e46gInput -PathType Leaf) {
|
|
Assert-Sha256 $e46gInput $streamSha 'E46G controlled RIGHT stream'
|
|
Copy-Item -LiteralPath $e46gInput -Destination $inputPath
|
|
}
|
|
elseif (Test-Path -LiteralPath $e46eInput -PathType Leaf) {
|
|
Assert-Sha256 $e46eInput $streamSha 'E46E controlled RIGHT stream'
|
|
Copy-Item -LiteralPath $e46eInput -Destination $inputPath
|
|
}
|
|
else {
|
|
& (Join-Path $package 'runtime\Prepare-RectifiedCameraReplay.ps1') `
|
|
-JobRoot $sourceJob -OutputPath $inputPath
|
|
}
|
|
Assert-Sha256 $inputPath $streamSha 'E46H reconstructed RIGHT stream'
|
|
|
|
$modelRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46e\models\trafficcamnet_transformer_lite\deployable_resnet50_v2.0'
|
|
$modelPath = Ensure-Model $profile.detector $modelRoot
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = 'Continue'
|
|
try {
|
|
& docker image inspect $image *> $null
|
|
$imageCached = $LASTEXITCODE -eq 0
|
|
}
|
|
finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
if (-not $imageCached) {
|
|
Invoke-Docker @('pull', $image) 'DeepStream image pull'
|
|
}
|
|
|
|
$runId = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ')
|
|
$runRoot = Join-Path $runsRoot $runId
|
|
$rawRoot = Join-Path $runRoot 'raw'
|
|
$sourceInputMount = Join-Path $runRoot 'source-input'
|
|
$geometryRoot = Join-Path $rawRoot 'geometry'
|
|
$runOutput = Join-Path $rawRoot 'run'
|
|
$frontInputMount = Join-Path $runRoot 'front-input'
|
|
foreach ($path in @(
|
|
$rawRoot,
|
|
$sourceInputMount,
|
|
$geometryRoot,
|
|
$runOutput,
|
|
$frontInputMount,
|
|
(Join-Path $runOutput 'detections'),
|
|
(Join-Path $runOutput 'tracks')
|
|
)) {
|
|
New-Item -ItemType Directory -Force -Path $path | Out-Null
|
|
}
|
|
Copy-Item -LiteralPath $inputPath -Destination (Join-Path $sourceInputMount 'right.mp4')
|
|
$workerLog = Join-Path $rawRoot 'worker.log'
|
|
"E46H run $runId`nsource=$streamSha`nselection=0..4487`nview=front" |
|
|
Set-Content -LiteralPath $workerLog -Encoding UTF8
|
|
$startedAt = (Get-Date).ToUniversalTime().ToString('o')
|
|
|
|
$dewarperConfig = Join-Path $package "runtime\$([string]$profile.rectification.config_file)"
|
|
Assert-Sha256 $dewarperConfig ([string]$profile.rectification.config_sha256) `
|
|
'FRONT dewarper config'
|
|
$frontPath = Join-Path $geometryRoot 'front.mp4'
|
|
$dewarperLog = Join-Path $geometryRoot 'front.log'
|
|
$dewarperCommand = @"
|
|
set -euo pipefail
|
|
gst-launch-1.0 -e filesrc location=/workspace/input/right.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=RGBA' ! nvdewarper config-file=/workspace/package/runtime/$([string]$profile.rectification.config_file) source-id=0 num-batch-buffers=1 ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=NV12' ! nvv4l2h264enc bitrate=6000000 ! h264parse ! qtmux ! filesink location=/workspace/output/front.mp4
|
|
"@
|
|
$dewarperArguments = @(
|
|
'run', '--rm', '--name', "ndc-mission-core-e46h-dewarper-$runId",
|
|
'--gpus', 'all', '--network', 'none', '--cap-drop', 'ALL',
|
|
'--security-opt', 'no-new-privileges', '--shm-size', '4g',
|
|
'--label', 'com.nodedc.product=mission-core',
|
|
'--label', 'com.nodedc.stack=perception',
|
|
'--label', 'com.nodedc.role=nvdewarper-e46h-front',
|
|
'--mount', "type=bind,src=$sourceInputMount,dst=/workspace/input,readonly",
|
|
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
|
'--mount', "type=bind,src=$geometryRoot,dst=/workspace/output",
|
|
'--entrypoint', '/bin/bash', $image, '-lc', $dewarperCommand
|
|
)
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = 'Continue'
|
|
try {
|
|
& docker @dewarperArguments 2>&1 | Tee-Object -LiteralPath $dewarperLog
|
|
$dewarperExit = $LASTEXITCODE
|
|
}
|
|
finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
if ($dewarperExit -ne 0 -or -not (Test-Path -LiteralPath $frontPath -PathType Leaf)) {
|
|
throw "NVIDIA nvdewarper failed with exit code $dewarperExit"
|
|
}
|
|
if ((Get-VideoFrameCount $frontPath) -ne $frameCount) {
|
|
throw 'E46H FRONT frame coverage changed.'
|
|
}
|
|
$normalizedFront = Join-Path $geometryRoot 'front-normalized.mp4'
|
|
& ffmpeg -hide_banner -loglevel error -y -fflags +genpts -i $frontPath `
|
|
-vf 'setpts=N/(10*TB)' -an -c:v libx264 -preset fast -crf 18 `
|
|
-pix_fmt yuv420p -r 10 -movflags +faststart $normalizedFront
|
|
if ($LASTEXITCODE -ne 0 -or (Get-VideoFrameCount $normalizedFront) -ne $frameCount) {
|
|
throw 'E46H FRONT timestamp normalization failed.'
|
|
}
|
|
Move-Item -LiteralPath $normalizedFront -Destination $frontPath -Force
|
|
Copy-Item -LiteralPath $frontPath -Destination (Join-Path $frontInputMount 'view.mp4')
|
|
|
|
$appConfig = Join-Path $package "runtime\$([string]$profile.detector.deepstream_app_config)"
|
|
$detectorConfig = Join-Path $package "runtime\$([string]$profile.detector.detector_config)"
|
|
Assert-Sha256 $appConfig ([string]$profile.detector.deepstream_app_config_sha256) `
|
|
'TrafficCamNet DeepStream app config'
|
|
Assert-Sha256 $detectorConfig ([string]$profile.detector.detector_config_sha256) `
|
|
'TrafficCamNet detector config'
|
|
$deepstreamLog = Join-Path $runOutput 'deepstream.log'
|
|
$deepstreamCommand = @"
|
|
set -euo pipefail
|
|
cp /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml /workspace/output/tracker-config.yml
|
|
deepstream-app -c /workspace/package/runtime/$([string]$profile.detector.deepstream_app_config)
|
|
"@
|
|
$deepstreamArguments = @(
|
|
'run', '--rm', '--name', "ndc-mission-core-e46h-front-$runId",
|
|
'--gpus', 'all', '--network', 'none', '--cap-drop', 'ALL',
|
|
'--security-opt', 'no-new-privileges', '--shm-size', '4g',
|
|
'--label', 'com.nodedc.product=mission-core',
|
|
'--label', 'com.nodedc.stack=perception',
|
|
'--label', 'com.nodedc.role=e46h-front-trafficcamnet',
|
|
'--mount', "type=bind,src=$frontInputMount,dst=/workspace/input,readonly",
|
|
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
|
'--mount', "type=bind,src=$modelRoot,dst=/workspace/model",
|
|
'--mount', "type=bind,src=$runOutput,dst=/workspace/output",
|
|
'--entrypoint', '/bin/bash', $image, '-lc', $deepstreamCommand
|
|
)
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = 'Continue'
|
|
try {
|
|
& docker @deepstreamArguments 2>&1 | Tee-Object -LiteralPath $deepstreamLog
|
|
$deepstreamExit = $LASTEXITCODE
|
|
}
|
|
finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
if ($deepstreamExit -ne 0) {
|
|
throw "DeepStream failed with exit code $deepstreamExit"
|
|
}
|
|
$overlayPath = Join-Path $runOutput 'overlay.mp4'
|
|
$trackerPath = Join-Path $runOutput 'tracker-config.yml'
|
|
$detections = @(Get-ChildItem -LiteralPath (Join-Path $runOutput 'detections') -File)
|
|
$tracks = @(Get-ChildItem -LiteralPath (Join-Path $runOutput 'tracks') -File)
|
|
if (-not (Test-Path -LiteralPath $overlayPath -PathType Leaf) -or
|
|
$detections.Count -ne $frameCount -or $tracks.Count -ne $frameCount) {
|
|
throw "DeepStream output coverage changed: detections=$($detections.Count), tracks=$($tracks.Count)"
|
|
}
|
|
$fastOverlay = Join-Path $runOutput 'overlay-faststart.mp4'
|
|
& ffmpeg -hide_banner -loglevel error -y -i $overlayPath -c copy -movflags +faststart $fastOverlay
|
|
if ($LASTEXITCODE -ne 0 -or (Get-VideoFrameCount $fastOverlay) -ne $frameCount) {
|
|
throw 'E46H fast-start overlay normalization failed.'
|
|
}
|
|
Move-Item -LiteralPath $fastOverlay -Destination $overlayPath -Force
|
|
$enginePath = "$modelPath`_b1_gpu0_fp16.engine"
|
|
if (-not (Test-Path -LiteralPath $enginePath -PathType Leaf)) {
|
|
throw 'TrafficCamNet TensorRT engine is missing.'
|
|
}
|
|
|
|
$runtime = [ordered]@{
|
|
schema_version = 'missioncore.e46h-full-rectified-front-runtime/v1'
|
|
status = 'completed'
|
|
worker_host = $env:COMPUTERNAME
|
|
gpu_name = ((& nvidia-smi --query-gpu=name --format=csv,noheader | Select-Object -First 1).Trim())
|
|
started_at_utc = $startedAt
|
|
completed_at_utc = (Get-Date).ToUniversalTime().ToString('o')
|
|
container_image = $image
|
|
container_image_digest = $imageDigest
|
|
source_stream_sha256 = Get-Sha256 $inputPath
|
|
frame_count = $frameCount
|
|
retained_source_frame_index_range = @(0, 4487)
|
|
geometry = [ordered]@{
|
|
video_path = 'geometry/front.mp4'
|
|
video_sha256 = Get-Sha256 $frontPath
|
|
log_path = 'geometry/front.log'
|
|
log_sha256 = Get-Sha256 $dewarperLog
|
|
config_sha256 = Get-Sha256 $dewarperConfig
|
|
frame_count = $frameCount
|
|
dewarper_exit_code = $dewarperExit
|
|
}
|
|
run = [ordered]@{
|
|
overlay_path = 'run/overlay.mp4'
|
|
overlay_sha256 = Get-Sha256 $overlayPath
|
|
deepstream_log_path = 'run/deepstream.log'
|
|
deepstream_log_sha256 = Get-Sha256 $deepstreamLog
|
|
model_sha256 = Get-Sha256 $modelPath
|
|
model_engine_sha256 = Get-Sha256 $enginePath
|
|
parser_library_sha256 = Get-Sha256 $parserPath
|
|
deepstream_app_config_sha256 = Get-Sha256 $appConfig
|
|
detector_config_sha256 = Get-Sha256 $detectorConfig
|
|
tracker_config_sha256 = Get-Sha256 $trackerPath
|
|
frame_count = $frameCount
|
|
deepstream_exit_code = $deepstreamExit
|
|
}
|
|
}
|
|
$runtime | ConvertTo-Json -Depth 12 |
|
|
Set-Content -LiteralPath (Join-Path $rawRoot 'runtime.json') -Encoding UTF8
|
|
Add-Content -LiteralPath $workerLog -Value "completed=$(Get-Date -Format o)"
|
|
|
|
$consolidatorImage = 'nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794'
|
|
$consolidatorArguments = @(
|
|
'run', '--rm', '--name', "ndc-mission-core-e46h-consolidator-$runId",
|
|
'--network', 'none', '--read-only', '--cap-drop', 'ALL',
|
|
'--security-opt', 'no-new-privileges', '--tmpfs', '/tmp:rw,noexec,nosuid,size=64m',
|
|
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
|
'--mount', "type=bind,src=$sourceJob,dst=/workspace/source-job,readonly",
|
|
'--mount', "type=bind,src=$rawRoot,dst=/workspace/raw,readonly",
|
|
'--mount', "type=bind,src=$resultsRoot,dst=/workspace/results",
|
|
'-e', 'PYTHONPATH=/workspace/package/runtime',
|
|
'-e', 'PYTHONDONTWRITEBYTECODE=1',
|
|
$consolidatorImage,
|
|
'python3', '/workspace/package/runtime/run_e46h_full_rectified_front_replay.py',
|
|
'--source-job', '/workspace/source-job',
|
|
'--raw-root', '/workspace/raw',
|
|
'--profile', '/workspace/package/profile.json',
|
|
'--output-root', '/workspace/results'
|
|
)
|
|
Invoke-Docker $consolidatorArguments 'E46H consolidation'
|
|
Write-Host "E46H_FULL_RECTIFIED_FRONT_COMPLETED run=$runId results=$resultsRoot"
|
|
}
|
|
finally {
|
|
if ($LogPath) { Stop-Transcript | Out-Null }
|
|
}
|