feat(perception): add recorded replay maturation labs
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,319 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$PackageRoot,
|
||||
|
||||
[string]$SourceJobRoot = 'D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d',
|
||||
|
||||
[string]$RuntimeRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46e',
|
||||
|
||||
[string]$LogPath = ''
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
|
||||
function Get-Sha256([string]$Path) {
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Assert-Sha256([string]$Path, [string]$Expected, [string]$Label) {
|
||||
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
||||
throw "$Label is missing: $Path"
|
||||
}
|
||||
$actual = Get-Sha256 $Path
|
||||
if ($actual -ne $Expected) {
|
||||
throw "$Label SHA-256 changed: expected $Expected, got $actual"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Docker([string[]]$Arguments, [string]$Label) {
|
||||
& docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Label failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
$package = (Resolve-Path -LiteralPath $PackageRoot).Path
|
||||
$sourceJob = (Resolve-Path -LiteralPath $SourceJobRoot).Path
|
||||
$profilePath = Join-Path $package 'profile.json'
|
||||
$manifestPath = Join-Path $package 'manifest.json'
|
||||
if (-not (Test-Path -LiteralPath $profilePath -PathType Leaf) -or
|
||||
-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) {
|
||||
throw 'E46E package is incomplete.'
|
||||
}
|
||||
$manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
|
||||
if ($manifest.schema_version -ne 'missioncore.e46e-worker-package/v1' -or
|
||||
$manifest.package_id -ne (Split-Path -Leaf $package)) {
|
||||
throw 'E46E package identity is invalid.'
|
||||
}
|
||||
$expectedPaths = @($manifest.identity.artifact_paths)
|
||||
foreach ($artifact in @($manifest.artifacts)) {
|
||||
if ($expectedPaths -notcontains [string]$artifact.path) {
|
||||
throw "Unexpected E46E package artifact: $($artifact.path)"
|
||||
}
|
||||
$artifactPath = Join-Path $package ([string]$artifact.path)
|
||||
Assert-Sha256 $artifactPath ([string]$artifact.sha256) "package artifact $($artifact.path)"
|
||||
if ((Get-Item -LiteralPath $artifactPath).Length -ne [int64]$artifact.byte_length) {
|
||||
throw "Package artifact length changed: $($artifact.path)"
|
||||
}
|
||||
}
|
||||
$actualPaths = @(Get-ChildItem -LiteralPath $package -Recurse -File | ForEach-Object {
|
||||
$_.FullName.Substring($package.Length + 1).Replace('\', '/')
|
||||
})
|
||||
if (@($actualPaths | Where-Object { $_ -ne 'manifest.json' -and $expectedPaths -notcontains $_ }).Count -ne 0 -or
|
||||
@($expectedPaths | Where-Object { $actualPaths -notcontains $_ }).Count -ne 0) {
|
||||
throw 'E46E package file set changed.'
|
||||
}
|
||||
|
||||
$profile = Get-Content -LiteralPath $profilePath -Raw | ConvertFrom-Json
|
||||
if ($profile.schema_version -ne 'missioncore.e46e-ready-stack-profile/v1') {
|
||||
throw 'E46E profile is incompatible.'
|
||||
}
|
||||
$image = [string]$profile.runtime.container_image
|
||||
$imageDigestMatch = [regex]::Match($image, '@sha256:([0-9a-f]{64})$')
|
||||
if (-not $imageDigestMatch.Success) {
|
||||
throw 'E46E runtime image must be pinned by a full SHA-256 digest.'
|
||||
}
|
||||
$imageDigest = $imageDigestMatch.Groups[1].Value
|
||||
$modelSha = [string]$profile.detector.model_sha256
|
||||
$streamSha = [string]$profile.source.stream_sha256
|
||||
$modelFile = [string]$profile.detector.model_file
|
||||
$modelUrl = [string]$profile.detector.model_url
|
||||
$parserFile = [string]$profile.parser.library_file
|
||||
$parserSha = [string]$profile.parser.library_sha256
|
||||
$parserLibraryPath = Join-Path $package "runtime\$parserFile"
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $RuntimeRoot | Out-Null
|
||||
$logsRoot = Join-Path $RuntimeRoot 'logs'
|
||||
$modelsRoot = Join-Path $RuntimeRoot 'models\trafficcamnet_transformer_lite\deployable_resnet50_v2.0'
|
||||
$inputsRoot = Join-Path $RuntimeRoot 'inputs'
|
||||
$runsRoot = Join-Path $RuntimeRoot 'runs'
|
||||
$resultsRoot = Join-Path $RuntimeRoot 'ready-stack-results'
|
||||
foreach ($path in @($logsRoot, $modelsRoot, $inputsRoot, $runsRoot, $resultsRoot)) {
|
||||
New-Item -ItemType Directory -Force -Path $path | Out-Null
|
||||
}
|
||||
if ($LogPath) {
|
||||
$logParent = Split-Path -Parent $LogPath
|
||||
if ($logParent) { New-Item -ItemType Directory -Force -Path $logParent | Out-Null }
|
||||
Start-Transcript -LiteralPath $LogPath -Append | Out-Null
|
||||
}
|
||||
|
||||
try {
|
||||
Write-Host "E46E package: $($manifest.package_id)"
|
||||
Write-Host "E46E source: $sourceJob"
|
||||
Write-Host "E46E image: $image"
|
||||
Assert-Sha256 $parserLibraryPath $parserSha 'official NVIDIA DeepStream TAO parser'
|
||||
|
||||
$modelPath = Join-Path $modelsRoot $modelFile
|
||||
if (Test-Path -LiteralPath $modelPath -PathType Leaf) {
|
||||
Assert-Sha256 $modelPath $modelSha 'TrafficCamNet Transformer Lite model'
|
||||
}
|
||||
else {
|
||||
$modelTemp = "$modelPath.$([Guid]::NewGuid().ToString('N')).download"
|
||||
Write-Host 'Downloading exact NVIDIA TrafficCamNet Transformer Lite model...'
|
||||
& curl.exe --fail --location --retry 3 --output $modelTemp $modelUrl
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "NVIDIA model download failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
Assert-Sha256 $modelTemp $modelSha 'downloaded model'
|
||||
Move-Item -LiteralPath $modelTemp -Destination $modelPath
|
||||
}
|
||||
|
||||
$jobPath = Join-Path $sourceJob 'job.json'
|
||||
$job = Get-Content -LiteralPath $jobPath -Raw | ConvertFrom-Json
|
||||
if ($job.job_id -ne $profile.source.job_id -or
|
||||
$job.input.archive_index_sha256 -ne $profile.source.archive_index_sha256 -or
|
||||
$job.input.archive_summary_sha256 -ne $profile.source.archive_summary_sha256) {
|
||||
throw 'Exact E46E source job binding changed.'
|
||||
}
|
||||
$cameraRoot = Join-Path $sourceJob 'input\camera\sensor.camera.right\epoch-1'
|
||||
$indexPath = Join-Path $cameraRoot 'index.jsonl'
|
||||
$summaryPath = Join-Path $cameraRoot 'summary.json'
|
||||
Assert-Sha256 $indexPath ([string]$profile.source.archive_index_sha256) 'source index'
|
||||
Assert-Sha256 $summaryPath ([string]$profile.source.archive_summary_sha256) 'source summary'
|
||||
$summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json
|
||||
if ($summary.stream_sha256 -ne $streamSha -or
|
||||
[int]$summary.segment_count -ne [int]$profile.source.segment_count) {
|
||||
throw 'Source stream identity changed.'
|
||||
}
|
||||
|
||||
$inputPath = Join-Path $inputsRoot "right-$streamSha.mp4"
|
||||
if (Test-Path -LiteralPath $inputPath -PathType Leaf) {
|
||||
Assert-Sha256 $inputPath $streamSha 'reconstructed RIGHT stream'
|
||||
}
|
||||
else {
|
||||
$inputTemp = "$inputPath.$([Guid]::NewGuid().ToString('N')).tmp"
|
||||
$destinationStream = [System.IO.File]::Open(
|
||||
$inputTemp,
|
||||
[System.IO.FileMode]::CreateNew,
|
||||
[System.IO.FileAccess]::Write,
|
||||
[System.IO.FileShare]::None
|
||||
)
|
||||
$incremental = [System.Security.Cryptography.IncrementalHash]::CreateHash(
|
||||
[System.Security.Cryptography.HashAlgorithmName]::SHA256
|
||||
)
|
||||
try {
|
||||
$sourceParts = [System.Collections.Generic.List[string]]::new()
|
||||
$sourceParts.Add((Join-Path $cameraRoot 'init.mp4'))
|
||||
foreach ($line in [System.IO.File]::ReadLines($indexPath)) {
|
||||
$row = $line | ConvertFrom-Json
|
||||
$sourceParts.Add((Join-Path $cameraRoot ([string]$row.path)))
|
||||
}
|
||||
if ($sourceParts.Count -ne ([int]$profile.source.segment_count + 1)) {
|
||||
throw 'Source stream part count changed.'
|
||||
}
|
||||
$buffer = New-Object byte[] (4MB)
|
||||
foreach ($part in $sourceParts) {
|
||||
$inputStream = [System.IO.File]::OpenRead($part)
|
||||
try {
|
||||
while (($read = $inputStream.Read($buffer, 0, $buffer.Length)) -gt 0) {
|
||||
$destinationStream.Write($buffer, 0, $read)
|
||||
$incremental.AppendData($buffer, 0, $read)
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$inputStream.Dispose()
|
||||
}
|
||||
}
|
||||
$destinationStream.Flush($true)
|
||||
$actualStreamSha = ([BitConverter]::ToString(
|
||||
$incremental.GetHashAndReset()
|
||||
)).Replace('-', '').ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$incremental.Dispose()
|
||||
$destinationStream.Dispose()
|
||||
}
|
||||
if ($actualStreamSha -ne $streamSha) {
|
||||
throw "Reconstructed stream SHA-256 changed: $actualStreamSha"
|
||||
}
|
||||
Move-Item -LiteralPath $inputTemp -Destination $inputPath
|
||||
}
|
||||
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& docker image inspect $image *> $null
|
||||
$imageCached = $LASTEXITCODE -eq 0
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($imageCached) {
|
||||
Write-Host 'Using the exact cached NVIDIA DeepStream image.'
|
||||
}
|
||||
else {
|
||||
Write-Host 'Pulling exact NVIDIA DeepStream image in the interactive user session...'
|
||||
Invoke-Docker @('pull', $image) 'DeepStream image pull'
|
||||
}
|
||||
|
||||
$runId = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ')
|
||||
$runRoot = Join-Path $runsRoot $runId
|
||||
$rawRoot = Join-Path $runRoot 'raw'
|
||||
$inputMount = Join-Path $runRoot 'input'
|
||||
New-Item -ItemType Directory -Force -Path $rawRoot | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path (Join-Path $rawRoot 'detections') | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path (Join-Path $rawRoot 'tracks') | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path $inputMount | Out-Null
|
||||
Copy-Item -LiteralPath $inputPath -Destination (Join-Path $inputMount 'right.mp4')
|
||||
$deepstreamLog = Join-Path $rawRoot 'deepstream.log'
|
||||
$trackerCopy = Join-Path $rawRoot 'tracker-config.yml'
|
||||
$startedAt = (Get-Date).ToUniversalTime().ToString('o')
|
||||
|
||||
$containerCommand = @"
|
||||
set -euo pipefail
|
||||
cp /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml /workspace/output/tracker-config.yml
|
||||
deepstream-app -c /workspace/package/runtime/e46e_deepstream_app.txt
|
||||
"@
|
||||
$dockerArguments = @(
|
||||
'run', '--rm', '--name', "ndc-mission-core-deepstream-e46e-$runId",
|
||||
'--gpus', 'all', '--network', 'none', '--cap-drop', 'ALL',
|
||||
'--security-opt', 'no-new-privileges', '--shm-size', '4g',
|
||||
'--label', 'com.nodedc.product=mission-core',
|
||||
'--label', 'com.nodedc.stack=perception',
|
||||
'--label', 'com.nodedc.role=deepstream-ready-stack-e46e',
|
||||
'--label', 'com.nodedc.managed-by=mission-core-worker',
|
||||
'--mount', "type=bind,src=$inputMount,dst=/workspace/input,readonly",
|
||||
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
||||
'--mount', "type=bind,src=$modelsRoot,dst=/workspace/model",
|
||||
'--mount', "type=bind,src=$rawRoot,dst=/workspace/output",
|
||||
'--entrypoint', '/bin/bash', $image, '-lc', $containerCommand
|
||||
)
|
||||
Write-Host 'Running full 4489-frame NVIDIA detector + NvDCF replay...'
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& docker @dockerArguments 2>&1 | Tee-Object -LiteralPath $deepstreamLog
|
||||
$deepstreamExit = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($deepstreamExit -ne 0) {
|
||||
throw "DeepStream replay failed with exit code $deepstreamExit"
|
||||
}
|
||||
|
||||
$overlayPath = Join-Path $rawRoot 'overlay.mp4'
|
||||
$enginePath = Join-Path $modelsRoot "$modelFile`_b1_gpu0_fp16.engine"
|
||||
if (-not (Test-Path -LiteralPath $overlayPath -PathType Leaf) -or
|
||||
(Get-Item -LiteralPath $overlayPath).Length -eq 0) {
|
||||
throw 'DeepStream did not produce an overlay video.'
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $enginePath -PathType Leaf)) {
|
||||
throw 'DeepStream did not produce the exact TensorRT engine.'
|
||||
}
|
||||
$detectionFiles = @(Get-ChildItem -LiteralPath (Join-Path $rawRoot 'detections') -File)
|
||||
$trackFiles = @(Get-ChildItem -LiteralPath (Join-Path $rawRoot 'tracks') -File)
|
||||
if ($detectionFiles.Count -ne [int]$profile.source.segment_count -or
|
||||
$trackFiles.Count -ne [int]$profile.source.segment_count) {
|
||||
throw "DeepStream frame coverage changed: detections=$($detectionFiles.Count), tracks=$($trackFiles.Count)"
|
||||
}
|
||||
$runtime = [ordered]@{
|
||||
schema_version = 'missioncore.e46e-deepstream-runtime/v1'
|
||||
status = 'completed'
|
||||
worker_host = $env:COMPUTERNAME
|
||||
gpu_name = ((& nvidia-smi --query-gpu=name --format=csv,noheader | Select-Object -First 1).Trim())
|
||||
started_at_utc = $startedAt
|
||||
completed_at_utc = (Get-Date).ToUniversalTime().ToString('o')
|
||||
container_image = $image
|
||||
container_image_digest = $imageDigest
|
||||
model_sha256 = Get-Sha256 $modelPath
|
||||
model_engine_sha256 = Get-Sha256 $enginePath
|
||||
deepstream_config_sha256 = Get-Sha256 (Join-Path $package 'runtime\e46e_deepstream_app.txt')
|
||||
detector_config_sha256 = Get-Sha256 (Join-Path $package 'runtime\e46e_trafficcamnet_rtdetr.txt')
|
||||
parser_library_sha256 = Get-Sha256 $parserLibraryPath
|
||||
tracker_config_sha256 = Get-Sha256 $trackerCopy
|
||||
input_stream_sha256 = Get-Sha256 $inputPath
|
||||
overlay_sha256 = Get-Sha256 $overlayPath
|
||||
frame_count = [int]$profile.source.segment_count
|
||||
deepstream_exit_code = $deepstreamExit
|
||||
}
|
||||
$runtimePath = Join-Path $rawRoot 'runtime.json'
|
||||
$runtime | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $runtimePath -Encoding UTF8
|
||||
|
||||
$consolidatorImage = 'nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794'
|
||||
$consolidatorArguments = @(
|
||||
'run', '--rm', '--name', "ndc-mission-core-e46e-consolidator-$runId",
|
||||
'--network', 'none', '--read-only', '--cap-drop', 'ALL',
|
||||
'--security-opt', 'no-new-privileges', '--tmpfs', '/tmp:rw,noexec,nosuid,size=64m',
|
||||
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
||||
'--mount', "type=bind,src=$sourceJob,dst=/workspace/source-job,readonly",
|
||||
'--mount', "type=bind,src=$rawRoot,dst=/workspace/raw,readonly",
|
||||
'--mount', "type=bind,src=$resultsRoot,dst=/workspace/results",
|
||||
'-e', 'PYTHONPATH=/workspace/package/runtime',
|
||||
'-e', 'PYTHONDONTWRITEBYTECODE=1',
|
||||
$consolidatorImage,
|
||||
'python3', '/workspace/package/runtime/run_e46e_ready_stack.py',
|
||||
'--source-job', '/workspace/source-job',
|
||||
'--raw-root', '/workspace/raw',
|
||||
'--profile', '/workspace/package/profile.json',
|
||||
'--output-root', '/workspace/results'
|
||||
)
|
||||
Write-Host 'Freezing immutable E46E evidence...'
|
||||
Invoke-Docker $consolidatorArguments 'E46E consolidation'
|
||||
Write-Host "E46E_READY_STACK_COMPLETED run=$runId results=$resultsRoot"
|
||||
}
|
||||
finally {
|
||||
if ($LogPath) { Stop-Transcript | Out-Null }
|
||||
}
|
||||
Reference in New Issue
Block a user