252 lines
11 KiB
PowerShell
252 lines
11 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$EvaluationPack,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$E2Prelabels,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ValidFovRoot,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$RunnerPath,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$BaselineRunnerPath,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ProfilePath,
|
|
|
|
[ValidateRange(0, 64)]
|
|
[int]$MaxFrames = 0,
|
|
|
|
[ValidateRange(1, 1000)]
|
|
[int]$FreeGiBFloor = 360,
|
|
|
|
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
|
|
|
|
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$ProgressPreference = "SilentlyContinue"
|
|
|
|
function Assert-LastExitCode {
|
|
param([string]$Operation)
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "$Operation failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
function Assert-RegularFile {
|
|
param([string]$Path, [string]$Label)
|
|
$item = Get-Item -LiteralPath $Path -Force
|
|
if (-not $item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
|
return $item.FullName
|
|
}
|
|
throw "$Label is not a regular file"
|
|
}
|
|
|
|
function Assert-Directory {
|
|
param([string]$Path, [string]$Label)
|
|
$item = Get-Item -LiteralPath $Path -Force
|
|
if ($item.PSIsContainer -and -not ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
|
return $item.FullName
|
|
}
|
|
throw "$Label is not a real directory"
|
|
}
|
|
|
|
function Assert-DDrivePath {
|
|
param([string]$Path, [string]$Label)
|
|
$fullPath = [IO.Path]::GetFullPath($Path)
|
|
if ([IO.Path]::GetPathRoot($fullPath).TrimEnd("\") -ine "D:") {
|
|
throw "$Label must be stored on D:"
|
|
}
|
|
return $fullPath
|
|
}
|
|
|
|
function Assert-FreeSpace {
|
|
param([string]$Phase)
|
|
$freeBytes = [int64](Get-PSDrive -Name D).Free
|
|
$floorBytes = [int64]$FreeGiBFloor * 1GB
|
|
$freeGiB = [math]::Round($freeBytes / 1GB, 3)
|
|
Write-Output ("DISK_GUARD PHASE={0} DRIVE=D FREE_GIB={1} FLOOR_GIB={2}" -f $Phase, $freeGiB, $FreeGiBFloor)
|
|
if ($freeBytes -lt $floorBytes) {
|
|
throw "D: free-space floor was crossed during $Phase"
|
|
}
|
|
}
|
|
|
|
function Convert-ToDockerPath {
|
|
param([string]$Path)
|
|
return $Path.Replace("\", "/")
|
|
}
|
|
|
|
$packRoot = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $EvaluationPack).Path "Evaluation pack") "Evaluation pack"
|
|
$prelabelsRoot = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $E2Prelabels).Path "E2 prelabels") "E2 prelabels"
|
|
$validFov = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $ValidFovRoot).Path "Valid-FOV root") "Valid-FOV root"
|
|
$runner = Assert-DDrivePath (Assert-RegularFile (Resolve-Path -LiteralPath $RunnerPath).Path "LAB E3 runner") "LAB E3 runner"
|
|
$baselineRunner = Assert-DDrivePath (Assert-RegularFile (Resolve-Path -LiteralPath $BaselineRunnerPath).Path "Baseline runner") "Baseline runner"
|
|
$profile = Assert-DDrivePath (Assert-RegularFile (Resolve-Path -LiteralPath $ProfilePath).Path "LAB E3 profile") "LAB E3 profile"
|
|
$runtime = Assert-DDrivePath (Assert-Directory (Resolve-Path -LiteralPath $RuntimeRoot).Path "Runtime root") "Runtime root"
|
|
if (
|
|
(Split-Path $runner -Parent) -ne (Split-Path $baselineRunner -Parent) -or
|
|
(Split-Path $runner -Parent) -ne (Split-Path $profile -Parent)
|
|
) {
|
|
throw "LAB E3 runner, baseline runner, and profile must share one read-only mount"
|
|
}
|
|
|
|
$pack = Get-Content -LiteralPath (Join-Path $packRoot "manifest.json") -Raw | ConvertFrom-Json
|
|
$prelabels = Get-Content -LiteralPath (Join-Path $prelabelsRoot "result.json") -Raw | ConvertFrom-Json
|
|
$profileDocument = Get-Content -LiteralPath $profile -Raw | ConvertFrom-Json
|
|
$packFrameCount = @($pack.identity.frames).Count
|
|
if (
|
|
$pack.schema_version -ne "missioncore.perception-evaluation-pack/v1" -or
|
|
$pack.generation_id -ne (Split-Path $packRoot -Leaf) -or
|
|
$pack.identity.source_id -ne "sensor.camera.right" -or
|
|
$pack.identity.calibration_slot -ne "camera_1" -or
|
|
$packFrameCount -ne 64 -or
|
|
$prelabels.schema_version -ne "missioncore.perception-evaluation-prelabels/v1" -or
|
|
$prelabels.identity.evaluation_pack_id -ne $pack.generation_id -or
|
|
$profileDocument.schema_version -ne "missioncore.k1-e3-rectified-segmentation-profile/v1" -or
|
|
$profileDocument.source.source_id -ne $pack.identity.source_id -or
|
|
$profileDocument.source.calibration_slot -ne $pack.identity.calibration_slot -or
|
|
$profileDocument.source.calibration_sha256 -ne $pack.identity.calibration_sha256
|
|
) {
|
|
throw "LAB E3 inputs are incompatible"
|
|
}
|
|
$null = Assert-RegularFile (Join-Path $validFov "manifest.json") "Valid-FOV manifest"
|
|
$null = Assert-RegularFile (Join-Path $validFov "mask.png") "Valid-FOV mask"
|
|
|
|
& docker image inspect $ContainerImage *> $null
|
|
Assert-LastExitCode "Existing container image inspection"
|
|
Assert-FreeSpace "preflight"
|
|
|
|
$cacheRoot = Join-Path $runtime "cache\perception-e3-models-v1"
|
|
$derivedRoot = Join-Path $runtime "derived\e3-segmentation"
|
|
$environmentRoot = Join-Path $runtime "derived\perception-e3-opencv413092-v1"
|
|
$torchEnvironment = Assert-Directory (Join-Path $runtime "derived\perception-p0-env-v1") "Torch environment"
|
|
$transformersEnvironment = Assert-Directory (Join-Path $runtime "derived\perception-p0-transformers4576-v1") "Transformers environment"
|
|
$runnerRoot = Split-Path $runner -Parent
|
|
$runnerName = Split-Path $runner -Leaf
|
|
$profileName = Split-Path $profile -Leaf
|
|
$null = New-Item -ItemType Directory -Path $cacheRoot -Force
|
|
$null = New-Item -ItemType Directory -Path $derivedRoot -Force
|
|
|
|
if (-not (Test-Path -LiteralPath $environmentRoot)) {
|
|
$environmentToken = [Guid]::NewGuid().ToString("N")
|
|
$environmentStaging = Join-Path (Split-Path $environmentRoot -Parent) (".e3-environment-{0}.publish" -f $environmentToken)
|
|
$null = New-Item -ItemType Directory -Path $environmentStaging
|
|
$null = New-Item -ItemType Directory -Path (Join-Path $environmentStaging "tmp")
|
|
try {
|
|
$prepareArgs = @(
|
|
"run", "--rm", "--network", "bridge", "--read-only",
|
|
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
|
"--pids-limit", "512", "--tmpfs", "/tmp:rw,noexec,nosuid,size=1g",
|
|
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
|
|
"-e", "HF_HOME=/cache/huggingface",
|
|
"-e", "PIP_NO_CACHE_DIR=1",
|
|
"-e", "HOME=/environment/tmp",
|
|
"-e", "TMPDIR=/environment/tmp",
|
|
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:rw"),
|
|
"-v", ((Convert-ToDockerPath $environmentStaging) + ":/environment:rw"),
|
|
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
|
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
|
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
|
"--entrypoint", "python3",
|
|
$ContainerImage,
|
|
("/runner/{0}" -f $runnerName), "prepare",
|
|
"--profile", ("/runner/{0}" -f $profileName),
|
|
"--cache", "/cache",
|
|
"--environment", "/environment"
|
|
)
|
|
Write-Output "PHASE=e3-dependency-prepare-start"
|
|
& docker @prepareArgs
|
|
Assert-LastExitCode "LAB E3 dependency preparation"
|
|
$null = Assert-RegularFile (Join-Path $environmentStaging "manifest.json") "LAB E3 dependency manifest"
|
|
Remove-Item -LiteralPath (Join-Path $environmentStaging "tmp") -Recurse -Force
|
|
Move-Item -LiteralPath $environmentStaging -Destination $environmentRoot
|
|
Write-Output "PHASE=e3-dependency-prepare-complete"
|
|
}
|
|
finally {
|
|
if (Test-Path -LiteralPath $environmentStaging) {
|
|
Remove-Item -LiteralPath $environmentStaging -Recurse -Force
|
|
}
|
|
}
|
|
}
|
|
$environment = Assert-Directory $environmentRoot "LAB E3 dependency environment"
|
|
$null = Assert-RegularFile (Join-Path $environment "manifest.json") "LAB E3 dependency manifest"
|
|
Assert-FreeSpace "post-dependency-prepare"
|
|
|
|
$runToken = [Guid]::NewGuid().ToString("N")
|
|
$publishRoot = Join-Path $derivedRoot (".{0}-{1}.publish" -f $pack.generation_id, $runToken)
|
|
$stagingRoot = Join-Path $publishRoot "output"
|
|
$null = New-Item -ItemType Directory -Path $publishRoot
|
|
|
|
try {
|
|
$runArgs = @(
|
|
"run", "--rm", "--gpus", "all", "--network", "none", "--read-only",
|
|
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
|
"--pids-limit", "512", "--shm-size", "4g",
|
|
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
|
|
"-e", "PYTHONPATH=/opt/transformers:/opt/env",
|
|
"-e", "TORCH_HOME=/cache/torch",
|
|
"-e", "HF_HOME=/cache/huggingface",
|
|
"-e", "HF_HUB_OFFLINE=1",
|
|
"-e", "TRANSFORMERS_OFFLINE=1",
|
|
"-e", "HOME=/tmp",
|
|
"-v", ((Convert-ToDockerPath $packRoot) + ":/evaluation-pack:ro"),
|
|
"-v", ((Convert-ToDockerPath $prelabelsRoot) + ":/e2-prelabels:ro"),
|
|
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
|
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
|
|
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"),
|
|
"-v", ((Convert-ToDockerPath $environment) + ":/environment:ro"),
|
|
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
|
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
|
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro"),
|
|
"--entrypoint", "python3",
|
|
$ContainerImage,
|
|
("/runner/{0}" -f $runnerName), "run",
|
|
"--profile", ("/runner/{0}" -f $profileName),
|
|
"--evaluation-pack", "/evaluation-pack",
|
|
"--e2-prelabels", "/e2-prelabels",
|
|
"--valid-fov-root", "/valid-fov",
|
|
"--cache", "/cache",
|
|
"--environment", "/environment",
|
|
"--output", "/publish/output",
|
|
"--telemetry-interval-seconds", "1"
|
|
)
|
|
if ($MaxFrames -gt 0) {
|
|
$runArgs += @("--max-frames", [string]$MaxFrames)
|
|
}
|
|
$expectedFrames = if ($MaxFrames -gt 0) { $MaxFrames } else { $packFrameCount }
|
|
Write-Output ("PHASE=e3-run-start PACK={0} FRAMES={1}" -f $pack.generation_id, $expectedFrames)
|
|
& docker @runArgs
|
|
Assert-LastExitCode "LAB E3 segmentation"
|
|
|
|
$result = Get-Content -LiteralPath (Join-Path $stagingRoot "result.json") -Raw | ConvertFrom-Json
|
|
if (
|
|
$result.schema_version -ne "missioncore.k1-e3-rectified-segmentation-result/v1" -or
|
|
$result.result_id -notmatch "^e3-segmentation-[a-f0-9]{64}$" -or
|
|
$result.identity.evaluation_pack_id -ne $pack.generation_id -or
|
|
[int]$result.identity.frame_count -ne $expectedFrames -or
|
|
$result.ground_truth -ne $false
|
|
) {
|
|
throw "LAB E3 result manifest is incompatible"
|
|
}
|
|
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
|
if (Test-Path -LiteralPath $finalRoot) {
|
|
throw "An immutable LAB E3 result with the same identity already exists: $finalRoot"
|
|
}
|
|
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
|
|
Remove-Item -LiteralPath $publishRoot -Force
|
|
Assert-FreeSpace "post-run"
|
|
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
|
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
|
}
|
|
finally {
|
|
if (Test-Path -LiteralPath $publishRoot) {
|
|
Remove-Item -LiteralPath $publishRoot -Recurse -Force
|
|
}
|
|
}
|