313 lines
14 KiB
PowerShell
313 lines
14 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ReleaseRoot,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[a-f0-9]{64}$")]
|
|
[string]$ExpectedWheelSha256,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[A-Za-z0-9._-]{1,96}$")]
|
|
[string]$RunId,
|
|
[ValidateRange(0, 5000)]
|
|
[int]$MaximumImages = 0,
|
|
[string]$DatasetRoot = "D:\NDC_MISSIONCORE\datasets\coco-2017-val",
|
|
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m48t-upstream-parity"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$ProgressPreference = "SilentlyContinue"
|
|
|
|
function Assert-LastExitCode([string]$Operation) {
|
|
if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" }
|
|
}
|
|
|
|
function Get-Sha256([string]$Path) {
|
|
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
}
|
|
|
|
function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) {
|
|
if ($Create -and -not (Test-Path -LiteralPath $Path)) {
|
|
$null = New-Item -ItemType Directory -Path $Path
|
|
}
|
|
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
|
if (
|
|
-not $item.PSIsContainer -or
|
|
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
|
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
|
|
) {
|
|
throw "$Label must be a real D: directory"
|
|
}
|
|
return $item.FullName
|
|
}
|
|
|
|
function Assert-RegularFile([string]$Path, [string]$Label) {
|
|
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
|
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
|
throw "$Label must be a regular file"
|
|
}
|
|
return $item.FullName
|
|
}
|
|
|
|
function Convert-ToDockerPath([string]$Path) { return $Path.Replace("\", "/") }
|
|
|
|
function Get-Container([string]$Name) {
|
|
$rows = @((& docker inspect $Name) | ConvertFrom-Json)
|
|
Assert-LastExitCode "Docker inspection for $Name"
|
|
if ($rows.Count -ne 1) { throw "Container identity for $Name is not unique" }
|
|
return $rows[0]
|
|
}
|
|
|
|
if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
|
|
throw "M48T upstream parity is pinned to DESKTOP-OPJ8J04"
|
|
}
|
|
|
|
$release = Resolve-DDirectory $ReleaseRoot "M48T parity release root" $false
|
|
$dataset = Resolve-DDirectory $DatasetRoot "M48T parity dataset root" $false
|
|
$output = Resolve-DDirectory $OutputRoot "M48T parity output root" $true
|
|
$runOutput = Join-Path $output $RunId
|
|
if (Test-Path -LiteralPath $runOutput) { throw "M48T parity output already exists" }
|
|
$null = New-Item -ItemType Directory -Path $runOutput
|
|
$runOutput = Resolve-DDirectory $runOutput "M48T parity run output" $false
|
|
|
|
$wheel = Assert-RegularFile (Join-Path $release "nodedc_mission_core-0.1.0-py3-none-any.whl") "wheel"
|
|
if ((Get-Sha256 $wheel) -cne $ExpectedWheelSha256) { throw "wheel SHA-256 changed" }
|
|
$profile = Assert-RegularFile (Join-Path $release "m48t-upstream-parity-v1.json") "profile"
|
|
$runner = Assert-RegularFile (Join-Path $release "run_m48t_upstream_parity_worker.py") "runner"
|
|
$annotations = Assert-RegularFile (Join-Path $dataset "instances_val2017.json") "COCO annotations"
|
|
$imagesRoot = Resolve-DDirectory (Join-Path $dataset "val2017") "COCO val2017 images" $false
|
|
if (@(Get-ChildItem -LiteralPath $imagesRoot -File -Filter "*.jpg").Count -ne 5000) {
|
|
throw "COCO val2017 image count changed"
|
|
}
|
|
|
|
$experimentRoot = Resolve-DDirectory (
|
|
"D:\NDC_MISSIONCORE\runtime\experiments\m48t-fixed-detector-20260825T095425Z"
|
|
) "RF-DETR experiment root" $false
|
|
$checkpoint = Assert-RegularFile (Join-Path $experimentRoot "weights\rf-detr-large-2026.pth") "checkpoint"
|
|
if ((Get-Sha256 $checkpoint) -cne "0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38") {
|
|
throw "RF-DETR checkpoint SHA-256 changed"
|
|
}
|
|
$modelRoot = Resolve-DDirectory (Join-Path $experimentRoot "triton-models") "model root" $false
|
|
$engine = Assert-RegularFile (Join-Path $modelRoot "rf_detr_large\1\model.plan") "TensorRT engine"
|
|
if ((Get-Sha256 $engine) -cne "986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8") {
|
|
throw "RF-DETR TensorRT engine SHA-256 changed"
|
|
}
|
|
|
|
$historical = Get-Container "ndc-mission-core-triton"
|
|
if (-not $historical.State.Running -or $historical.State.Health.Status -cne "healthy") {
|
|
throw "Historical Triton must remain healthy"
|
|
}
|
|
$historicalId = [string]$historical.Id
|
|
$baseImage = "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
|
|
& docker image inspect $baseImage *> $null
|
|
Assert-LastExitCode "pinned base image inspection"
|
|
$baseImageId = (& docker image inspect $baseImage --format "{{.Id}}").Trim()
|
|
Assert-LastExitCode "pinned base image identity"
|
|
$runtimeVolume = "ndc-mission-core-m48t-upstream-parity-env"
|
|
if (-not (& docker volume ls --quiet --filter "name=^$runtimeVolume$")) {
|
|
& docker volume create `
|
|
--label "com.nodedc.product=mission-core" `
|
|
--label "com.nodedc.stack=ndc-mission-core-compute" `
|
|
--label "com.nodedc.role=bounded-rf-detr-upstream-parity" `
|
|
--label "com.nodedc.managed-by=codex-bounded-experiment" `
|
|
$runtimeVolume *> $null
|
|
Assert-LastExitCode "M48T upstream parity dependency volume creation"
|
|
}
|
|
$torchReady = $false
|
|
$strictErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
& docker run --rm `
|
|
--read-only `
|
|
--security-opt "no-new-privileges:true" `
|
|
--cap-drop ALL `
|
|
--tmpfs "/tmp:rw,noexec,nosuid,size=512m" `
|
|
-e "PYTHONPATH=/opt/parity" `
|
|
-v ($runtimeVolume + ":/opt/parity:ro") `
|
|
--entrypoint python3 `
|
|
$baseImage `
|
|
-c "import importlib.metadata as m; assert m.version('numpy') == '1.26.4'; assert m.version('torch') == '2.9.1+cu130'; assert m.version('torchvision') == '0.24.1+cu130'" *> $null
|
|
if ($LASTEXITCODE -eq 0) { $torchReady = $true }
|
|
$ErrorActionPreference = $strictErrorActionPreference
|
|
if (-not $torchReady) {
|
|
& docker run --rm `
|
|
--name "ndc-mission-core-m48t-upstream-parity-env-torch" `
|
|
--read-only `
|
|
--security-opt "no-new-privileges:true" `
|
|
--cap-drop ALL `
|
|
--pids-limit 512 `
|
|
--tmpfs "/tmp:rw,noexec,nosuid,size=8g" `
|
|
-e "PIP_DISABLE_PIP_VERSION_CHECK=1" `
|
|
-v ($runtimeVolume + ":/opt/parity:rw") `
|
|
--entrypoint python3 `
|
|
$baseImage `
|
|
-m pip install --no-cache-dir --target /opt/parity `
|
|
--index-url https://download.pytorch.org/whl/cu130 `
|
|
"numpy==1.26.4" "torch==2.9.1+cu130" "torchvision==0.24.1+cu130"
|
|
Assert-LastExitCode "M48T upstream parity PyTorch environment initialization"
|
|
}
|
|
$rfdetrReady = $false
|
|
$ErrorActionPreference = "Continue"
|
|
& docker run --rm `
|
|
--read-only `
|
|
--security-opt "no-new-privileges:true" `
|
|
--cap-drop ALL `
|
|
--tmpfs "/tmp:rw,noexec,nosuid,size=512m" `
|
|
-e "PYTHONPATH=/opt/parity" `
|
|
-v ($runtimeVolume + ":/opt/parity:ro") `
|
|
--entrypoint python3 `
|
|
$baseImage `
|
|
-c "import importlib.metadata as m; import rfdetr; assert m.version('rfdetr') == '1.9.4'; assert m.version('pycocotools') == '2.0.10'; assert m.version('tritonclient') == '2.71.0'" *> $null
|
|
if ($LASTEXITCODE -eq 0) { $rfdetrReady = $true }
|
|
$ErrorActionPreference = $strictErrorActionPreference
|
|
if (-not $rfdetrReady) {
|
|
& docker run --rm `
|
|
--name "ndc-mission-core-m48t-upstream-parity-env-dependencies" `
|
|
--read-only `
|
|
--security-opt "no-new-privileges:true" `
|
|
--cap-drop ALL `
|
|
--pids-limit 512 `
|
|
--tmpfs "/tmp:rw,noexec,nosuid,size=8g" `
|
|
-e "PIP_DISABLE_PIP_VERSION_CHECK=1" `
|
|
-e "PYTHONPATH=/opt/parity" `
|
|
-v ($runtimeVolume + ":/opt/parity:rw") `
|
|
--entrypoint python3 `
|
|
$baseImage `
|
|
-m pip install --no-cache-dir --upgrade --target /opt/parity `
|
|
"numpy==1.26.4" "requests" "tqdm" "transformers>=5.1.0,<6.0.0" `
|
|
"pydantic>=2.0,<3.0" "supervision>=0.29.0,<1.0" "pyDeprecate>=0.9,<0.10" `
|
|
"pycocotools==2.0.10" "tritonclient[http]==2.71.0"
|
|
Assert-LastExitCode "M48T upstream parity dependency initialization"
|
|
& docker run --rm `
|
|
--name "ndc-mission-core-m48t-upstream-parity-env-rfdetr" `
|
|
--read-only `
|
|
--security-opt "no-new-privileges:true" `
|
|
--cap-drop ALL `
|
|
--pids-limit 512 `
|
|
--tmpfs "/tmp:rw,noexec,nosuid,size=2g" `
|
|
-e "PIP_DISABLE_PIP_VERSION_CHECK=1" `
|
|
-e "PYTHONPATH=/opt/parity" `
|
|
-v ($runtimeVolume + ":/opt/parity:rw") `
|
|
--entrypoint python3 `
|
|
$baseImage `
|
|
-m pip install --no-cache-dir --no-deps --upgrade --target /opt/parity "rfdetr==1.9.4"
|
|
Assert-LastExitCode "M48T upstream parity RF-DETR package initialization"
|
|
}
|
|
& docker run --rm `
|
|
--read-only `
|
|
--security-opt "no-new-privileges:true" `
|
|
--cap-drop ALL `
|
|
--tmpfs "/tmp:rw,noexec,nosuid,size=512m" `
|
|
-e "PYTHONPATH=/opt/parity" `
|
|
-v ($runtimeVolume + ":/opt/parity:ro") `
|
|
--entrypoint python3 `
|
|
$baseImage `
|
|
-c "import importlib.metadata as m; import rfdetr, torch, torchvision; assert m.version('numpy') == '1.26.4'; assert m.version('rfdetr') == '1.9.4'; assert m.version('torch') == '2.9.1+cu130'; assert m.version('torchvision') == '0.24.1+cu130'; assert m.version('pycocotools') == '2.0.10'; assert m.version('tritonclient') == '2.71.0'"
|
|
Assert-LastExitCode "M48T upstream parity dependency volume verification"
|
|
$runtimeIdentity = $baseImageId + "+volume:" + $runtimeVolume
|
|
|
|
$tritonName = "ndc-mission-core-m48t-upstream-parity-triton"
|
|
$runnerName = "ndc-mission-core-m48t-upstream-parity"
|
|
foreach ($name in @($tritonName, $runnerName)) {
|
|
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
|
|
throw "M48T parity container $name already exists"
|
|
}
|
|
}
|
|
|
|
try {
|
|
& docker create `
|
|
--name $tritonName `
|
|
--read-only `
|
|
--security-opt "no-new-privileges:true" `
|
|
--cap-drop ALL `
|
|
--pids-limit 512 `
|
|
--shm-size 1g `
|
|
--gpus all `
|
|
--tmpfs "/tmp:rw,noexec,nosuid,size=2g" `
|
|
--health-cmd "curl --fail --silent http://127.0.0.1:8000/v2/health/ready" `
|
|
--health-interval 5s `
|
|
--health-timeout 3s `
|
|
--health-start-period 20s `
|
|
--health-retries 24 `
|
|
-v ((Convert-ToDockerPath $modelRoot) + ":/models:ro") `
|
|
$baseImage `
|
|
tritonserver `
|
|
--model-repository=/models `
|
|
--model-control-mode=explicit `
|
|
--load-model=rf_detr_large `
|
|
--disable-auto-complete-config `
|
|
--strict-readiness=true `
|
|
--exit-on-error=true `
|
|
--allow-http=true `
|
|
--allow-grpc=false `
|
|
--allow-metrics=false *> $null
|
|
Assert-LastExitCode "M48T parity Triton creation"
|
|
& docker start $tritonName *> $null
|
|
Assert-LastExitCode "M48T parity Triton start"
|
|
$ready = $false
|
|
foreach ($attempt in 1..60) {
|
|
Start-Sleep -Seconds 2
|
|
$candidate = Get-Container $tritonName
|
|
if (-not $candidate.State.Running) { throw "M48T parity Triton stopped during startup" }
|
|
if ($candidate.State.Health.Status -ceq "healthy") { $ready = $true; break }
|
|
}
|
|
if (-not $ready) { throw "M48T parity Triton did not become healthy" }
|
|
|
|
$maximumArguments = @()
|
|
if ($MaximumImages -gt 0) { $maximumArguments = @("--maximum-images", ([string]$MaximumImages)) }
|
|
$arguments = @(
|
|
"run", "--name", $runnerName,
|
|
"--network", ("container:{0}" -f $tritonName),
|
|
"--read-only",
|
|
"--security-opt", "no-new-privileges:true",
|
|
"--cap-drop", "ALL",
|
|
"--pids-limit", "512",
|
|
"--gpus", "all",
|
|
"--shm-size", "2g",
|
|
"--tmpfs", "/tmp:rw,noexec,nosuid,size=8g",
|
|
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
|
"-e", "PYTHONPATH=/opt/parity:/release/nodedc_mission_core-0.1.0-py3-none-any.whl",
|
|
"-v", ($runtimeVolume + ":/opt/parity:ro"),
|
|
"-v", ((Convert-ToDockerPath $release) + ":/release:ro"),
|
|
"-v", ((Convert-ToDockerPath $imagesRoot) + ":/dataset/val2017:ro"),
|
|
"-v", ((Convert-ToDockerPath $annotations) + ":/dataset/instances_val2017.json:ro"),
|
|
"-v", ((Convert-ToDockerPath $checkpoint) + ":/model/rf-detr-large-2026.pth:ro"),
|
|
"-v", ((Convert-ToDockerPath $runOutput) + ":/output:rw"),
|
|
"--entrypoint", "python3",
|
|
$baseImage,
|
|
"/release/run_m48t_upstream_parity_worker.py",
|
|
"--profile", "/release/m48t-upstream-parity-v1.json",
|
|
"--annotations", "/dataset/instances_val2017.json",
|
|
"--images-root", "/dataset/val2017",
|
|
"--checkpoint", "/model/rf-detr-large-2026.pth",
|
|
"--triton-origin", "http://127.0.0.1:8000",
|
|
"--output", "/output/result.json",
|
|
"--pytorch-predictions", "/output/pytorch-predictions.jsonl.gz",
|
|
"--tensorrt-predictions", "/output/tensorrt-predictions.jsonl.gz",
|
|
"--progress", "/output/progress.jsonl",
|
|
"--runtime-image", $runtimeIdentity
|
|
) + $maximumArguments
|
|
& docker @arguments
|
|
Assert-LastExitCode "M48T upstream parity evaluation"
|
|
if (-not (Test-Path -LiteralPath (Join-Path $runOutput "result.json") -PathType Leaf)) {
|
|
throw "M48T upstream parity result was not written"
|
|
}
|
|
} finally {
|
|
foreach ($name in @($runnerName, $tritonName)) {
|
|
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
|
|
& docker rm -f $name *> $null
|
|
}
|
|
}
|
|
$historicalAfter = Get-Container "ndc-mission-core-triton"
|
|
if (
|
|
$historicalAfter.Id -cne $historicalId -or
|
|
-not $historicalAfter.State.Running -or
|
|
$historicalAfter.State.Health.Status -cne "healthy"
|
|
) {
|
|
throw "Historical Triton changed during M48T upstream parity"
|
|
}
|
|
}
|
|
|
|
Write-Output ("M48T_UPSTREAM_PARITY_RESULT={0}" -f (Join-Path $runOutput "result.json"))
|
|
Write-Output ("RUNTIME_IDENTITY={0}" -f $runtimeIdentity)
|
|
Write-Output "HISTORICAL_TRITON_ACTION=none"
|
|
Write-Output "SEMANTIC_AUTHORITY_CHANGED=false"
|