489 lines
24 KiB
PowerShell
489 lines
24 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)] [string]$JobRoot,
|
|
[Parameter(Mandatory = $true)] [string]$RunnerPath,
|
|
[Parameter(Mandatory = $true)] [string]$LiveProfilePath,
|
|
[Parameter(Mandatory = $true)] [string]$E14ProfilePath,
|
|
[Parameter(Mandatory = $true)] [string]$DetectorProfilePath,
|
|
[Parameter(Mandatory = $true)] [string]$SemanticProfilePath,
|
|
[Parameter(Mandatory = $true)] [string]$ValidFovRoot,
|
|
[Parameter(Mandatory = $true)] [string]$ProjectionPackRoot,
|
|
[Parameter(Mandatory = $true)] [string]$PackageRoot,
|
|
[string]$StabilityProfilePath = "",
|
|
[switch]$PreflightOnly,
|
|
[switch]$PersistentService,
|
|
[switch]$TokenStdin,
|
|
[ValidateRange(5, 3600)] [int]$MaximumDurationSeconds = 20,
|
|
[ValidateRange(1, 1000)] [int]$FreeGiBFloor = 360,
|
|
[ValidateRange(1024, 65535)] [int]$SourcePort = 18012,
|
|
[string]$SourceHost = "host.docker.internal",
|
|
[string]$SourcePath = "/api/v1/device-plugins/nodedc.device.xgrids-lixelkity-k1/live-perception-shadow",
|
|
[string]$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime",
|
|
[string]$ModelRoot = "D:\NDC_MISSIONCORE\runtime\models\yolox_s",
|
|
[string]$MediaRuntimeRoot = "D:\NDC_MISSIONCORE\runtime\derived\perception-e15-media-pyav180-lz445-v1",
|
|
[string]$TritonContainer = "mission-core-triton",
|
|
[string]$PersistentContainer = "mission-core-perception-worker",
|
|
[string]$PersistentOutputRoot = "D:\NDC_MISSIONCORE\runtime\derived\.perception-persistent-publish",
|
|
[ValidateRange(1024, 65535)] [int]$PersistentPort = 18020,
|
|
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$ProgressPreference = "SilentlyContinue"
|
|
|
|
function Assert-LastExitCode([string]$Operation) {
|
|
if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" }
|
|
}
|
|
|
|
function Resolve-DDirectory([string]$Path, [string]$Label) {
|
|
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
|
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
|
if (-not $item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") {
|
|
throw "$Label must be a real D: directory"
|
|
}
|
|
return $item.FullName
|
|
}
|
|
|
|
function Resolve-DFile([string]$Path, [string]$Label) {
|
|
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
|
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
|
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:") {
|
|
throw "$Label must be a regular D: file"
|
|
}
|
|
return $item.FullName
|
|
}
|
|
|
|
function Convert-ToDockerPath([string]$Path) { return $Path.Replace("\", "/") }
|
|
function Get-DFreeBytes { return [int64](Get-PSDrive -Name D).Free }
|
|
|
|
function Assert-FreeSpace([string]$Phase, [int64]$RequiredAdditionalBytes = 0) {
|
|
$free = Get-DFreeBytes
|
|
$floor = [int64]$FreeGiBFloor * 1GB
|
|
Write-Host (
|
|
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3} REQUIRED_ADDITIONAL_GIB={4}" -f
|
|
$Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor,
|
|
[math]::Round($RequiredAdditionalBytes / 1GB, 3)
|
|
)
|
|
if ($free -lt ($floor + $RequiredAdditionalBytes)) {
|
|
throw "D: lacks the guarded LAB E15 reserve during $Phase"
|
|
}
|
|
return $free
|
|
}
|
|
|
|
function Test-TritonModelReady {
|
|
try {
|
|
$response = Invoke-WebRequest -UseBasicParsing -TimeoutSec 10 `
|
|
-Uri "http://127.0.0.1:8000/v2/models/yolox_s/ready"
|
|
return $response.StatusCode -eq 200
|
|
}
|
|
catch { return $false }
|
|
}
|
|
|
|
$jobDirectory = Resolve-DDirectory $JobRoot "Job root"
|
|
$runner = Resolve-DFile $RunnerPath "LAB E15 runner"
|
|
$liveProfile = Resolve-DFile $LiveProfilePath "LAB E15 live profile"
|
|
$e14Profile = Resolve-DFile $E14ProfilePath "Accepted E14 profile"
|
|
$detectorProfile = Resolve-DFile $DetectorProfilePath "Detector profile"
|
|
$semanticProfile = Resolve-DFile $SemanticProfilePath "Semantic profile"
|
|
$stabilityProfile = $null
|
|
if ($StabilityProfilePath) {
|
|
$stabilityProfile = Resolve-DFile $StabilityProfilePath "LAB E23 stability profile"
|
|
}
|
|
$validFov = Resolve-DDirectory $ValidFovRoot "Valid-FOV root"
|
|
$projectionPack = Resolve-DDirectory $ProjectionPackRoot "E15 projection pack"
|
|
$package = Resolve-DDirectory $PackageRoot "Mission Core package root"
|
|
$runtime = Resolve-DDirectory $RuntimeRoot "Runtime root"
|
|
$model = Resolve-DDirectory $ModelRoot "YOLOX model root"
|
|
$mediaRuntime = Resolve-DDirectory $MediaRuntimeRoot "E15 media runtime"
|
|
$runnerRoot = Split-Path $runner -Parent
|
|
|
|
foreach ($path in @($liveProfile, $e14Profile, $detectorProfile, $semanticProfile)) {
|
|
if ((Split-Path $path -Parent) -ne $runnerRoot) {
|
|
throw "Runner and profiles must share one immutable mount"
|
|
}
|
|
}
|
|
if ($stabilityProfile -and (Split-Path $stabilityProfile -Parent) -ne $runnerRoot) {
|
|
throw "Runner and LAB E23 stability profile must share one immutable mount"
|
|
}
|
|
foreach ($dependency in @(
|
|
"e10_fusion_runtime.py",
|
|
"e15_shadow_runtime.py",
|
|
"run_e10_integrated_perception.py",
|
|
"run_e12_shadow_transport_probe.py",
|
|
"run_e9_multirate_perception.py",
|
|
"run_e8_realtime_tracking.py",
|
|
"run_e5_instance_tracking.py",
|
|
"run_e4_full_session_segmentation.py",
|
|
"run_recorded_perception_epoch.py",
|
|
"run_e3_rectified_segmentation.py",
|
|
"run_evaluation_prelabels.py"
|
|
)) {
|
|
$null = Resolve-DFile (Join-Path $runnerRoot $dependency) "LAB E15 dependency"
|
|
}
|
|
if (-not (Test-Path -LiteralPath (Join-Path $package "k1link\compute\live_perception.py") -PathType Leaf)) {
|
|
throw "Mission Core package mount lacks live perception synchronization"
|
|
}
|
|
if (-not (Test-Path -LiteralPath (Join-Path $package "k1link\compute\inline_temporal.py") -PathType Leaf)) {
|
|
throw "Mission Core package mount lacks inline temporal state"
|
|
}
|
|
|
|
$job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json
|
|
if (
|
|
$job.schema_version -ne "missioncore.compute-job/v1" -or
|
|
$job.job_id -ne (Split-Path $jobDirectory -Leaf) -or
|
|
$job.input.source_id -ne "sensor.camera.right"
|
|
) { throw "Compute bootstrap job manifest is incompatible" }
|
|
$live = Get-Content -LiteralPath $liveProfile -Raw | ConvertFrom-Json
|
|
if (
|
|
$live.schema_version -ne "missioncore.e15-shadow-inference-profile/v1" -or
|
|
$live.mode -ne "replay-shadow-gate" -or
|
|
[bool]$live.authority.commands_enabled -or
|
|
[bool]$live.authority.navigation_or_safety_accepted -or
|
|
$live.transport.pyav_version -ne "18.0.0"
|
|
) { throw "LAB E15 replay-shadow authority contract changed" }
|
|
if ($stabilityProfile) {
|
|
$stability = Get-Content -LiteralPath $stabilityProfile -Raw | ConvertFrom-Json
|
|
if (
|
|
$stability.schema_version -ne "missioncore.e23-inline-temporal-profile/v1" -or
|
|
$stability.mode -ne "inline-shadow-qualification" -or
|
|
$stability.stage -ne "warm-worker-after-fusion-before-result-publication" -or
|
|
$stability.source.calibration_sha256 -ne $live.source.calibration_sha256 -or
|
|
[bool]$stability.authority.commands_enabled -or
|
|
[bool]$stability.authority.navigation_or_safety_accepted
|
|
) { throw "LAB E23 inline temporal authority contract changed" }
|
|
}
|
|
$projection = Get-Content -LiteralPath (Join-Path $projectionPack "manifest.json") -Raw | ConvertFrom-Json
|
|
if (
|
|
$projection.schema_version -ne "missioncore.e15-live-projection-pack/v1" -or
|
|
$projection.identity.source_id -ne "sensor.camera.right" -or
|
|
$projection.identity.calibration_slot -ne "camera_1" -or
|
|
$projection.identity.calibration_sha256 -ne $live.source.calibration_sha256
|
|
) { throw "LAB E15 projection pack binding changed" }
|
|
$packageManifest = Get-Content -LiteralPath (Join-Path $package "manifest.json") -Raw | ConvertFrom-Json
|
|
if (
|
|
$packageManifest.schema_version -ne "missioncore.e15-worker-package/v1" -or
|
|
$packageManifest.package_id -ne (Split-Path $package -Leaf) -or
|
|
$packageManifest.identity_sha256 -notmatch "^[a-f0-9]{64}$"
|
|
) { throw "LAB E15 worker package manifest changed" }
|
|
$mediaManifest = Get-Content -LiteralPath (Join-Path $mediaRuntime "manifest.json") -Raw | ConvertFrom-Json
|
|
if (
|
|
$mediaManifest.schema_version -ne "missioncore.e15-media-runtime/v1" -or
|
|
$mediaManifest.container_image -ne $ContainerImage -or
|
|
$mediaManifest.packages.av -ne "18.0.0" -or
|
|
$mediaManifest.packages.lz4 -ne "4.4.5"
|
|
) { throw "LAB E15 media runtime identity changed" }
|
|
|
|
$derivedRoot = Resolve-DDirectory (Join-Path $runtime "derived") "Runtime derived root"
|
|
$cacheRoot = Resolve-DDirectory (Join-Path $runtime "cache\perception-e3-models-v1") "EoMT cache"
|
|
$e3Environment = Resolve-DDirectory (Join-Path $derivedRoot "perception-e3-opencv413092-v1") "E3 environment"
|
|
$torchEnvironment = Resolve-DDirectory (Join-Path $derivedRoot "perception-p0-env-v1") "Torch environment"
|
|
$transformersEnvironment = Resolve-DDirectory (Join-Path $derivedRoot "perception-p0-transformers4576-v1") "Transformers environment"
|
|
$freeBytesBefore = Assert-FreeSpace "preflight" 512MB
|
|
|
|
& docker image inspect $ContainerImage *> $null
|
|
Assert-LastExitCode "Pinned container image inspection"
|
|
$tritonState = docker inspect --format "{{.State.Running}}" $TritonContainer
|
|
Assert-LastExitCode "Triton container inspection"
|
|
if ($tritonState.Trim().ToLowerInvariant() -ne "true") {
|
|
throw "LAB E15 requires the existing Triton container"
|
|
}
|
|
|
|
$runnerName = Split-Path $runner -Leaf
|
|
$liveProfileName = Split-Path $liveProfile -Leaf
|
|
$e14ProfileName = Split-Path $e14Profile -Leaf
|
|
$detectorProfileName = Split-Path $detectorProfile -Leaf
|
|
$semanticProfileName = Split-Path $semanticProfile -Leaf
|
|
$stabilityProfileName = if ($stabilityProfile) { Split-Path $stabilityProfile -Leaf } else { $null }
|
|
$projectionMount = "/" + (Split-Path $projectionPack -Leaf)
|
|
$packageMount = "/" + (Split-Path $package -Leaf)
|
|
$orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$commonMounts = @(
|
|
"-e", ("PYTHONPATH=/runner:{0}:/opt/media:/opt/transformers:/opt/env" -f $packageMount),
|
|
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
|
"-e", "TORCH_HOME=/cache/torch",
|
|
"-e", "HF_HOME=/cache/huggingface",
|
|
"-e", "HF_HUB_OFFLINE=1",
|
|
"-e", "TRANSFORMERS_OFFLINE=1",
|
|
"-v", ((Convert-ToDockerPath $jobDirectory) + ":/job:ro"),
|
|
"-v", ((Convert-ToDockerPath $validFov) + ":/valid-fov:ro"),
|
|
"-v", ((Convert-ToDockerPath $projectionPack) + (":{0}:ro" -f $projectionMount)),
|
|
"-v", ((Convert-ToDockerPath $model) + ":/model:ro"),
|
|
"-v", ((Convert-ToDockerPath $cacheRoot) + ":/cache:ro"),
|
|
"-v", ((Convert-ToDockerPath $e3Environment) + ":/environment:ro"),
|
|
"-v", ((Convert-ToDockerPath $torchEnvironment) + ":/opt/env:ro"),
|
|
"-v", ((Convert-ToDockerPath $transformersEnvironment) + ":/opt/transformers:ro"),
|
|
"-v", ((Convert-ToDockerPath $mediaRuntime) + ":/opt/media:ro"),
|
|
"-v", ((Convert-ToDockerPath $package) + (":{0}:ro" -f $packageMount)),
|
|
"-v", ((Convert-ToDockerPath $runnerRoot) + ":/runner:ro")
|
|
)
|
|
$commonRunnerArgs = @(
|
|
"--job", "/job",
|
|
"--live-profile", ("/runner/{0}" -f $liveProfileName),
|
|
"--e14-profile", ("/runner/{0}" -f $e14ProfileName),
|
|
"--detector-profile", ("/runner/{0}" -f $detectorProfileName),
|
|
"--semantic-profile", ("/runner/{0}" -f $semanticProfileName),
|
|
"--valid-fov-root", "/valid-fov",
|
|
"--projection-pack", $projectionMount,
|
|
"--model-root", "/model",
|
|
"--cache", "/cache",
|
|
"--environment", "/environment",
|
|
"--worker-package", $packageMount
|
|
)
|
|
if ($stabilityProfileName) {
|
|
$commonRunnerArgs += @(
|
|
"--stability-profile", ("/runner/{0}" -f $stabilityProfileName)
|
|
)
|
|
}
|
|
|
|
if ($PersistentService) {
|
|
if ($PreflightOnly -or $TokenStdin) {
|
|
throw "Persistent service cannot be combined with one-shot switches"
|
|
}
|
|
$persistentParent = Resolve-DDirectory (Split-Path $PersistentOutputRoot -Parent) "Persistent output parent"
|
|
if (-not (Test-Path -LiteralPath $PersistentOutputRoot)) {
|
|
$null = New-Item -ItemType Directory -Path $PersistentOutputRoot
|
|
}
|
|
$persistentOutput = Resolve-DDirectory $PersistentOutputRoot "Persistent output root"
|
|
if ((Split-Path $persistentOutput -Parent) -ne $persistentParent) {
|
|
throw "Persistent output root must be a direct child of its guarded D: parent"
|
|
}
|
|
$runnerSha256 = (Get-FileHash -LiteralPath $runner -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$liveProfileSha256 = (Get-FileHash -LiteralPath $liveProfile -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$e14ProfileSha256 = (Get-FileHash -LiteralPath $e14Profile -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$detectorProfileSha256 = (Get-FileHash -LiteralPath $detectorProfile -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$semanticProfileSha256 = (Get-FileHash -LiteralPath $semanticProfile -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$stabilityProfileSha256 = if ($stabilityProfile) {
|
|
(Get-FileHash -LiteralPath $stabilityProfile -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
}
|
|
else { "none" }
|
|
$existingContainerId = docker ps -a --filter ("name=^{0}$" -f $PersistentContainer) --format "{{.ID}}"
|
|
Assert-LastExitCode "Persistent worker container lookup"
|
|
if ($existingContainerId) {
|
|
$labels = (docker inspect --format "{{json .Config.Labels}}" $PersistentContainer) | ConvertFrom-Json
|
|
Assert-LastExitCode "Persistent worker label inspection"
|
|
if (
|
|
$labels.'missioncore.role' -ne "perception-persistent-worker" -or
|
|
$labels.'missioncore.runner.sha256' -ne $runnerSha256 -or
|
|
$labels.'missioncore.live.sha256' -ne $liveProfileSha256 -or
|
|
$labels.'missioncore.e14.sha256' -ne $e14ProfileSha256 -or
|
|
$labels.'missioncore.detector.sha256' -ne $detectorProfileSha256 -or
|
|
$labels.'missioncore.semantic.sha256' -ne $semanticProfileSha256 -or
|
|
$labels.'missioncore.stability.sha256' -ne $stabilityProfileSha256 -or
|
|
$labels.'missioncore.worker-package.identity' -ne $packageManifest.identity_sha256 -or
|
|
$labels.'missioncore.projection.identity' -ne $projection.identity_sha256
|
|
) { throw "Existing persistent worker has a different immutable identity" }
|
|
$running = docker inspect --format "{{.State.Running}}" $PersistentContainer
|
|
Assert-LastExitCode "Persistent worker state inspection"
|
|
if ($running.Trim().ToLowerInvariant() -ne "true") {
|
|
throw "Matching persistent worker exists but is not running"
|
|
}
|
|
$healthProbe = (
|
|
"import json,urllib.request;d=json.load(urllib.request.urlopen(" +
|
|
"'http://127.0.0.1:{0}/health',timeout=5));" -f $PersistentPort
|
|
) + "raise SystemExit(0 if d.get('ok') and d.get('models_loaded') else 2)"
|
|
$previousErrorPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
& docker exec $PersistentContainer python3 -c $healthProbe 2>$null
|
|
$healthExitCode = $LASTEXITCODE
|
|
$ErrorActionPreference = $previousErrorPreference
|
|
if ($healthExitCode -ne 0) { throw "Existing persistent worker health probe failed" }
|
|
Write-Output "STATE=persistent-worker-reused"
|
|
Write-Output ("CONTAINER={0}" -f $PersistentContainer)
|
|
return
|
|
}
|
|
|
|
if (-not (Test-TritonModelReady)) {
|
|
Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 `
|
|
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/load" `
|
|
-Method Post -ContentType "application/json" -Body "{}" *> $null
|
|
$deadline = [DateTime]::UtcNow.AddSeconds(60)
|
|
while (-not (Test-TritonModelReady)) {
|
|
if ([DateTime]::UtcNow -ge $deadline) { throw "YOLOX-S did not become ready" }
|
|
Start-Sleep -Milliseconds 500
|
|
}
|
|
}
|
|
|
|
$serveArgs = @(
|
|
"run", "--detach", "--name", $PersistentContainer,
|
|
"--label", "missioncore.role=perception-persistent-worker",
|
|
"--label", ("missioncore.runner.sha256={0}" -f $runnerSha256),
|
|
"--label", ("missioncore.live.sha256={0}" -f $liveProfileSha256),
|
|
"--label", ("missioncore.e14.sha256={0}" -f $e14ProfileSha256),
|
|
"--label", ("missioncore.detector.sha256={0}" -f $detectorProfileSha256),
|
|
"--label", ("missioncore.semantic.sha256={0}" -f $semanticProfileSha256),
|
|
"--label", ("missioncore.stability.sha256={0}" -f $stabilityProfileSha256),
|
|
"--label", ("missioncore.worker-package.identity={0}" -f $packageManifest.identity_sha256),
|
|
"--label", ("missioncore.projection.identity={0}" -f $projection.identity_sha256),
|
|
"--gpus", "all", "--network", ("container:{0}" -f $TritonContainer),
|
|
"--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
|
"--pids-limit", "512", "--shm-size", "4g",
|
|
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g"
|
|
) + $commonMounts + @(
|
|
"-v", ((Convert-ToDockerPath $persistentOutput) + ":/publish:rw"),
|
|
"--entrypoint", "python3", $ContainerImage,
|
|
("/runner/{0}" -f $runnerName), "serve"
|
|
) + $commonRunnerArgs + @(
|
|
"--host", $SourceHost,
|
|
"--port", [string]$SourcePort,
|
|
"--path", $SourcePath,
|
|
"--triton-url", "http://127.0.0.1:8000",
|
|
"--output-root", "/publish",
|
|
"--listen-host", "127.0.0.1",
|
|
"--listen-port", [string]$PersistentPort,
|
|
"--max-duration-seconds", [string]$MaximumDurationSeconds,
|
|
"--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB),
|
|
"--orchestrator-sha256", $orchestratorSha256,
|
|
"--container-image", $ContainerImage
|
|
)
|
|
Write-Output "PHASE=persistent-worker-model-load-start"
|
|
& docker @serveArgs
|
|
Assert-LastExitCode "Persistent worker container start"
|
|
$healthProbe = (
|
|
"import json,urllib.request;d=json.load(urllib.request.urlopen(" +
|
|
"'http://127.0.0.1:{0}/health',timeout=5));" -f $PersistentPort
|
|
) + "print(json.dumps(d,sort_keys=True));raise SystemExit(0 if d.get('ok') and d.get('models_loaded') else 2)"
|
|
$deadline = [DateTime]::UtcNow.AddSeconds(240)
|
|
do {
|
|
Start-Sleep -Seconds 2
|
|
$running = docker inspect --format "{{.State.Running}}" $PersistentContainer
|
|
if ($LASTEXITCODE -ne 0 -or $running.Trim().ToLowerInvariant() -ne "true") {
|
|
& docker logs --tail 80 $PersistentContainer
|
|
throw "Persistent worker exited during model load"
|
|
}
|
|
$previousErrorPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
$health = & docker exec $PersistentContainer python3 -c $healthProbe 2>$null
|
|
$healthExitCode = $LASTEXITCODE
|
|
$ErrorActionPreference = $previousErrorPreference
|
|
$healthy = $healthExitCode -eq 0
|
|
} while (-not $healthy -and [DateTime]::UtcNow -lt $deadline)
|
|
if (-not $healthy) {
|
|
& docker logs --tail 80 $PersistentContainer
|
|
throw "Persistent worker model load did not become ready"
|
|
}
|
|
Write-Output "STATE=persistent-worker-ready"
|
|
Write-Output ("CONTAINER={0}" -f $PersistentContainer)
|
|
Write-Output ("HEALTH={0}" -f $health)
|
|
Write-Output ("DISK_FREE_BYTES={0}" -f (Get-DFreeBytes))
|
|
return
|
|
}
|
|
|
|
$preflightArgs = @(
|
|
"run", "--rm", "--gpus", "all", "--network", "none", "--read-only",
|
|
"--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
|
"--pids-limit", "512", "--shm-size", "4g",
|
|
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g"
|
|
) + $commonMounts + @(
|
|
"--entrypoint", "python3", $ContainerImage,
|
|
("/runner/{0}" -f $runnerName), "preflight"
|
|
) + $commonRunnerArgs
|
|
Write-Output "PHASE=e15-preflight-start"
|
|
& docker @preflightArgs
|
|
Assert-LastExitCode "LAB E15 preflight"
|
|
Write-Output "PHASE=e15-preflight-complete"
|
|
if ($PreflightOnly) {
|
|
Write-Output "STATE=preflight-ready"
|
|
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
|
|
Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f (Get-DFreeBytes))
|
|
return
|
|
}
|
|
if (-not $TokenStdin) { throw "LAB E15 requires the shadow token through stdin" }
|
|
$shadowToken = [Console]::In.ReadLine()
|
|
if (-not $shadowToken -or $shadowToken.Length -lt 40 -or $shadowToken.Length -gt 512) {
|
|
throw "LAB E15 shadow token is missing or malformed"
|
|
}
|
|
|
|
$modelWasReady = Test-TritonModelReady
|
|
$loadedByRun = $false
|
|
if (-not $modelWasReady) {
|
|
Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 `
|
|
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/load" `
|
|
-Method Post -ContentType "application/json" -Body "{}" *> $null
|
|
$loadedByRun = $true
|
|
$deadline = [DateTime]::UtcNow.AddSeconds(60)
|
|
while (-not (Test-TritonModelReady)) {
|
|
if ([DateTime]::UtcNow -ge $deadline) { throw "YOLOX-S did not become ready" }
|
|
Start-Sleep -Milliseconds 500
|
|
}
|
|
}
|
|
Write-Output ("PHASE=e15-model-ready LOADED_BY_RUN={0}" -f $loadedByRun)
|
|
|
|
$token = [Guid]::NewGuid().ToString("N")
|
|
$publishRoot = Join-Path $derivedRoot (".{0}-e15-{1}.publish" -f $job.job_id, $token)
|
|
$stagingRoot = Join-Path $publishRoot "output"
|
|
$null = New-Item -ItemType Directory -Path $publishRoot
|
|
$completed = $false
|
|
$totalWatch = [Diagnostics.Stopwatch]::StartNew()
|
|
try {
|
|
$runArgs = @(
|
|
"run", "--rm", "--interactive", "--gpus", "all",
|
|
"--network", ("container:{0}" -f $TritonContainer),
|
|
"--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
|
"--pids-limit", "512", "--shm-size", "4g",
|
|
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g"
|
|
) + $commonMounts + @(
|
|
"-v", ((Convert-ToDockerPath $publishRoot) + ":/publish:rw"),
|
|
"--entrypoint", "python3", $ContainerImage,
|
|
("/runner/{0}" -f $runnerName), "run"
|
|
) + $commonRunnerArgs + @(
|
|
"--host", $SourceHost,
|
|
"--port", [string]$SourcePort,
|
|
"--path", $SourcePath,
|
|
"--token-stdin",
|
|
"--triton-url", "http://127.0.0.1:8000",
|
|
"--output", "/publish/output",
|
|
"--max-duration-seconds", [string]$MaximumDurationSeconds,
|
|
"--free-bytes-floor", [string]([int64]$FreeGiBFloor * 1GB),
|
|
"--orchestrator-sha256", $orchestratorSha256,
|
|
"--container-image", $ContainerImage
|
|
)
|
|
Write-Output ("PHASE=e15-shadow-inference-start MAX_DURATION_SECONDS={0}" -f $MaximumDurationSeconds)
|
|
$shadowToken | & docker @runArgs
|
|
$runnerExitCode = $LASTEXITCODE
|
|
$shadowToken = $null
|
|
if ($runnerExitCode -ne 0 -and $runnerExitCode -ne 2) {
|
|
throw "LAB E15 shadow inference failed with exit code $runnerExitCode"
|
|
}
|
|
$null = Assert-FreeSpace "post-shadow-inference"
|
|
|
|
$resultPath = Join-Path $stagingRoot "result.json"
|
|
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
|
|
throw "LAB E15 result manifest is missing"
|
|
}
|
|
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
|
|
if (
|
|
$result.schema_version -ne "missioncore.e15-shadow-inference-result/v1" -or
|
|
$result.result_id -notmatch "^e15-shadow-inference-[a-f0-9]{64}$" -or
|
|
$result.publication_scope -ne "live-shadow-diagnostic-only"
|
|
) { throw "LAB E15 result manifest is incompatible" }
|
|
$finalRoot = Join-Path $derivedRoot ([string]$result.result_id)
|
|
if (Test-Path -LiteralPath $finalRoot) { throw "Immutable LAB E15 result already exists" }
|
|
Move-Item -LiteralPath $stagingRoot -Destination $finalRoot
|
|
Remove-Item -LiteralPath $publishRoot -Force
|
|
$completed = $true
|
|
$totalWatch.Stop()
|
|
$freeFinal = Assert-FreeSpace "post-publication"
|
|
Write-Output ("RESULT_ROOT={0}" -f $finalRoot)
|
|
Write-Output ("RESULT_ID={0}" -f $result.result_id)
|
|
Write-Output ("ACCEPTANCE_STATE={0}" -f $result.acceptance_state)
|
|
Write-Output ("RUNNER_EXIT_CODE={0}" -f $runnerExitCode)
|
|
Write-Output ("ORCHESTRATOR_WALL_SECONDS={0}" -f $totalWatch.Elapsed.TotalSeconds)
|
|
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBytesBefore)
|
|
Write-Output ("DISK_FREE_BYTES_FINAL={0}" -f $freeFinal)
|
|
}
|
|
finally {
|
|
$shadowToken = $null
|
|
if (-not $completed -and (Test-Path -LiteralPath $publishRoot)) {
|
|
Remove-Item -LiteralPath $publishRoot -Recurse -Force
|
|
}
|
|
if ($loadedByRun) {
|
|
try {
|
|
Invoke-WebRequest -UseBasicParsing -TimeoutSec 60 `
|
|
-Uri "http://127.0.0.1:8000/v2/repository/models/yolox_s/unload" `
|
|
-Method Post -ContentType "application/json" -Body "{}" *> $null
|
|
Write-Output "PHASE=e15-model-state-restored"
|
|
}
|
|
catch { Write-Warning "LAB E15 could not restore the prior YOLOX-S state" }
|
|
}
|
|
}
|