Files
NODEDC_MISSION_CORE/scripts/Invoke-M47CanonicalGraphShadow.ps1
T

494 lines
19 KiB
PowerShell

[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[string]$ReleaseRoot,
[Parameter(Mandatory = $true)]
[string]$ArtifactPath,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[a-f0-9]{64}$")]
[string]$ExpectedArtifactSha256,
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m47-reference-graph",
[ValidateRange(1, 1000)]
[int]$FreeGiBFloor = 300,
[switch]$PreflightOnly
)
$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue"
function Assert-LastExitCode([string]$Operation) {
if ($LASTEXITCODE -ne 0) {
throw "$Operation failed with exit code $LASTEXITCODE"
}
}
function Get-Sha256([string]$Path) {
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
}
function Assert-FileSha256([string]$Path, [string]$Expected, [string]$Label) {
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
throw "$Label must be a regular file"
}
$observed = Get-Sha256 $item.FullName
if ($observed -cne $Expected) {
throw "$Label SHA-256 changed: expected $Expected, observed $observed"
}
return $item.FullName
}
function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) {
if ($Create -and -not (Test-Path -LiteralPath $Path)) {
$null = New-Item -ItemType Directory -Path $Path
}
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
if (
-not $item.PSIsContainer -or
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
$root -ine "D:"
) {
throw "$Label must be a real D: directory"
}
return $item.FullName
}
function Convert-ToDockerPath([string]$Path) {
return $Path.Replace("\", "/")
}
function Assert-FreeSpace([string]$Phase) {
$free = [int64](Get-PSDrive -Name D).Free
$floor = [int64]$FreeGiBFloor * 1GB
Write-Host (
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3}" -f
$Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor
)
if ($free -lt ($floor + 1GB)) {
throw "D: lacks the guarded M4.7 reserve during $Phase"
}
return $free
}
function Get-ContainerIdentity([string]$Name) {
$json = & docker inspect $Name
Assert-LastExitCode "Docker inspection for $Name"
$rows = @($json | ConvertFrom-Json)
if ($rows.Count -ne 1) {
throw "Docker identity for $Name is not unique"
}
return $rows[0]
}
function Assert-ContainerIdentity(
[string]$Name,
[string]$ExpectedId,
[string]$ExpectedImageId,
[bool]$RequireHealthy
) {
$container = Get-ContainerIdentity $Name
if (
$container.Id -cne $ExpectedId -or
$container.Image -cne $ExpectedImageId -or
-not $container.State.Running
) {
throw "$Name identity or running state changed"
}
if ($RequireHealthy -and $container.State.Health.Status -cne "healthy") {
throw "$Name is not healthy"
}
return $container
}
function Get-DirectoryTreeSha256([string]$Root, [object[]]$Files) {
$rootPrefix = $Root.TrimEnd("\") + "\"
$rows = @(
foreach ($file in $Files) {
if (-not $file.FullName.StartsWith($rootPrefix, [StringComparison]::OrdinalIgnoreCase)) {
throw "M4.7 dependency file escaped its declared root"
}
[pscustomobject]@{
Relative = $file.FullName.Substring($rootPrefix.Length).Replace("\", "/")
File = $file
}
}
)
$rows = @($rows | Sort-Object -Property Relative -CaseSensitive)
$digest = [Security.Cryptography.IncrementalHash]::CreateHash(
[Security.Cryptography.HashAlgorithmName]::SHA256
)
$encoding = New-Object System.Text.UTF8Encoding($false)
try {
foreach ($row in $rows) {
$record = "{0}`t{1}`t{2}`n" -f `
$row.Relative, $row.File.Length, (Get-Sha256 $row.File.FullName)
$digest.AppendData($encoding.GetBytes($record))
}
return ([BitConverter]::ToString($digest.GetHashAndReset())).Replace("-", "").ToLowerInvariant()
} finally {
$digest.Dispose()
}
}
function Write-Utf8NoBom([string]$Path, [string]$Value) {
$encoding = New-Object System.Text.UTF8Encoding($false)
[IO.File]::WriteAllText($Path, $Value, $encoding)
}
function Test-NoPublishedPorts([object]$Container) {
return @($Container.HostConfig.PortBindings.PSObject.Properties).Count -eq 0
}
if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
throw "M4.7 shadow release is pinned to DESKTOP-OPJ8J04"
}
$release = Resolve-DDirectory $ReleaseRoot "M4.7 release root" $false
$payload = Resolve-DDirectory (Join-Path $release "payload") "M4.7 payload" $false
$artifact = Assert-FileSha256 $ArtifactPath $ExpectedArtifactSha256 "M4.7 release artifact"
$descriptorPath = Join-Path $payload "mission-core-worker-m47-graph-shadow-v3.json"
$descriptor = Get-Content -LiteralPath $descriptorPath -Raw | ConvertFrom-Json
if (
$descriptor.schema_version -cne "nodedc.mission-core-worker.shadow-release/v3" -or
$descriptor.transition -cne "m47-canonical-graph-isolated-shadow-v1" -or
$descriptor.component -cne "mission-core-worker" -or
$descriptor.host.node -cne $env:COMPUTERNAME -or
$descriptor.host.worker_id -cne "worker-006" -or
$descriptor.boundary.repository -cne "NODEDC_MISSION_CORE" -or
$descriptor.boundary.external_deploy_registry -ne $false -or
$descriptor.acceptance.run_mode -cne "lossless-replay" -or
$descriptor.acceptance.expected_frames -ne 4489 -or
$descriptor.acceptance.delivered_frames -ne 4489 -or
$descriptor.acceptance.failed_frames -ne 0 -or
$descriptor.acceptance.stale_frames -ne 0 -or
$descriptor.acceptance.superseded_frames -ne 0 -or
$descriptor.acceptance.accepted_parity -ne $true -or
$descriptor.readiness.graph.graph_id -cne "reference-perception-graph/v2" -or
$descriptor.readiness.graph.actuation_allowed -ne $false
) {
throw "M4.7 shadow descriptor contract changed"
}
$null = Assert-FileSha256 $PSCommandPath $descriptor.release.runner.sha256 "M4.7 runner"
$wheelPath = Assert-FileSha256 (
Join-Path $payload $descriptor.release.wheel.name
) $descriptor.release.wheel.sha256 "M4.7 wheel"
foreach ($entry in $descriptor.release.configs.PSObject.Properties) {
$null = Assert-FileSha256 (
Join-Path $payload $entry.Value.name
) $entry.Value.sha256 ("M4.7 config {0}" -f $entry.Name)
}
foreach ($entry in $descriptor.inputs.PSObject.Properties) {
$null = Assert-FileSha256 $entry.Value.host_path $entry.Value.sha256 (
"M4.7 input {0}" -f $entry.Name
)
}
foreach ($dependency in $descriptor.dependencies) {
$root = Resolve-DDirectory $dependency.host_path ("M4.7 dependency {0}" -f $dependency.id) $false
$files = @()
foreach ($include in $dependency.includes) {
$candidate = if ($include -eq ".") { $root } else { Join-Path $root $include }
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $candidate).Path -Force
if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) {
throw "M4.7 dependency $($dependency.id) contains a reparse point"
}
if ($item.PSIsContainer) {
$files += @(Get-ChildItem -LiteralPath $item.FullName -File -Recurse -Force)
} else {
$files += @($item)
}
}
$files = @($files | Sort-Object -Property FullName -Unique)
$bytes = [int64]0
foreach ($file in $files) {
if ($file.Attributes -band [IO.FileAttributes]::ReparsePoint) {
throw "M4.7 dependency $($dependency.id) contains a reparse-point file"
}
$bytes += [int64]$file.Length
}
if (
$files.Count -ne [int]$dependency.file_count -or
$bytes -ne [int64]$dependency.bytes -or
(Get-DirectoryTreeSha256 $root $files) -cne [string]$dependency.tree_sha256
) {
throw "M4.7 dependency $($dependency.id) inventory changed"
}
}
$imageRef = [string]$descriptor.container.image_ref
& docker image inspect $imageRef *> $null
Assert-LastExitCode "Pinned M4.7 image inspection"
function Get-PreservedContainerSnapshot([object]$Expected, [string]$Label) {
$container = Get-ContainerIdentity $Expected.name
if (
$container.Id -cne $Expected.container_id -or
$container.Image -cne $Expected.image_id
) {
throw "$Label identity changed"
}
return [pscustomobject]@{
Id = [string]$container.Id
Image = [string]$container.Image
Running = [bool]$container.State.Running
}
}
function Assert-PreservedContainerSnapshot(
[object]$Expected,
[object]$Before,
[string]$Label
) {
$after = Get-PreservedContainerSnapshot $Expected $Label
if ($after.Running -ne $Before.Running) {
throw "$Label running state changed during isolated shadow"
}
return $after
}
$durableExpected = $descriptor.predecessor.durable_worker
$historicalTritonExpected = $descriptor.predecessor.triton
$durableBefore = Get-PreservedContainerSnapshot $durableExpected "Historical durable worker"
$historicalTritonBefore = Get-PreservedContainerSnapshot (
$historicalTritonExpected
) "Historical Triton"
$modelRepository = Resolve-DDirectory (
[string]$descriptor.container.model_repository_host_path
) "M4.7 model repository" $false
$output = Resolve-DDirectory $OutputRoot "M4.7 output root" $true
$freeBefore = Assert-FreeSpace "preflight"
$candidateName = [string]$descriptor.container.name
$tritonCandidateName = [string]$descriptor.container.triton_name
foreach ($name in @($candidateName, $tritonCandidateName)) {
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
throw "M4.7 isolated candidate $name already exists"
}
}
$scratch = Join-Path $output (".runtime-{0}" -f $descriptor.patch_id)
if (Test-Path -LiteralPath $scratch) {
throw "M4.7 runtime scratch already exists"
}
$null = New-Item -ItemType Directory -Path $scratch
$scratch = Resolve-DDirectory $scratch "M4.7 runtime scratch" $false
$runtimeIdentityPath = Join-Path $scratch "runtime-identity.json"
$dockerPayload = Convert-ToDockerPath $payload
$dockerOutput = Convert-ToDockerPath $output
$dockerScratch = Convert-ToDockerPath $scratch
$dockerModelRepository = Convert-ToDockerPath $modelRepository
$tritonArguments = @(
"create",
"--name", $tritonCandidateName,
"--read-only",
"--security-opt", "no-new-privileges:true",
"--cap-drop", "ALL",
"--pids-limit", "512",
"--shm-size", "1g",
"--gpus", "all",
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
"--health-cmd", "curl --fail --silent http://127.0.0.1:8000/v2/health/ready",
"--health-interval", "5s",
"--health-timeout", "3s",
"--health-start-period", "20s",
"--health-retries", "24",
"-v", ("{0}:/models:ro" -f $dockerModelRepository),
$imageRef,
"tritonserver",
"--model-repository=/models",
"--model-control-mode=explicit",
"--load-model=yolox_s",
"--disable-auto-complete-config",
"--strict-readiness=true",
"--exit-on-error=true",
"--allow-http=true",
"--allow-grpc=false",
"--allow-metrics=false"
)
$graphArguments = @(
"create",
"--name", $candidateName,
"--network", ("container:{0}" -f $tritonCandidateName),
"--read-only",
"--security-opt", "no-new-privileges:true",
"--cap-drop", "ALL",
"--pids-limit", "256",
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
"-e", "PYTHONDONTWRITEBYTECODE=1",
"-e", ("PYTHONPATH=/release/{0}:/opt/media:/opt/opencv:/opt/pillow" -f $descriptor.release.wheel.name),
"-v", ("{0}:/release:ro" -f $dockerPayload),
"-v", ("{0}:/output:rw" -f $dockerOutput),
"-v", ("{0}:/run/mission-core:ro" -f $dockerScratch)
)
foreach ($entry in $descriptor.inputs.PSObject.Properties) {
if ($null -ne $entry.Value.container_path) {
$graphArguments += @(
"-v", ("{0}:{1}:ro" -f (
Convert-ToDockerPath $entry.Value.host_path
), $entry.Value.container_path)
)
}
}
foreach ($dependency in $descriptor.dependencies) {
$graphArguments += @(
"-v", ("{0}:{1}:ro" -f (
Convert-ToDockerPath $dependency.host_path
), $dependency.container_path)
)
}
$graphArguments += @(
"--entrypoint", "python3",
$imageRef,
"-m", "k1link.perception.reference_graph_cli",
"--graph-config", "/release/m4-reference-graph-v2.json",
"--baseline-profile", "/release/m4-recorded-realtime-baseline-v1.json",
"--geometry-profile", "/release/m4-geometry-association-v1.json",
"--temporal-motion-profile", "/release/m4-temporal-motion-v1.json",
"--rolling-map-profile", "/release/m4-rolling-local-map-v1.json",
"--threat-profile", "/release/m4-replay-threat-v3.json",
"--camera-index", $descriptor.inputs.camera_index.container_path,
"--source-pack", $descriptor.inputs.source_pack.container_path,
"--local-surface", $descriptor.inputs.local_surface.container_path,
"--video", $descriptor.inputs.video.container_path,
"--valid-fov-mask", $descriptor.inputs.valid_fov_mask.container_path,
"--temporal-parity-frames", $descriptor.inputs.accepted_temporal_frames.container_path,
"--threat-parity-frames", $descriptor.inputs.accepted_threat_frames.container_path,
"--triton-origin", $descriptor.container.triton_origin,
"--mode", $descriptor.acceptance.run_mode,
"--expected-frames", ([string]$descriptor.acceptance.expected_frames),
"--runtime-identity", "/run/mission-core/runtime-identity.json",
"--output-root", "/output"
)
$tritonCreated = $false
$graphCreated = $false
$providerAccepted = $false
$graphAccepted = $false
$runFailure = $null
try {
$tritonCandidateId = (& docker @tritonArguments).Trim()
Assert-LastExitCode "M4.7 isolated Triton creation"
if ($tritonCandidateId -notmatch "^[a-f0-9]{64}$") {
throw "M4.7 isolated Triton id is invalid"
}
$tritonCreated = $true
& docker start $tritonCandidateName *> $null
Assert-LastExitCode "M4.7 isolated Triton start"
$tritonCandidate = $null
foreach ($attempt in 1..120) {
$tritonCandidate = Get-ContainerIdentity $tritonCandidateName
if (-not $tritonCandidate.State.Running) {
throw "M4.7 isolated Triton stopped before readiness"
}
if ($tritonCandidate.State.Health.Status -ceq "healthy") {
break
}
if ($attempt -eq 120) {
throw "M4.7 isolated Triton readiness timed out"
}
Start-Sleep -Seconds 2
}
if (
$tritonCandidate.Id -cne $tritonCandidateId -or
$tritonCandidate.Image -cne $descriptor.container.image_id -or
-not $tritonCandidate.HostConfig.ReadonlyRootfs -or
-not (Test-NoPublishedPorts $tritonCandidate)
) {
throw "M4.7 isolated Triton contract changed"
}
$providerAccepted = $true
if (-not $PreflightOnly) {
$candidateId = (& docker @graphArguments).Trim()
Assert-LastExitCode "M4.7 graph candidate creation"
if ($candidateId -notmatch "^[a-f0-9]{64}$") {
throw "M4.7 graph candidate id is invalid"
}
$graphCreated = $true
$candidate = Get-ContainerIdentity $candidateName
if (
$candidate.Id -cne $candidateId -or
$candidate.Image -cne $descriptor.container.image_id -or
$candidate.HostConfig.NetworkMode -cne ("container:{0}" -f $tritonCandidate.Id) -or
-not $candidate.HostConfig.ReadonlyRootfs -or
-not (Test-NoPublishedPorts $candidate)
) {
throw "M4.7 graph candidate isolation contract changed"
}
$runtimeIdentity = [ordered]@{
schema_version = "missioncore.reference-graph-runtime-identity/v3"
worker_id = "worker-006"
worker_node = $env:COMPUTERNAME
worker_container_id = $candidate.Id
worker_image_id = $candidate.Image
isolated_triton_container_id = $tritonCandidate.Id
isolated_triton_image_id = $tritonCandidate.Image
historical_worker_container_id = $durableBefore.Id
historical_worker_running = $durableBefore.Running
historical_triton_container_id = $historicalTritonBefore.Id
historical_triton_running = $historicalTritonBefore.Running
artifact_sha256 = $ExpectedArtifactSha256
patch_id = $descriptor.patch_id
code_revision = $descriptor.code_revision
graph_id = $descriptor.readiness.graph.graph_id
started_at_utc = [DateTime]::UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ")
source_mount_read_only = $true
isolated_model_service = $true
public_worker_port_added = $false
commands_enabled = $false
actuation_allowed = $false
}
Write-Utf8NoBom $runtimeIdentityPath ($runtimeIdentity | ConvertTo-Json -Depth 4)
& docker start --attach $candidateName
Assert-LastExitCode "M4.7 canonical graph isolated shadow"
$graphAccepted = $true
}
} catch {
$runFailure = $_
} finally {
if ($graphCreated) {
& docker rm --force $candidateName *> $null
if ($LASTEXITCODE -ne 0 -and $null -eq $runFailure) {
$runFailure = "M4.7 graph candidate cleanup failed"
}
}
if ($tritonCreated) {
& docker rm --force $tritonCandidateName *> $null
if ($LASTEXITCODE -ne 0 -and $null -eq $runFailure) {
$runFailure = "M4.7 isolated Triton cleanup failed"
}
}
Remove-Item -LiteralPath $scratch -Force -Recurse -ErrorAction SilentlyContinue
}
$durableAfter = Assert-PreservedContainerSnapshot (
$durableExpected
) $durableBefore "Historical durable worker"
$historicalTritonAfter = Assert-PreservedContainerSnapshot (
$historicalTritonExpected
) $historicalTritonBefore "Historical Triton"
$freeAfter = Assert-FreeSpace "completed"
Write-Output ("PATCH_ID={0}" -f $descriptor.patch_id)
Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256)
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore)
Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f $freeAfter)
Write-Output ("HISTORICAL_DURABLE_WORKER_RUNNING={0}" -f $durableAfter.Running)
Write-Output ("HISTORICAL_TRITON_RUNNING={0}" -f $historicalTritonAfter.Running)
Write-Output "DURABLE_WORKER_ACTION=none"
Write-Output "HISTORICAL_TRITON_ACTION=none"
Write-Output "ISOLATED_TRITON_ACTION=removed"
if ($null -ne $runFailure) {
throw $runFailure
}
if (-not $providerAccepted) {
throw "M4.7 provider readiness was not accepted"
}
Write-Output "PROVIDER_READINESS=accepted"
if ($PreflightOnly) {
Write-Output "GRAPH_READINESS=not-run"
Write-Output "PREFLIGHT=accepted"
} elseif ($graphAccepted) {
Write-Output "GRAPH_READINESS=accepted"
}