406 lines
15 KiB
PowerShell
406 lines
15 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ReleaseRoot,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ArtifactPath,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[a-f0-9]{64}$")]
|
|
[string]$ExpectedArtifactSha256,
|
|
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m4-detector-replay",
|
|
[ValidateRange(1, 1000)]
|
|
[int]$FreeGiBFloor = 300,
|
|
[switch]$PreflightOnly
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$ProgressPreference = "SilentlyContinue"
|
|
|
|
function Assert-LastExitCode([string]$Operation) {
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "$Operation failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
function Get-Sha256([string]$Path) {
|
|
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
}
|
|
|
|
function Assert-FileSha256([string]$Path, [string]$Expected, [string]$Label) {
|
|
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
|
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
|
throw "$Label must be a regular file"
|
|
}
|
|
$observed = Get-Sha256 $item.FullName
|
|
if ($observed -cne $Expected) {
|
|
throw "$Label SHA-256 changed: expected $Expected, observed $observed"
|
|
}
|
|
return $item.FullName
|
|
}
|
|
|
|
function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) {
|
|
if ($Create -and -not (Test-Path -LiteralPath $Path)) {
|
|
$null = New-Item -ItemType Directory -Path $Path
|
|
}
|
|
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
|
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
|
if (
|
|
-not $item.PSIsContainer -or
|
|
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
|
$root -ine "D:"
|
|
) {
|
|
throw "$Label must be a real D: directory"
|
|
}
|
|
return $item.FullName
|
|
}
|
|
|
|
function Convert-ToDockerPath([string]$Path) {
|
|
return $Path.Replace("\", "/")
|
|
}
|
|
|
|
function Assert-FreeSpace([string]$Phase) {
|
|
$free = [int64](Get-PSDrive -Name D).Free
|
|
$floor = [int64]$FreeGiBFloor * 1GB
|
|
Write-Host (
|
|
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3}" -f
|
|
$Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor
|
|
)
|
|
if ($free -lt ($floor + 1GB)) {
|
|
throw "D: lacks the guarded M4 reserve during $Phase"
|
|
}
|
|
return $free
|
|
}
|
|
|
|
function Get-ContainerIdentity([string]$Name) {
|
|
$json = & docker inspect $Name
|
|
Assert-LastExitCode "Docker inspection for $Name"
|
|
$rows = @($json | ConvertFrom-Json)
|
|
if ($rows.Count -ne 1) {
|
|
throw "Docker identity for $Name is not unique"
|
|
}
|
|
return $rows[0]
|
|
}
|
|
|
|
function Assert-ContainerIdentity(
|
|
[string]$Name,
|
|
[string]$ExpectedId,
|
|
[string]$ExpectedImageId,
|
|
[bool]$RequireHealthy
|
|
) {
|
|
$container = Get-ContainerIdentity $Name
|
|
if (
|
|
$container.Id -cne $ExpectedId -or
|
|
$container.Image -cne $ExpectedImageId -or
|
|
-not $container.State.Running
|
|
) {
|
|
throw "$Name identity or running state changed"
|
|
}
|
|
if ($RequireHealthy -and $container.State.Health.Status -cne "healthy") {
|
|
throw "$Name is not healthy"
|
|
}
|
|
return $container
|
|
}
|
|
|
|
function Get-RequiredText([object]$Value, [string]$Label) {
|
|
if ($Value -isnot [string] -or [string]::IsNullOrWhiteSpace($Value)) {
|
|
throw "$Label must be a non-empty string"
|
|
}
|
|
return [string]$Value
|
|
}
|
|
|
|
function Write-Utf8NoBom([string]$Path, [string]$Value) {
|
|
$encoding = New-Object System.Text.UTF8Encoding($false)
|
|
[IO.File]::WriteAllText($Path, $Value, $encoding)
|
|
}
|
|
|
|
function Get-DirectoryTreeSha256([string]$Root, [object[]]$Files) {
|
|
$rootPrefix = $Root.TrimEnd("\") + "\"
|
|
$rows = @(
|
|
foreach ($file in $Files) {
|
|
if (-not $file.FullName.StartsWith(
|
|
$rootPrefix,
|
|
[StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "M4 dependency file escaped its declared root"
|
|
}
|
|
[pscustomobject]@{
|
|
Relative = $file.FullName.Substring($rootPrefix.Length).Replace("\", "/")
|
|
File = $file
|
|
}
|
|
}
|
|
)
|
|
$rows = @($rows | Sort-Object -Property Relative -CaseSensitive)
|
|
$digest = [Security.Cryptography.IncrementalHash]::CreateHash(
|
|
[Security.Cryptography.HashAlgorithmName]::SHA256
|
|
)
|
|
$encoding = New-Object System.Text.UTF8Encoding($false)
|
|
try {
|
|
foreach ($row in $rows) {
|
|
$fileSha256 = Get-Sha256 $row.File.FullName
|
|
$record = "{0}`t{1}`t{2}`n" -f $row.Relative, $row.File.Length, $fileSha256
|
|
$digest.AppendData($encoding.GetBytes($record))
|
|
}
|
|
return ([BitConverter]::ToString($digest.GetHashAndReset())).Replace("-", "").ToLowerInvariant()
|
|
} finally {
|
|
$digest.Dispose()
|
|
}
|
|
}
|
|
|
|
if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
|
|
throw "M4 shadow release is pinned to DESKTOP-OPJ8J04"
|
|
}
|
|
|
|
$release = Resolve-DDirectory $ReleaseRoot "M4 release root" $false
|
|
$payload = Resolve-DDirectory (Join-Path $release "payload") "M4 payload" $false
|
|
$artifact = Assert-FileSha256 $ArtifactPath $ExpectedArtifactSha256 "M4 release artifact"
|
|
$descriptorPath = Join-Path $payload "mission-core-worker-shadow-v1.json"
|
|
$descriptor = Get-Content -LiteralPath $descriptorPath -Raw | ConvertFrom-Json
|
|
|
|
if (
|
|
$descriptor.schema_version -cne "nodedc.mission-core-worker.shadow-release/v1" -or
|
|
$descriptor.component -cne "mission-core-worker" -or
|
|
$descriptor.artifact_type -cne "shadow-release" -or
|
|
$descriptor.transition -cne "m4-detector-shadow-v1" -or
|
|
$descriptor.dependency_tree_identity -cne (
|
|
"relative-path-tab-size-tab-file-sha256-lf/v1"
|
|
) -or
|
|
$descriptor.boundary.repository -cne "NODEDC_MISSION_CORE" -or
|
|
$descriptor.boundary.nodedc_platform_repository -ne $false -or
|
|
$descriptor.boundary.external_deploy_registry -ne $false -or
|
|
$descriptor.boundary.server_docker_runtime -ne $false -or
|
|
$descriptor.host.node -cne $env:COMPUTERNAME -or
|
|
$descriptor.host.worker_id -cne "worker-006" -or
|
|
$descriptor.acceptance.expected_frames -ne 4489 -or
|
|
[double]$descriptor.acceptance.minimum_end_to_end_fps -ne 10.004 -or
|
|
$descriptor.acceptance.failed_frames -ne 0 -or
|
|
$descriptor.acceptance.class_routing_used -ne $false
|
|
) {
|
|
throw "M4 shadow descriptor contract changed"
|
|
}
|
|
|
|
$wheelName = Get-RequiredText $descriptor.release.wheel.name "Wheel name"
|
|
$baselineName = Get-RequiredText $descriptor.release.baseline.name "Baseline name"
|
|
$runnerName = Get-RequiredText $descriptor.release.runner.name "Runner name"
|
|
if ($runnerName -cne "Invoke-M4DetectorShadow.ps1") {
|
|
throw "M4 runner name changed"
|
|
}
|
|
$null = Assert-FileSha256 $PSCommandPath $descriptor.release.runner.sha256 "M4 local runner"
|
|
$wheelPath = Assert-FileSha256 (
|
|
Join-Path $payload $wheelName
|
|
) $descriptor.release.wheel.sha256 "M4 runtime wheel"
|
|
$baselinePath = Assert-FileSha256 (
|
|
Join-Path $payload $baselineName
|
|
) $descriptor.release.baseline.sha256 "M4 baseline"
|
|
|
|
foreach ($entry in $descriptor.inputs.PSObject.Properties) {
|
|
$input = $entry.Value
|
|
$null = Assert-FileSha256 $input.host_path $input.sha256 ("M4 input {0}" -f $entry.Name)
|
|
}
|
|
$dependencyFailures = @()
|
|
foreach ($dependency in $descriptor.dependencies) {
|
|
$root = Resolve-DDirectory $dependency.host_path ("M4 dependency {0}" -f $dependency.id) $false
|
|
$files = @()
|
|
foreach ($include in $dependency.includes) {
|
|
$candidate = if ($include -eq ".") { $root } else { Join-Path $root $include }
|
|
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $candidate).Path -Force
|
|
if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) {
|
|
throw "M4 dependency $($dependency.id) contains a reparse-point root"
|
|
}
|
|
if ($item.PSIsContainer) {
|
|
$files += @(Get-ChildItem -LiteralPath $item.FullName -File -Recurse -Force)
|
|
} else {
|
|
$files += @($item)
|
|
}
|
|
}
|
|
$uniqueFiles = @($files | Sort-Object -Property FullName -Unique)
|
|
$bytes = [int64]0
|
|
foreach ($file in $uniqueFiles) {
|
|
if ($file.Attributes -band [IO.FileAttributes]::ReparsePoint) {
|
|
throw "M4 dependency $($dependency.id) contains a reparse-point file"
|
|
}
|
|
$bytes += [int64]$file.Length
|
|
}
|
|
$treeSha256 = Get-DirectoryTreeSha256 $root $uniqueFiles
|
|
if (
|
|
$uniqueFiles.Count -ne [int]$dependency.file_count -or
|
|
$bytes -ne [int64]$dependency.bytes -or
|
|
$treeSha256 -cne [string]$dependency.tree_sha256
|
|
) {
|
|
$dependencyFailures += (
|
|
"M4 dependency {0} inventory changed: expected {1}/{2}/{3}, observed {4}/{5}/{6}" -f
|
|
$dependency.id,
|
|
$dependency.file_count,
|
|
$dependency.bytes,
|
|
$dependency.tree_sha256,
|
|
$uniqueFiles.Count,
|
|
$bytes,
|
|
$treeSha256
|
|
)
|
|
}
|
|
}
|
|
if ($dependencyFailures.Count -gt 0) {
|
|
$dependencyFailures | ForEach-Object { Write-Output ("DEPENDENCY_MISMATCH={0}" -f $_) }
|
|
throw "M4 dependency inventory changed"
|
|
}
|
|
|
|
$imageRef = Get-RequiredText $descriptor.container.image_ref "Container image reference"
|
|
& docker image inspect $imageRef *> $null
|
|
Assert-LastExitCode "Pinned M4 image inspection"
|
|
$predecessor = $descriptor.predecessor.durable_worker
|
|
$tritonExpected = $descriptor.predecessor.triton
|
|
$durable = Assert-ContainerIdentity (
|
|
Get-RequiredText $predecessor.name "Durable worker name"
|
|
) $predecessor.container_id $predecessor.image_id $false
|
|
$triton = Assert-ContainerIdentity (
|
|
Get-RequiredText $tritonExpected.name "Triton name"
|
|
) $tritonExpected.container_id $tritonExpected.image_id $true
|
|
|
|
$output = Resolve-DDirectory $OutputRoot "M4 output root" $true
|
|
$freeBefore = Assert-FreeSpace "preflight"
|
|
$patchId = Get-RequiredText $descriptor.patch_id "Patch id"
|
|
if ($patchId -notmatch "^[A-Za-z0-9._-]{1,96}$") {
|
|
throw "M4 patch id is invalid"
|
|
}
|
|
if ($PreflightOnly) {
|
|
Write-Output ("PATCH_ID={0}" -f $patchId)
|
|
Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256)
|
|
Write-Output ("DURABLE_WORKER_ID={0}" -f $durable.Id)
|
|
Write-Output ("TRITON_CONTAINER_ID={0}" -f $triton.Id)
|
|
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore)
|
|
Write-Output "PREFLIGHT=accepted"
|
|
return
|
|
}
|
|
$candidateName = "ndc-mission-core-m4-detector-shadow"
|
|
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$candidateName$") {
|
|
throw "M4 candidate container already exists"
|
|
}
|
|
$scratch = Join-Path $output (".runtime-{0}" -f $patchId)
|
|
if (Test-Path -LiteralPath $scratch) {
|
|
throw "M4 runtime scratch already exists"
|
|
}
|
|
$null = New-Item -ItemType Directory -Path $scratch
|
|
$scratch = Resolve-DDirectory $scratch "M4 runtime scratch" $false
|
|
$runtimeIdentityPath = Join-Path $scratch "runtime-identity.json"
|
|
Write-Utf8NoBom $runtimeIdentityPath "{}"
|
|
|
|
$dockerPayload = Convert-ToDockerPath $payload
|
|
$dockerOutput = Convert-ToDockerPath $output
|
|
$dockerScratch = Convert-ToDockerPath $scratch
|
|
$dockerArguments = @(
|
|
"create",
|
|
"--name", $candidateName,
|
|
"--network", ("container:{0}" -f $tritonExpected.name),
|
|
"--read-only",
|
|
"--security-opt", "no-new-privileges:true",
|
|
"--cap-drop", "ALL",
|
|
"--pids-limit", "256",
|
|
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
|
|
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
|
"-e", ("PYTHONPATH=/release/{0}:/opt/media:/opt/opencv:/opt/pillow" -f $wheelName),
|
|
"-v", ("{0}:/release:ro" -f $dockerPayload),
|
|
"-v", ("{0}:/output:rw" -f $dockerOutput),
|
|
"-v", ("{0}:/run/mission-core:ro" -f $dockerScratch)
|
|
)
|
|
|
|
foreach ($entry in $descriptor.inputs.PSObject.Properties) {
|
|
$input = $entry.Value
|
|
if ($null -ne $input.container_path) {
|
|
$dockerArguments += @(
|
|
"-v",
|
|
("{0}:{1}:ro" -f (Convert-ToDockerPath $input.host_path), $input.container_path)
|
|
)
|
|
}
|
|
}
|
|
foreach ($dependency in $descriptor.dependencies) {
|
|
$dockerArguments += @(
|
|
"-v",
|
|
("{0}:{1}:ro" -f (Convert-ToDockerPath $dependency.host_path), $dependency.container_path)
|
|
)
|
|
}
|
|
$dockerArguments += @(
|
|
"--entrypoint", "python3",
|
|
$imageRef,
|
|
"-m", "k1link.perception.detector_replay_cli",
|
|
"--baseline", ("/release/{0}" -f $baselineName),
|
|
"--camera-summary", $descriptor.inputs.camera_summary.container_path,
|
|
"--camera-index", $descriptor.inputs.camera_index.container_path,
|
|
"--source-pack-manifest", $descriptor.inputs.source_pack_manifest.container_path,
|
|
"--source-pack", $descriptor.inputs.source_pack.container_path,
|
|
"--video", $descriptor.inputs.video.container_path,
|
|
"--valid-fov-mask", $descriptor.inputs.valid_fov_mask.container_path,
|
|
"--triton-origin", $descriptor.container.triton_origin,
|
|
"--runtime-identity", "/run/mission-core/runtime-identity.json",
|
|
"--output-root", "/output"
|
|
)
|
|
|
|
$candidateCreated = $false
|
|
$runFailure = $null
|
|
try {
|
|
$candidateId = (& docker @dockerArguments).Trim()
|
|
Assert-LastExitCode "M4 candidate creation"
|
|
if ($candidateId -notmatch "^[a-f0-9]{64}$") {
|
|
throw "M4 candidate id is invalid"
|
|
}
|
|
$candidateCreated = $true
|
|
$candidate = Get-ContainerIdentity $candidateName
|
|
if (
|
|
$candidate.Id -cne $candidateId -or
|
|
$candidate.Image -cne $descriptor.container.image_id -or
|
|
$candidate.HostConfig.NetworkMode -cne ("container:{0}" -f $triton.Id) -or
|
|
-not $candidate.HostConfig.ReadonlyRootfs
|
|
) {
|
|
throw "M4 candidate isolation contract changed"
|
|
}
|
|
$runtimeIdentity = [ordered]@{
|
|
schema_version = "missioncore.perception-runtime-identity/v1"
|
|
worker_id = "worker-006"
|
|
worker_node = $env:COMPUTERNAME
|
|
worker_container_id = $candidate.Id
|
|
worker_image_id = $candidate.Image
|
|
triton_container_id = $triton.Id
|
|
triton_image_id = $triton.Image
|
|
triton_model_sha256 = $descriptor.inputs.yolox_model.sha256
|
|
triton_model_config_sha256 = $descriptor.inputs.yolox_config.sha256
|
|
valid_fov_mask_sha256 = $descriptor.inputs.valid_fov_mask.sha256
|
|
artifact_sha256 = $ExpectedArtifactSha256
|
|
code_revision = $descriptor.code_revision
|
|
source_mount_read_only = $true
|
|
model_service_reused = $true
|
|
public_worker_port_added = $false
|
|
same_host_tensor_transport = $true
|
|
}
|
|
Write-Utf8NoBom $runtimeIdentityPath ($runtimeIdentity | ConvertTo-Json -Depth 4)
|
|
Write-Output ("PATCH_ID={0}" -f $patchId)
|
|
Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256)
|
|
Write-Output ("CANDIDATE_CONTAINER_ID={0}" -f $candidate.Id)
|
|
Write-Output ("DURABLE_WORKER_ID={0}" -f $durable.Id)
|
|
Write-Output ("TRITON_CONTAINER_ID={0}" -f $triton.Id)
|
|
& docker start --attach $candidateName
|
|
Assert-LastExitCode "M4 detector shadow"
|
|
} catch {
|
|
$runFailure = $_
|
|
} finally {
|
|
if ($candidateCreated) {
|
|
& docker rm --force $candidateName *> $null
|
|
if ($LASTEXITCODE -ne 0 -and $null -eq $runFailure) {
|
|
$runFailure = "M4 candidate cleanup failed"
|
|
}
|
|
}
|
|
Remove-Item -LiteralPath $scratch -Force -Recurse -ErrorAction SilentlyContinue
|
|
}
|
|
|
|
$null = Assert-ContainerIdentity $predecessor.name $predecessor.container_id (
|
|
$predecessor.image_id
|
|
) $false
|
|
$null = Assert-ContainerIdentity $tritonExpected.name $tritonExpected.container_id (
|
|
$tritonExpected.image_id
|
|
) $true
|
|
$freeAfter = Assert-FreeSpace "completed"
|
|
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore)
|
|
Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f $freeAfter)
|
|
Write-Output "DURABLE_WORKER_ACTION=none"
|
|
Write-Output "TRITON_ACTION=none"
|
|
if ($null -ne $runFailure) {
|
|
throw $runFailure
|
|
}
|