[CmdletBinding()] param( [Parameter(Mandatory = $true)] [string]$ReleaseRoot, [Parameter(Mandatory = $true)] [string]$ArtifactPath, [Parameter(Mandatory = $true)] [ValidatePattern("^[a-f0-9]{64}$")] [string]$ExpectedArtifactSha256, [string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m4-detector-replay", [ValidateRange(1, 1000)] [int]$FreeGiBFloor = 300, [switch]$PreflightOnly ) $ErrorActionPreference = "Stop" $ProgressPreference = "SilentlyContinue" function Assert-LastExitCode([string]$Operation) { if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" } } function Get-Sha256([string]$Path) { return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() } function Assert-FileSha256([string]$Path, [string]$Expected, [string]$Label) { $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) { throw "$Label must be a regular file" } $observed = Get-Sha256 $item.FullName if ($observed -cne $Expected) { throw "$Label SHA-256 changed: expected $Expected, observed $observed" } return $item.FullName } function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) { if ($Create -and -not (Test-Path -LiteralPath $Path)) { $null = New-Item -ItemType Directory -Path $Path } $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force $root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") if ( -not $item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $root -ine "D:" ) { throw "$Label must be a real D: directory" } return $item.FullName } function Convert-ToDockerPath([string]$Path) { return $Path.Replace("\", "/") } function Assert-FreeSpace([string]$Phase) { $free = [int64](Get-PSDrive -Name D).Free $floor = [int64]$FreeGiBFloor * 1GB Write-Output ( "DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3}" -f $Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor ) if ($free -lt ($floor + 1GB)) { throw "D: lacks the guarded M4 reserve during $Phase" } return $free } function Get-ContainerIdentity([string]$Name) { $json = & docker inspect $Name Assert-LastExitCode "Docker inspection for $Name" $rows = @($json | ConvertFrom-Json) if ($rows.Count -ne 1) { throw "Docker identity for $Name is not unique" } return $rows[0] } function Assert-ContainerIdentity( [string]$Name, [string]$ExpectedId, [string]$ExpectedImageId, [bool]$RequireHealthy ) { $container = Get-ContainerIdentity $Name if ( $container.Id -cne $ExpectedId -or $container.Image -cne $ExpectedImageId -or -not $container.State.Running ) { throw "$Name identity or running state changed" } if ($RequireHealthy -and $container.State.Health.Status -cne "healthy") { throw "$Name is not healthy" } return $container } function Get-RequiredText([object]$Value, [string]$Label) { if ($Value -isnot [string] -or [string]::IsNullOrWhiteSpace($Value)) { throw "$Label must be a non-empty string" } return [string]$Value } function Write-Utf8NoBom([string]$Path, [string]$Value) { $encoding = New-Object System.Text.UTF8Encoding($false) [IO.File]::WriteAllText($Path, $Value, $encoding) } function Get-DirectoryTreeSha256([string]$Root, [object[]]$Files) { $rootPrefix = $Root.TrimEnd("\") + "\" $rows = @( foreach ($file in $Files) { if (-not $file.FullName.StartsWith( $rootPrefix, [StringComparison]::OrdinalIgnoreCase )) { throw "M4 dependency file escaped its declared root" } [pscustomobject]@{ Relative = $file.FullName.Substring($rootPrefix.Length).Replace("\", "/") File = $file } } ) $rows = @($rows | Sort-Object -Property Relative -CaseSensitive) $digest = [Security.Cryptography.IncrementalHash]::CreateHash( [Security.Cryptography.HashAlgorithmName]::SHA256 ) $encoding = New-Object System.Text.UTF8Encoding($false) try { foreach ($row in $rows) { $fileSha256 = Get-Sha256 $row.File.FullName $record = "{0}`t{1}`t{2}`n" -f $row.Relative, $row.File.Length, $fileSha256 $digest.AppendData($encoding.GetBytes($record)) } return ([BitConverter]::ToString($digest.GetHashAndReset())).Replace("-", "").ToLowerInvariant() } finally { $digest.Dispose() } } if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") { throw "M4 shadow release is pinned to DESKTOP-OPJ8J04" } $release = Resolve-DDirectory $ReleaseRoot "M4 release root" $false $payload = Resolve-DDirectory (Join-Path $release "payload") "M4 payload" $false $artifact = Assert-FileSha256 $ArtifactPath $ExpectedArtifactSha256 "M4 release artifact" $descriptorPath = Join-Path $payload "mission-core-worker-shadow-v1.json" $descriptor = Get-Content -LiteralPath $descriptorPath -Raw | ConvertFrom-Json if ( $descriptor.schema_version -cne "nodedc.mission-core-worker.shadow-release/v1" -or $descriptor.component -cne "mission-core-worker" -or $descriptor.artifact_type -cne "shadow-release" -or $descriptor.transition -cne "m4-detector-shadow-v1" -or $descriptor.boundary.repository -cne "NODEDC_MISSION_CORE" -or $descriptor.boundary.nodedc_platform_repository -ne $false -or $descriptor.boundary.external_deploy_registry -ne $false -or $descriptor.boundary.server_docker_runtime -ne $false -or $descriptor.host.node -cne $env:COMPUTERNAME -or $descriptor.host.worker_id -cne "worker-006" -or $descriptor.acceptance.expected_frames -ne 4489 -or [double]$descriptor.acceptance.minimum_end_to_end_fps -ne 10.004 -or $descriptor.acceptance.failed_frames -ne 0 -or $descriptor.acceptance.class_routing_used -ne $false ) { throw "M4 shadow descriptor contract changed" } $wheelName = Get-RequiredText $descriptor.release.wheel.name "Wheel name" $baselineName = Get-RequiredText $descriptor.release.baseline.name "Baseline name" $wheelPath = Assert-FileSha256 ( Join-Path $payload $wheelName ) $descriptor.release.wheel.sha256 "M4 runtime wheel" $baselinePath = Assert-FileSha256 ( Join-Path $payload $baselineName ) $descriptor.release.baseline.sha256 "M4 baseline" foreach ($entry in $descriptor.inputs.PSObject.Properties) { $input = $entry.Value $null = Assert-FileSha256 $input.host_path $input.sha256 ("M4 input {0}" -f $entry.Name) } foreach ($dependency in $descriptor.dependencies) { $root = Resolve-DDirectory $dependency.host_path ("M4 dependency {0}" -f $dependency.id) $false $files = @() foreach ($include in $dependency.includes) { $candidate = if ($include -eq ".") { $root } else { Join-Path $root $include } $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $candidate).Path -Force if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) { throw "M4 dependency $($dependency.id) contains a reparse-point root" } if ($item.PSIsContainer) { $files += @(Get-ChildItem -LiteralPath $item.FullName -File -Recurse -Force) } else { $files += @($item) } } $uniqueFiles = @($files | Sort-Object -Property FullName -Unique) $bytes = [int64]0 foreach ($file in $uniqueFiles) { if ($file.Attributes -band [IO.FileAttributes]::ReparsePoint) { throw "M4 dependency $($dependency.id) contains a reparse-point file" } $bytes += [int64]$file.Length } $treeSha256 = Get-DirectoryTreeSha256 $root $uniqueFiles if ( $uniqueFiles.Count -ne [int]$dependency.file_count -or $bytes -ne [int64]$dependency.bytes -or $treeSha256 -cne [string]$dependency.tree_sha256 ) { throw ( "M4 dependency {0} inventory changed: expected {1}/{2}/{3}, observed {4}/{5}/{6}" -f $dependency.id, $dependency.file_count, $dependency.bytes, $dependency.tree_sha256, $uniqueFiles.Count, $bytes, $treeSha256 ) } } $imageRef = Get-RequiredText $descriptor.container.image_ref "Container image reference" & docker image inspect $imageRef *> $null Assert-LastExitCode "Pinned M4 image inspection" $predecessor = $descriptor.predecessor.durable_worker $tritonExpected = $descriptor.predecessor.triton $durable = Assert-ContainerIdentity ( Get-RequiredText $predecessor.name "Durable worker name" ) $predecessor.container_id $predecessor.image_id $false $triton = Assert-ContainerIdentity ( Get-RequiredText $tritonExpected.name "Triton name" ) $tritonExpected.container_id $tritonExpected.image_id $true $output = Resolve-DDirectory $OutputRoot "M4 output root" $true $freeBefore = Assert-FreeSpace "preflight" $patchId = Get-RequiredText $descriptor.patch_id "Patch id" if ($patchId -notmatch "^[A-Za-z0-9._-]{1,96}$") { throw "M4 patch id is invalid" } if ($PreflightOnly) { Write-Output ("PATCH_ID={0}" -f $patchId) Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256) Write-Output ("DURABLE_WORKER_ID={0}" -f $durable.Id) Write-Output ("TRITON_CONTAINER_ID={0}" -f $triton.Id) Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore) Write-Output "PREFLIGHT=accepted" return } $candidateName = "ndc-mission-core-m4-detector-shadow" if (& docker ps -a --format "{{.Names}}" --filter "name=^/$candidateName$") { throw "M4 candidate container already exists" } $scratch = Join-Path $output (".runtime-{0}" -f $patchId) if (Test-Path -LiteralPath $scratch) { throw "M4 runtime scratch already exists" } $null = New-Item -ItemType Directory -Path $scratch $scratch = Resolve-DDirectory $scratch "M4 runtime scratch" $false $runtimeIdentityPath = Join-Path $scratch "runtime-identity.json" Write-Utf8NoBom $runtimeIdentityPath "{}" $dockerPayload = Convert-ToDockerPath $payload $dockerOutput = Convert-ToDockerPath $output $dockerScratch = Convert-ToDockerPath $scratch $dockerArguments = @( "create", "--name", $candidateName, "--network", ("container:{0}" -f $tritonExpected.name), "--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL", "--pids-limit", "256", "--tmpfs", "/tmp:rw,noexec,nosuid,size=2g", "-e", "PYTHONDONTWRITEBYTECODE=1", "-e", ("PYTHONPATH=/release/{0}:/opt/media:/opt/opencv:/opt/pillow" -f $wheelName), "-v", ("{0}:/release:ro" -f $dockerPayload), "-v", ("{0}:/output:rw" -f $dockerOutput), "-v", ("{0}:/run/mission-core:ro" -f $dockerScratch) ) foreach ($entry in $descriptor.inputs.PSObject.Properties) { $input = $entry.Value if ($null -ne $input.container_path) { $dockerArguments += @( "-v", ("{0}:{1}:ro" -f (Convert-ToDockerPath $input.host_path), $input.container_path) ) } } foreach ($dependency in $descriptor.dependencies) { $dockerArguments += @( "-v", ("{0}:{1}:ro" -f (Convert-ToDockerPath $dependency.host_path), $dependency.container_path) ) } $dockerArguments += @( "--entrypoint", "python3", $imageRef, "-m", "k1link.perception.detector_replay_cli", "--baseline", ("/release/{0}" -f $baselineName), "--camera-summary", $descriptor.inputs.camera_summary.container_path, "--camera-index", $descriptor.inputs.camera_index.container_path, "--source-pack-manifest", $descriptor.inputs.source_pack_manifest.container_path, "--source-pack", $descriptor.inputs.source_pack.container_path, "--video", $descriptor.inputs.video.container_path, "--valid-fov-mask", $descriptor.inputs.valid_fov_mask.container_path, "--triton-origin", $descriptor.container.triton_origin, "--runtime-identity", "/run/mission-core/runtime-identity.json", "--output-root", "/output" ) $candidateCreated = $false $runFailure = $null try { $candidateId = (& docker @dockerArguments).Trim() Assert-LastExitCode "M4 candidate creation" if ($candidateId -notmatch "^[a-f0-9]{64}$") { throw "M4 candidate id is invalid" } $candidateCreated = $true $candidate = Get-ContainerIdentity $candidateName if ( $candidate.Id -cne $candidateId -or $candidate.Image -cne $descriptor.container.image_id -or $candidate.HostConfig.NetworkMode -cne ("container:{0}" -f $triton.Id) -or -not $candidate.HostConfig.ReadonlyRootfs ) { throw "M4 candidate isolation contract changed" } $runtimeIdentity = [ordered]@{ schema_version = "missioncore.perception-runtime-identity/v1" worker_id = "worker-006" worker_node = $env:COMPUTERNAME worker_container_id = $candidate.Id worker_image_id = $candidate.Image triton_container_id = $triton.Id triton_image_id = $triton.Image triton_model_sha256 = $descriptor.inputs.yolox_model.sha256 triton_model_config_sha256 = $descriptor.inputs.yolox_config.sha256 valid_fov_mask_sha256 = $descriptor.inputs.valid_fov_mask.sha256 artifact_sha256 = $ExpectedArtifactSha256 code_revision = $descriptor.code_revision source_mount_read_only = $true model_service_reused = $true public_worker_port_added = $false same_host_tensor_transport = $true } Write-Utf8NoBom $runtimeIdentityPath ($runtimeIdentity | ConvertTo-Json -Depth 4) Write-Output ("PATCH_ID={0}" -f $patchId) Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256) Write-Output ("CANDIDATE_CONTAINER_ID={0}" -f $candidate.Id) Write-Output ("DURABLE_WORKER_ID={0}" -f $durable.Id) Write-Output ("TRITON_CONTAINER_ID={0}" -f $triton.Id) & docker start --attach $candidateName Assert-LastExitCode "M4 detector shadow" } catch { $runFailure = $_ } finally { if ($candidateCreated) { & docker rm --force $candidateName *> $null if ($LASTEXITCODE -ne 0 -and $null -eq $runFailure) { $runFailure = "M4 candidate cleanup failed" } } Remove-Item -LiteralPath $scratch -Force -Recurse -ErrorAction SilentlyContinue } $null = Assert-ContainerIdentity $predecessor.name $predecessor.container_id ( $predecessor.image_id ) $false $null = Assert-ContainerIdentity $tritonExpected.name $tritonExpected.container_id ( $tritonExpected.image_id ) $true $freeAfter = Assert-FreeSpace "completed" Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore) Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f $freeAfter) Write-Output "DURABLE_WORKER_ACTION=none" Write-Output "TRITON_ACTION=none" if ($null -ne $runFailure) { throw $runFailure }