Добавление канонического графа M4.7
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,376 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ReleaseRoot,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ArtifactPath,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[a-f0-9]{64}$")]
|
||||
[string]$ExpectedArtifactSha256,
|
||||
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m47-reference-graph",
|
||||
[ValidateRange(1, 1000)]
|
||||
[int]$FreeGiBFloor = 300,
|
||||
[switch]$PreflightOnly
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode([string]$Operation) {
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-Sha256([string]$Path) {
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Assert-FileSha256([string]$Path, [string]$Expected, [string]$Label) {
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
throw "$Label must be a regular file"
|
||||
}
|
||||
$observed = Get-Sha256 $item.FullName
|
||||
if ($observed -cne $Expected) {
|
||||
throw "$Label SHA-256 changed: expected $Expected, observed $observed"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) {
|
||||
if ($Create -and -not (Test-Path -LiteralPath $Path)) {
|
||||
$null = New-Item -ItemType Directory -Path $Path
|
||||
}
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
||||
if (
|
||||
-not $item.PSIsContainer -or
|
||||
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
||||
$root -ine "D:"
|
||||
) {
|
||||
throw "$Label must be a real D: directory"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath([string]$Path) {
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
function Assert-FreeSpace([string]$Phase) {
|
||||
$free = [int64](Get-PSDrive -Name D).Free
|
||||
$floor = [int64]$FreeGiBFloor * 1GB
|
||||
Write-Host (
|
||||
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3}" -f
|
||||
$Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor
|
||||
)
|
||||
if ($free -lt ($floor + 1GB)) {
|
||||
throw "D: lacks the guarded M4.7 reserve during $Phase"
|
||||
}
|
||||
return $free
|
||||
}
|
||||
|
||||
function Get-ContainerIdentity([string]$Name) {
|
||||
$json = & docker inspect $Name
|
||||
Assert-LastExitCode "Docker inspection for $Name"
|
||||
$rows = @($json | ConvertFrom-Json)
|
||||
if ($rows.Count -ne 1) {
|
||||
throw "Docker identity for $Name is not unique"
|
||||
}
|
||||
return $rows[0]
|
||||
}
|
||||
|
||||
function Assert-ContainerIdentity(
|
||||
[string]$Name,
|
||||
[string]$ExpectedId,
|
||||
[string]$ExpectedImageId,
|
||||
[bool]$RequireHealthy
|
||||
) {
|
||||
$container = Get-ContainerIdentity $Name
|
||||
if (
|
||||
$container.Id -cne $ExpectedId -or
|
||||
$container.Image -cne $ExpectedImageId -or
|
||||
-not $container.State.Running
|
||||
) {
|
||||
throw "$Name identity or running state changed"
|
||||
}
|
||||
if ($RequireHealthy -and $container.State.Health.Status -cne "healthy") {
|
||||
throw "$Name is not healthy"
|
||||
}
|
||||
return $container
|
||||
}
|
||||
|
||||
function Get-DirectoryTreeSha256([string]$Root, [object[]]$Files) {
|
||||
$rootPrefix = $Root.TrimEnd("\") + "\"
|
||||
$rows = @(
|
||||
foreach ($file in $Files) {
|
||||
if (-not $file.FullName.StartsWith($rootPrefix, [StringComparison]::OrdinalIgnoreCase)) {
|
||||
throw "M4.7 dependency file escaped its declared root"
|
||||
}
|
||||
[pscustomobject]@{
|
||||
Relative = $file.FullName.Substring($rootPrefix.Length).Replace("\", "/")
|
||||
File = $file
|
||||
}
|
||||
}
|
||||
)
|
||||
$rows = @($rows | Sort-Object -Property Relative -CaseSensitive)
|
||||
$digest = [Security.Cryptography.IncrementalHash]::CreateHash(
|
||||
[Security.Cryptography.HashAlgorithmName]::SHA256
|
||||
)
|
||||
$encoding = New-Object System.Text.UTF8Encoding($false)
|
||||
try {
|
||||
foreach ($row in $rows) {
|
||||
$record = "{0}`t{1}`t{2}`n" -f `
|
||||
$row.Relative, $row.File.Length, (Get-Sha256 $row.File.FullName)
|
||||
$digest.AppendData($encoding.GetBytes($record))
|
||||
}
|
||||
return ([BitConverter]::ToString($digest.GetHashAndReset())).Replace("-", "").ToLowerInvariant()
|
||||
} finally {
|
||||
$digest.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Write-Utf8NoBom([string]$Path, [string]$Value) {
|
||||
$encoding = New-Object System.Text.UTF8Encoding($false)
|
||||
[IO.File]::WriteAllText($Path, $Value, $encoding)
|
||||
}
|
||||
|
||||
if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
|
||||
throw "M4.7 shadow release is pinned to DESKTOP-OPJ8J04"
|
||||
}
|
||||
|
||||
$release = Resolve-DDirectory $ReleaseRoot "M4.7 release root" $false
|
||||
$payload = Resolve-DDirectory (Join-Path $release "payload") "M4.7 payload" $false
|
||||
$artifact = Assert-FileSha256 $ArtifactPath $ExpectedArtifactSha256 "M4.7 release artifact"
|
||||
$descriptorPath = Join-Path $payload "mission-core-worker-m47-graph-shadow-v2.json"
|
||||
$descriptor = Get-Content -LiteralPath $descriptorPath -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$descriptor.schema_version -cne "nodedc.mission-core-worker.shadow-release/v2" -or
|
||||
$descriptor.transition -cne "m47-canonical-graph-shadow-v1" -or
|
||||
$descriptor.component -cne "mission-core-worker" -or
|
||||
$descriptor.host.node -cne $env:COMPUTERNAME -or
|
||||
$descriptor.host.worker_id -cne "worker-006" -or
|
||||
$descriptor.boundary.repository -cne "NODEDC_MISSION_CORE" -or
|
||||
$descriptor.boundary.external_deploy_registry -ne $false -or
|
||||
$descriptor.acceptance.run_mode -cne "lossless-replay" -or
|
||||
$descriptor.acceptance.expected_frames -ne 4489 -or
|
||||
$descriptor.acceptance.delivered_frames -ne 4489 -or
|
||||
$descriptor.acceptance.failed_frames -ne 0 -or
|
||||
$descriptor.acceptance.stale_frames -ne 0 -or
|
||||
$descriptor.acceptance.superseded_frames -ne 0 -or
|
||||
$descriptor.acceptance.accepted_parity -ne $true -or
|
||||
$descriptor.readiness.graph.graph_id -cne "reference-perception-graph/v2" -or
|
||||
$descriptor.readiness.graph.actuation_allowed -ne $false
|
||||
) {
|
||||
throw "M4.7 shadow descriptor contract changed"
|
||||
}
|
||||
|
||||
$null = Assert-FileSha256 $PSCommandPath $descriptor.release.runner.sha256 "M4.7 runner"
|
||||
$wheelPath = Assert-FileSha256 (
|
||||
Join-Path $payload $descriptor.release.wheel.name
|
||||
) $descriptor.release.wheel.sha256 "M4.7 wheel"
|
||||
foreach ($entry in $descriptor.release.configs.PSObject.Properties) {
|
||||
$null = Assert-FileSha256 (
|
||||
Join-Path $payload $entry.Value.name
|
||||
) $entry.Value.sha256 ("M4.7 config {0}" -f $entry.Name)
|
||||
}
|
||||
foreach ($entry in $descriptor.inputs.PSObject.Properties) {
|
||||
$null = Assert-FileSha256 $entry.Value.host_path $entry.Value.sha256 (
|
||||
"M4.7 input {0}" -f $entry.Name
|
||||
)
|
||||
}
|
||||
foreach ($dependency in $descriptor.dependencies) {
|
||||
$root = Resolve-DDirectory $dependency.host_path ("M4.7 dependency {0}" -f $dependency.id) $false
|
||||
$files = @()
|
||||
foreach ($include in $dependency.includes) {
|
||||
$candidate = if ($include -eq ".") { $root } else { Join-Path $root $include }
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $candidate).Path -Force
|
||||
if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) {
|
||||
throw "M4.7 dependency $($dependency.id) contains a reparse point"
|
||||
}
|
||||
if ($item.PSIsContainer) {
|
||||
$files += @(Get-ChildItem -LiteralPath $item.FullName -File -Recurse -Force)
|
||||
} else {
|
||||
$files += @($item)
|
||||
}
|
||||
}
|
||||
$files = @($files | Sort-Object -Property FullName -Unique)
|
||||
$bytes = [int64]0
|
||||
foreach ($file in $files) {
|
||||
if ($file.Attributes -band [IO.FileAttributes]::ReparsePoint) {
|
||||
throw "M4.7 dependency $($dependency.id) contains a reparse-point file"
|
||||
}
|
||||
$bytes += [int64]$file.Length
|
||||
}
|
||||
if (
|
||||
$files.Count -ne [int]$dependency.file_count -or
|
||||
$bytes -ne [int64]$dependency.bytes -or
|
||||
(Get-DirectoryTreeSha256 $root $files) -cne [string]$dependency.tree_sha256
|
||||
) {
|
||||
throw "M4.7 dependency $($dependency.id) inventory changed"
|
||||
}
|
||||
}
|
||||
|
||||
$imageRef = [string]$descriptor.container.image_ref
|
||||
& docker image inspect $imageRef *> $null
|
||||
Assert-LastExitCode "Pinned M4.7 image inspection"
|
||||
$predecessor = $descriptor.predecessor.durable_worker
|
||||
$tritonExpected = $descriptor.predecessor.triton
|
||||
$durable = Assert-ContainerIdentity (
|
||||
$predecessor.name
|
||||
) $predecessor.container_id $predecessor.image_id $false
|
||||
$triton = Assert-ContainerIdentity (
|
||||
$tritonExpected.name
|
||||
) $tritonExpected.container_id $tritonExpected.image_id $true
|
||||
$output = Resolve-DDirectory $OutputRoot "M4.7 output root" $true
|
||||
$freeBefore = Assert-FreeSpace "preflight"
|
||||
|
||||
if ($PreflightOnly) {
|
||||
Write-Output ("PATCH_ID={0}" -f $descriptor.patch_id)
|
||||
Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256)
|
||||
Write-Output ("DURABLE_WORKER_ID={0}" -f $durable.Id)
|
||||
Write-Output ("TRITON_CONTAINER_ID={0}" -f $triton.Id)
|
||||
Write-Output "PROVIDER_READINESS=accepted"
|
||||
Write-Output "GRAPH_READINESS=not-run"
|
||||
Write-Output "PREFLIGHT=accepted"
|
||||
return
|
||||
}
|
||||
|
||||
$candidateName = "ndc-mission-core-m47-graph-shadow"
|
||||
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$candidateName$") {
|
||||
throw "M4.7 candidate container already exists"
|
||||
}
|
||||
$scratch = Join-Path $output (".runtime-{0}" -f $descriptor.patch_id)
|
||||
if (Test-Path -LiteralPath $scratch) {
|
||||
throw "M4.7 runtime scratch already exists"
|
||||
}
|
||||
$null = New-Item -ItemType Directory -Path $scratch
|
||||
$scratch = Resolve-DDirectory $scratch "M4.7 runtime scratch" $false
|
||||
$runtimeIdentityPath = Join-Path $scratch "runtime-identity.json"
|
||||
$dockerPayload = Convert-ToDockerPath $payload
|
||||
$dockerOutput = Convert-ToDockerPath $output
|
||||
$dockerScratch = Convert-ToDockerPath $scratch
|
||||
$dockerArguments = @(
|
||||
"create",
|
||||
"--name", $candidateName,
|
||||
"--network", ("container:{0}" -f $tritonExpected.name),
|
||||
"--read-only",
|
||||
"--security-opt", "no-new-privileges:true",
|
||||
"--cap-drop", "ALL",
|
||||
"--pids-limit", "256",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
|
||||
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
||||
"-e", ("PYTHONPATH=/release/{0}:/opt/media:/opt/opencv:/opt/pillow" -f $descriptor.release.wheel.name),
|
||||
"-v", ("{0}:/release:ro" -f $dockerPayload),
|
||||
"-v", ("{0}:/output:rw" -f $dockerOutput),
|
||||
"-v", ("{0}:/run/mission-core:ro" -f $dockerScratch)
|
||||
)
|
||||
foreach ($entry in $descriptor.inputs.PSObject.Properties) {
|
||||
if ($null -ne $entry.Value.container_path) {
|
||||
$dockerArguments += @(
|
||||
"-v", ("{0}:{1}:ro" -f (
|
||||
Convert-ToDockerPath $entry.Value.host_path
|
||||
), $entry.Value.container_path)
|
||||
)
|
||||
}
|
||||
}
|
||||
foreach ($dependency in $descriptor.dependencies) {
|
||||
$dockerArguments += @(
|
||||
"-v", ("{0}:{1}:ro" -f (
|
||||
Convert-ToDockerPath $dependency.host_path
|
||||
), $dependency.container_path)
|
||||
)
|
||||
}
|
||||
$dockerArguments += @(
|
||||
"--entrypoint", "python3",
|
||||
$imageRef,
|
||||
"-m", "k1link.perception.reference_graph_cli",
|
||||
"--graph-config", "/release/m4-reference-graph-v2.json",
|
||||
"--baseline-profile", "/release/m4-recorded-realtime-baseline-v1.json",
|
||||
"--geometry-profile", "/release/m4-geometry-association-v1.json",
|
||||
"--temporal-motion-profile", "/release/m4-temporal-motion-v1.json",
|
||||
"--rolling-map-profile", "/release/m4-rolling-local-map-v1.json",
|
||||
"--threat-profile", "/release/m4-replay-threat-v3.json",
|
||||
"--camera-index", $descriptor.inputs.camera_index.container_path,
|
||||
"--source-pack", $descriptor.inputs.source_pack.container_path,
|
||||
"--local-surface", $descriptor.inputs.local_surface.container_path,
|
||||
"--video", $descriptor.inputs.video.container_path,
|
||||
"--valid-fov-mask", $descriptor.inputs.valid_fov_mask.container_path,
|
||||
"--temporal-parity-frames", $descriptor.inputs.accepted_temporal_frames.container_path,
|
||||
"--threat-parity-frames", $descriptor.inputs.accepted_threat_frames.container_path,
|
||||
"--triton-origin", $descriptor.container.triton_origin,
|
||||
"--mode", $descriptor.acceptance.run_mode,
|
||||
"--expected-frames", ([string]$descriptor.acceptance.expected_frames),
|
||||
"--output-root", "/output"
|
||||
)
|
||||
|
||||
$candidateCreated = $false
|
||||
$runFailure = $null
|
||||
try {
|
||||
$candidateId = (& docker @dockerArguments).Trim()
|
||||
Assert-LastExitCode "M4.7 candidate creation"
|
||||
if ($candidateId -notmatch "^[a-f0-9]{64}$") {
|
||||
throw "M4.7 candidate id is invalid"
|
||||
}
|
||||
$candidateCreated = $true
|
||||
$candidate = Get-ContainerIdentity $candidateName
|
||||
if (
|
||||
$candidate.Id -cne $candidateId -or
|
||||
$candidate.Image -cne $descriptor.container.image_id -or
|
||||
$candidate.HostConfig.NetworkMode -cne ("container:{0}" -f $triton.Id) -or
|
||||
-not $candidate.HostConfig.ReadonlyRootfs
|
||||
) {
|
||||
throw "M4.7 candidate isolation contract changed"
|
||||
}
|
||||
$runtimeIdentity = [ordered]@{
|
||||
schema_version = "missioncore.reference-graph-runtime-identity/v1"
|
||||
worker_id = "worker-006"
|
||||
worker_node = $env:COMPUTERNAME
|
||||
worker_container_id = $candidate.Id
|
||||
worker_image_id = $candidate.Image
|
||||
triton_container_id = $triton.Id
|
||||
triton_image_id = $triton.Image
|
||||
artifact_sha256 = $ExpectedArtifactSha256
|
||||
code_revision = $descriptor.code_revision
|
||||
graph_id = $descriptor.readiness.graph.graph_id
|
||||
source_mount_read_only = $true
|
||||
model_service_reused = $true
|
||||
public_worker_port_added = $false
|
||||
commands_enabled = $false
|
||||
actuation_allowed = $false
|
||||
}
|
||||
Write-Utf8NoBom $runtimeIdentityPath ($runtimeIdentity | ConvertTo-Json -Depth 4)
|
||||
Write-Output ("PATCH_ID={0}" -f $descriptor.patch_id)
|
||||
Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256)
|
||||
Write-Output ("CANDIDATE_CONTAINER_ID={0}" -f $candidate.Id)
|
||||
Write-Output "PROVIDER_READINESS=accepted"
|
||||
& docker start --attach $candidateName
|
||||
Assert-LastExitCode "M4.7 canonical graph shadow"
|
||||
Write-Output "GRAPH_READINESS=accepted"
|
||||
} catch {
|
||||
$runFailure = $_
|
||||
} finally {
|
||||
if ($candidateCreated) {
|
||||
& docker rm --force $candidateName *> $null
|
||||
if ($LASTEXITCODE -ne 0 -and $null -eq $runFailure) {
|
||||
$runFailure = "M4.7 candidate cleanup failed"
|
||||
}
|
||||
}
|
||||
Remove-Item -LiteralPath $scratch -Force -Recurse -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
$null = Assert-ContainerIdentity $predecessor.name $predecessor.container_id (
|
||||
$predecessor.image_id
|
||||
) $false
|
||||
$null = Assert-ContainerIdentity $tritonExpected.name $tritonExpected.container_id (
|
||||
$tritonExpected.image_id
|
||||
) $true
|
||||
$freeAfter = Assert-FreeSpace "completed"
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f $freeAfter)
|
||||
Write-Output "DURABLE_WORKER_ACTION=none"
|
||||
Write-Output "TRITON_ACTION=none"
|
||||
if ($null -ne $runFailure) {
|
||||
throw $runFailure
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the deterministic Worker 006 M4.7 canonical-graph shadow artifact."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_TEMPLATE = REPOSITORY_ROOT / "config/deployment/mission-core-worker-shadow-v1.template.json"
|
||||
RUNNER = REPOSITORY_ROOT / "scripts/Invoke-M47CanonicalGraphShadow.ps1"
|
||||
WHEEL_NAME = "nodedc_mission_core-0.1.0-py3-none-any.whl"
|
||||
DESCRIPTOR_NAME = "mission-core-worker-m47-graph-shadow-v2.json"
|
||||
PATCH_ID = re.compile(r"^[A-Za-z0-9._-]{1,96}$")
|
||||
EXPECTED_BASE_TEMPLATE_SHA256 = (
|
||||
"319e7ac7f14e5911ad44234c9ec918c73e11a3406724d5cee3e2ef64bb036e0c"
|
||||
)
|
||||
CONFIG_PATHS = (
|
||||
Path("config/perception/m4-recorded-realtime-baseline-v1.json"),
|
||||
Path("config/perception/m4-reference-graph-v2.json"),
|
||||
Path("config/perception/m4-geometry-association-v1.json"),
|
||||
Path("config/perception/m4-temporal-motion-v1.json"),
|
||||
Path("config/perception/m4-rolling-local-map-v1.json"),
|
||||
Path("config/perception/m4-replay-threat-v3.json"),
|
||||
)
|
||||
|
||||
|
||||
class ArtifactBuildError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def git_revision(*, require_clean: bool) -> str:
|
||||
revision_result = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"],
|
||||
cwd=REPOSITORY_ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
revision = revision_result.stdout.strip()
|
||||
if re.fullmatch(r"[a-f0-9]{40}", revision) is None:
|
||||
raise ArtifactBuildError("Git revision is not a full SHA-1")
|
||||
if require_clean:
|
||||
status = subprocess.run(
|
||||
["git", "status", "--porcelain"],
|
||||
cwd=REPOSITORY_ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if status.stdout.strip():
|
||||
raise ArtifactBuildError("production artifact requires a clean worktree")
|
||||
return revision
|
||||
|
||||
|
||||
def build_wheel(output: Path) -> Path:
|
||||
environment = os.environ.copy()
|
||||
environment["SOURCE_DATE_EPOCH"] = "0"
|
||||
result = subprocess.run(
|
||||
["uv", "build", "--wheel", "--out-dir", str(output)],
|
||||
cwd=REPOSITORY_ROOT,
|
||||
env=environment,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
detail = (result.stderr or result.stdout).strip()
|
||||
raise ArtifactBuildError(f"wheel build failed: {detail}")
|
||||
wheel = output / WHEEL_NAME
|
||||
if not wheel.is_file() or wheel.is_symlink():
|
||||
raise ArtifactBuildError("expected wheel was not built")
|
||||
return wheel
|
||||
|
||||
|
||||
def render_descriptor(
|
||||
patch_id: str,
|
||||
revision: str,
|
||||
*,
|
||||
wheel_sha256: str,
|
||||
) -> bytes:
|
||||
if sha256_file(BASE_TEMPLATE) != EXPECTED_BASE_TEMPLATE_SHA256:
|
||||
raise ArtifactBuildError("historical M4 descriptor base changed")
|
||||
descriptor = json.loads(BASE_TEMPLATE.read_text("utf-8"))
|
||||
descriptor.update(
|
||||
{
|
||||
"schema_version": "nodedc.mission-core-worker.shadow-release/v2",
|
||||
"patch_id": patch_id,
|
||||
"code_revision": revision,
|
||||
"transition": "m47-canonical-graph-shadow-v1",
|
||||
"artifact_type": "shadow-release",
|
||||
}
|
||||
)
|
||||
descriptor["container"].update(
|
||||
{
|
||||
"name": "ndc-mission-core-m47-graph-shadow",
|
||||
"python_path": (
|
||||
f"/release/{WHEEL_NAME}:/opt/media:/opt/opencv:/opt/pillow"
|
||||
),
|
||||
}
|
||||
)
|
||||
descriptor["inputs"]["local_surface"] = {
|
||||
"container_path": "/source/local-surface/local-surface.npz",
|
||||
"host_path": (
|
||||
"D:\\NDC_MISSIONCORE\\runtime\\derived\\"
|
||||
"k1-local-surface-23762244c8bdb97de26fb721ac957d7a00bc9a63571ac4cfa4be19c4effc7d55"
|
||||
"\\local-surface.npz"
|
||||
),
|
||||
"sha256": "f57eb2485b6cef47f2a97a2d9ff1aa9fd9265fe1eb69cd5852d12f39e13b8bc6",
|
||||
}
|
||||
descriptor["inputs"]["accepted_temporal_frames"] = {
|
||||
"container_path": "/parity/temporal/frames.jsonl",
|
||||
"host_path": (
|
||||
"D:\\NDC_MISSIONCORE\\runtime\\derived\\"
|
||||
"m4-temporal-replay-81e13d5654ac8d1219f6937dd425f7dbcdbacd5748d23fe653297134f0de6c22"
|
||||
"\\frames.jsonl"
|
||||
),
|
||||
"sha256": "e83b80ea06b3462c2d04c5a1b74289a0ec401596c7150ae90f5a1b748b639c3a",
|
||||
}
|
||||
descriptor["inputs"]["accepted_threat_frames"] = {
|
||||
"container_path": "/parity/threat/frames.jsonl",
|
||||
"host_path": (
|
||||
"D:\\NDC_MISSIONCORE\\runtime\\derived\\"
|
||||
"m4-threat-replay-2a953c5f27f2a5b1dddc5c658c1de2c323d7796084a099c024987a1da03aa324"
|
||||
"\\frames.jsonl"
|
||||
),
|
||||
"sha256": "b57be1839f5915e3b80b54355b694e0bd8c9ac318d0cbe6de2bff713082cfa4e",
|
||||
}
|
||||
descriptor["release"] = {
|
||||
"wheel": {"name": WHEEL_NAME, "sha256": wheel_sha256},
|
||||
"runner": {"name": RUNNER.name, "sha256": sha256_file(RUNNER)},
|
||||
"configs": {
|
||||
path.name: {
|
||||
"name": path.name,
|
||||
"sha256": sha256_file(REPOSITORY_ROOT / path),
|
||||
}
|
||||
for path in CONFIG_PATHS
|
||||
},
|
||||
}
|
||||
descriptor["acceptance"] = {
|
||||
"run_mode": "lossless-replay",
|
||||
"expected_frames": 4489,
|
||||
"delivered_frames": 4489,
|
||||
"failed_frames": 0,
|
||||
"stale_frames": 0,
|
||||
"superseded_frames": 0,
|
||||
"accepted_parity": True,
|
||||
"class_routing_used": False,
|
||||
}
|
||||
descriptor["readiness"] = {
|
||||
"provider": [
|
||||
"triton-yolox-s-raw-kb4/v1",
|
||||
"ravnoves00-geometry-association/v1",
|
||||
"bounded-spatial-temporal-layer/v1",
|
||||
"class-independent-motion-estimator/v1",
|
||||
"rolling-local-obstacle-map/v1",
|
||||
"dual-evidence-replay-threat/v3",
|
||||
],
|
||||
"graph": {
|
||||
"graph_id": "reference-perception-graph/v2",
|
||||
"terminal_accounting_required": True,
|
||||
"actuation_allowed": False,
|
||||
},
|
||||
}
|
||||
if (
|
||||
descriptor["predecessor"]["durable_worker"]["name"]
|
||||
!= "ndc-mission-core-perception-worker"
|
||||
or descriptor["rollback"]["durable_worker_action"] != "none"
|
||||
or descriptor["boundary"]["external_deploy_registry"] is not False
|
||||
):
|
||||
raise ArtifactBuildError("M4.7 predecessor or deployment boundary changed")
|
||||
return (
|
||||
json.dumps(descriptor, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _tar_info(path: Path, arcname: str) -> tarfile.TarInfo:
|
||||
info = tarfile.TarInfo(arcname)
|
||||
info.uid = 0
|
||||
info.gid = 0
|
||||
info.uname = "root"
|
||||
info.gname = "root"
|
||||
info.mtime = 0
|
||||
if path.is_dir():
|
||||
info.type = tarfile.DIRTYPE
|
||||
info.mode = 0o755
|
||||
else:
|
||||
info.type = tarfile.REGTYPE
|
||||
info.mode = 0o644
|
||||
info.size = path.stat().st_size
|
||||
return info
|
||||
|
||||
|
||||
def write_canonical_archive(stage: Path, target: Path) -> None:
|
||||
members = [stage / "manifest.env", stage / "files.txt", stage / "payload"]
|
||||
members.extend(sorted((stage / "payload").rglob("*")))
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with (
|
||||
target.open("wb") as raw,
|
||||
gzip.GzipFile(filename="", mode="wb", fileobj=raw, compresslevel=9, mtime=0) as gz,
|
||||
tarfile.open(fileobj=gz, mode="w", format=tarfile.PAX_FORMAT) as archive,
|
||||
):
|
||||
for path in members:
|
||||
info = _tar_info(path, path.relative_to(stage).as_posix())
|
||||
if path.is_file():
|
||||
with path.open("rb") as source:
|
||||
archive.addfile(info, source)
|
||||
else:
|
||||
archive.addfile(info, io.BytesIO())
|
||||
|
||||
|
||||
def build_artifact(
|
||||
patch_id: str,
|
||||
output_directory: Path,
|
||||
*,
|
||||
revision: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
if PATCH_ID.fullmatch(patch_id) is None:
|
||||
raise ArtifactBuildError("patch id is invalid")
|
||||
selected_revision = revision or git_revision(require_clean=True)
|
||||
if re.fullmatch(r"[a-f0-9]{40}", selected_revision) is None:
|
||||
raise ArtifactBuildError("artifact revision is invalid")
|
||||
with tempfile.TemporaryDirectory(prefix="mission-core-m47-graph-shadow-") as directory:
|
||||
stage = Path(directory)
|
||||
payload = stage / "payload"
|
||||
payload.mkdir()
|
||||
wheel = build_wheel(stage / "wheel")
|
||||
wheel_sha256 = sha256_file(wheel)
|
||||
payload_files = [RUNNER.name, WHEEL_NAME, DESCRIPTOR_NAME]
|
||||
(payload / RUNNER.name).write_bytes(RUNNER.read_bytes())
|
||||
(payload / WHEEL_NAME).write_bytes(wheel.read_bytes())
|
||||
for relative in CONFIG_PATHS:
|
||||
source = REPOSITORY_ROOT / relative
|
||||
(payload / source.name).write_bytes(source.read_bytes())
|
||||
payload_files.append(source.name)
|
||||
(payload / DESCRIPTOR_NAME).write_bytes(
|
||||
render_descriptor(
|
||||
patch_id,
|
||||
selected_revision,
|
||||
wheel_sha256=wheel_sha256,
|
||||
)
|
||||
)
|
||||
payload_files = sorted(payload_files)
|
||||
(stage / "manifest.env").write_text(
|
||||
f"id={patch_id}\ncomponent=mission-core-worker\ntype=shadow-release\n",
|
||||
"utf-8",
|
||||
)
|
||||
(stage / "files.txt").write_text("\n".join(payload_files) + "\n", "utf-8")
|
||||
target = (
|
||||
output_directory.resolve()
|
||||
/ f"nodedc-mission-core-worker-{patch_id}.tgz"
|
||||
)
|
||||
write_canonical_archive(stage, target)
|
||||
return {
|
||||
"ok": True,
|
||||
"patch_id": patch_id,
|
||||
"component": "mission-core-worker",
|
||||
"type": "shadow-release",
|
||||
"artifact": str(target),
|
||||
"sha256": sha256_file(target),
|
||||
"code_revision": selected_revision,
|
||||
"wheel_sha256": wheel_sha256,
|
||||
"payload_files": payload_files,
|
||||
"transition": "m47-canonical-graph-shadow-v1",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("patch_id")
|
||||
parser.add_argument(
|
||||
"--output-directory",
|
||||
type=Path,
|
||||
default=REPOSITORY_ROOT / ".runtime/worker-artifacts",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = build_artifact(args.patch_id, args.output_directory)
|
||||
except (ArtifactBuildError, OSError, subprocess.SubprocessError, json.JSONDecodeError) as exc:
|
||||
parser.error(str(exc))
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user