feat(perception): stabilize pre-capture methodology
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,161 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$PackageRoot,
|
||||
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\derived\e40-product-gate",
|
||||
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
|
||||
[ValidateRange(1, 1000)]
|
||||
[int]$FreeGiBFloor = 300
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode([string]$Operation) {
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Resolve-DDirectory([string]$Path, [string]$Label) {
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
||||
if (
|
||||
-not $item.PSIsContainer -or
|
||||
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
||||
$root -ine "D:"
|
||||
) {
|
||||
throw "$Label must be a real D: directory"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath([string]$Path) {
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
function Assert-FreeSpace([string]$Phase) {
|
||||
$free = [int64](Get-PSDrive -Name D).Free
|
||||
$floor = [int64]$FreeGiBFloor * 1GB
|
||||
Write-Host (
|
||||
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3}" -f
|
||||
$Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor
|
||||
)
|
||||
if ($free -lt ($floor + 1GB)) {
|
||||
throw "D: lacks the guarded E40 reserve during $Phase"
|
||||
}
|
||||
return $free
|
||||
}
|
||||
|
||||
$package = Resolve-DDirectory $PackageRoot "E40 package"
|
||||
$packageManifestPath = Join-Path $package "manifest.json"
|
||||
if (-not (Test-Path -LiteralPath $packageManifestPath -PathType Leaf)) {
|
||||
throw "E40 package manifest is missing"
|
||||
}
|
||||
$packageManifest = Get-Content -LiteralPath $packageManifestPath -Raw |
|
||||
ConvertFrom-Json
|
||||
if (
|
||||
$packageManifest.schema_version -ne "missioncore.e40-worker-package/v1" -or
|
||||
$packageManifest.package_id -ne (Split-Path $package -Leaf) -or
|
||||
$packageManifest.package_id -notmatch "^e40-worker-package-[a-f0-9]{64}$"
|
||||
) {
|
||||
throw "E40 package manifest is incompatible"
|
||||
}
|
||||
|
||||
if (-not (Test-Path -LiteralPath $OutputRoot)) {
|
||||
$null = New-Item -ItemType Directory -Path $OutputRoot
|
||||
}
|
||||
$output = Resolve-DDirectory $OutputRoot "E40 output root"
|
||||
$freeBefore = Assert-FreeSpace "preflight"
|
||||
|
||||
& docker image inspect $ContainerImage *> $null
|
||||
Assert-LastExitCode "Pinned E40 container image inspection"
|
||||
|
||||
$dockerPackage = Convert-ToDockerPath $package
|
||||
$dockerOutput = Convert-ToDockerPath $output
|
||||
$packageName = Split-Path $package -Leaf
|
||||
$containerPackage = "/opt/e40-input/$packageName"
|
||||
$packageValidator = (
|
||||
"{0}/runtime/validate_e40_worker_package.py" -f $containerPackage
|
||||
)
|
||||
$validationCommand = @(
|
||||
"run", "--rm",
|
||||
"--name", "ndc-mission-core-e40-package-validation",
|
||||
"--network", "none",
|
||||
"--read-only",
|
||||
"--security-opt", "no-new-privileges:true",
|
||||
"--cap-drop", "ALL",
|
||||
"--pids-limit", "32",
|
||||
"--memory", "128m",
|
||||
"--memory-swap", "128m",
|
||||
"--cpus", "1",
|
||||
"-v", ("{0}:{1}:ro" -f $dockerPackage, $containerPackage),
|
||||
"--entrypoint", "python3",
|
||||
$ContainerImage,
|
||||
$packageValidator,
|
||||
$containerPackage
|
||||
)
|
||||
& docker @validationCommand
|
||||
Assert-LastExitCode "Independent E40 package integrity verification"
|
||||
|
||||
$command = @(
|
||||
"run", "--rm",
|
||||
"--name", "ndc-mission-core-e40-product-gate",
|
||||
"--network", "none",
|
||||
"--read-only",
|
||||
"--security-opt", "no-new-privileges:true",
|
||||
"--cap-drop", "ALL",
|
||||
"--pids-limit", "128",
|
||||
"--memory", "1g",
|
||||
"--memory-swap", "1g",
|
||||
"--cpus", "2",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=64m",
|
||||
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
||||
"-e", ("PYTHONPATH={0}/runtime" -f $containerPackage),
|
||||
"-e", ("E40_WORKER_NODE={0}" -f $env:COMPUTERNAME),
|
||||
"-v", ("{0}:{1}:ro" -f $dockerPackage, $containerPackage),
|
||||
"-v", ("{0}:/output:rw" -f $dockerOutput),
|
||||
"--entrypoint", "python3",
|
||||
$ContainerImage,
|
||||
("{0}/runtime/run_e40_perception_product_gate.py" -f $containerPackage),
|
||||
"--package", $containerPackage,
|
||||
"--output-root", "/output"
|
||||
)
|
||||
|
||||
Write-Output ("PACKAGE_ID={0}" -f $packageManifest.package_id)
|
||||
Write-Output ("PACKAGE_IDENTITY_SHA256={0}" -f $packageManifest.identity_sha256)
|
||||
Write-Output ("CONTAINER_IMAGE={0}" -f $ContainerImage)
|
||||
& docker @command
|
||||
Assert-LastExitCode "E40 perception product gate"
|
||||
|
||||
$matches = @(
|
||||
Get-ChildItem -LiteralPath $output -Directory -Filter "e40-perception-product-gate-*" |
|
||||
Where-Object {
|
||||
$manifestPath = Join-Path $_.FullName "manifest.json"
|
||||
if (-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) {
|
||||
return $false
|
||||
}
|
||||
$manifest = Get-Content -LiteralPath $manifestPath -Raw |
|
||||
ConvertFrom-Json
|
||||
return (
|
||||
$manifest.schema_version -eq
|
||||
"missioncore.e40-perception-product-gate/v1" -and
|
||||
$manifest.acceptance_state -eq
|
||||
"completed-leakage-resistant-product-gate" -and
|
||||
$manifest.identity.execution.worker_node -eq $env:COMPUTERNAME
|
||||
)
|
||||
}
|
||||
)
|
||||
if ($matches.Count -ne 1) {
|
||||
throw "E40 immutable result could not be resolved uniquely"
|
||||
}
|
||||
$resultRoot = $matches[0].FullName
|
||||
$resultManifest = Get-Content -LiteralPath (
|
||||
Join-Path $resultRoot "manifest.json"
|
||||
) -Raw | ConvertFrom-Json
|
||||
$freeAfter = Assert-FreeSpace "completed"
|
||||
Write-Output ("RESULT_ROOT={0}" -f $resultRoot)
|
||||
Write-Output ("RESULT_ID={0}" -f $resultManifest.result_id)
|
||||
Write-Output ("QUALITY_GATE_PASSED={0}" -f $resultManifest.quality_gate_passed)
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f $freeAfter)
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Execute the packaged E40 product gate in the pinned Worker 006 container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e40_perception_product_gate import (
|
||||
build_e40_perception_product_gate,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--package", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
package = args.package.resolve(strict=True)
|
||||
package_manifest = json.loads(
|
||||
(package / "manifest.json").read_text(encoding="utf-8-sig")
|
||||
)
|
||||
profile = json.loads((package / "profile.json").read_text(encoding="utf-8"))
|
||||
acceptance_id = profile["source"]["acceptance_result_id"]
|
||||
materialization_id = profile["source"]["materialization_id"]
|
||||
result = build_e40_perception_product_gate(
|
||||
acceptance_root=package / "input" / "acceptance" / acceptance_id,
|
||||
materialization_root=(package / "input" / "materialization" / materialization_id),
|
||||
profile_path=package / "profile.json",
|
||||
output_root=args.output_root,
|
||||
worker_node=os.environ.get("E40_WORKER_NODE"),
|
||||
execution_package={
|
||||
"mode": "verified-worker-package",
|
||||
"package_id": package_manifest["package_id"],
|
||||
"identity_sha256": package_manifest["identity_sha256"],
|
||||
},
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result.result_id,
|
||||
"result_root": str(result.result_root),
|
||||
"quality_gate_passed": result.quality_gate_passed,
|
||||
"metrics": result.report["metrics"],
|
||||
"blocking_checks": result.report["quality_gate"]["blocking_checks"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independently validate an E40 package before importing package code."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
|
||||
def _descriptors(
|
||||
rows: object,
|
||||
*,
|
||||
require_kind: bool,
|
||||
) -> dict[str, tuple[int, str]]:
|
||||
if not isinstance(rows, list):
|
||||
raise SystemExit("E40 package artifact catalog is missing")
|
||||
result: dict[str, tuple[int, str]] = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
raise SystemExit("E40 package artifact descriptor is invalid")
|
||||
relative = row.get("path")
|
||||
byte_length = row.get("byte_length")
|
||||
sha256 = row.get("sha256")
|
||||
if (
|
||||
not isinstance(relative, str)
|
||||
or relative in result
|
||||
or pathlib.PurePosixPath(relative).is_absolute()
|
||||
or ".." in pathlib.PurePosixPath(relative).parts
|
||||
or not isinstance(byte_length, int)
|
||||
or byte_length < 0
|
||||
or not isinstance(sha256, str)
|
||||
or len(sha256) != 64
|
||||
or (require_kind and row.get("kind") != relative)
|
||||
):
|
||||
raise SystemExit("E40 package artifact descriptor is invalid")
|
||||
result[relative] = (byte_length, sha256)
|
||||
return result
|
||||
|
||||
|
||||
def validate(root_argument: str) -> None:
|
||||
root = pathlib.Path(root_argument).resolve(strict=True)
|
||||
manifest = json.loads(
|
||||
(root / "manifest.json").read_text(encoding="utf-8-sig")
|
||||
)
|
||||
identity = manifest.get("identity")
|
||||
identity_sha256 = manifest.get("identity_sha256")
|
||||
package_id = manifest.get("package_id")
|
||||
if (
|
||||
manifest.get("schema_version") != "missioncore.e40-worker-package/v1"
|
||||
or not isinstance(identity, dict)
|
||||
or not isinstance(identity_sha256, str)
|
||||
or hashlib.sha256(
|
||||
json.dumps(
|
||||
identity,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode()
|
||||
).hexdigest()
|
||||
!= identity_sha256
|
||||
or package_id != f"e40-worker-package-{identity_sha256}"
|
||||
or root.name != package_id
|
||||
):
|
||||
raise SystemExit("E40 package identity verification failed")
|
||||
|
||||
bound = _descriptors(
|
||||
identity.get("source_artifacts"),
|
||||
require_kind=False,
|
||||
)
|
||||
catalog = _descriptors(manifest.get("artifacts"), require_kind=True)
|
||||
if bound != catalog or set(identity.get("artifact_paths", [])) != set(bound):
|
||||
raise SystemExit("E40 package artifact binding verification failed")
|
||||
actual = {
|
||||
path.relative_to(root).as_posix()
|
||||
for path in root.rglob("*")
|
||||
if path.is_file()
|
||||
}
|
||||
if actual != set(bound) | {"manifest.json"}:
|
||||
raise SystemExit("E40 package file set verification failed")
|
||||
for relative, (byte_length, sha256) in bound.items():
|
||||
path = root / relative
|
||||
payload = path.read_bytes()
|
||||
if (
|
||||
path.is_symlink()
|
||||
or len(payload) != byte_length
|
||||
or hashlib.sha256(payload).hexdigest() != sha256
|
||||
):
|
||||
raise SystemExit(
|
||||
f"E40 package member verification failed: {relative}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 2:
|
||||
raise SystemExit("usage: validate_e40_worker_package.py PACKAGE_ROOT")
|
||||
validate(sys.argv[1])
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user