feat(perception): add integrated TGS graph shadow gate

This commit is contained in:
DCCONSTRUCTIONS
2026-08-27 11:27:28 +03:00
parent 296cf610cd
commit 7720594790
10 changed files with 1489 additions and 3 deletions
@@ -0,0 +1,73 @@
{
"schema_version": "missioncore.m49-tgs-integrated-graph-shadow-profile/v1",
"profile_id": "m49-ravnoves00-tgs-native-risk-integrated-shadow/v1",
"source": {
"source_id": "RAVNOVES00",
"expected_timeline_frames": 4489,
"expected_available_lidar_frames": 3928,
"source_pack_sha256": "0685d24219d8236caf8b7f1685e93f6d6b59e7fd015a768d88a92bbe8b154944",
"requested_source_rate_hz": 12.0,
"shared_start_barrier": true
},
"stages": {
"reference_graph": {
"graph_config": "m48n-rf-detr-native-reference-graph-shadow-v0.json",
"graph_config_sha256": "1db6f6fa0561d819505a5e4f62fe256dab6fe4d9da6b4f1a9a5073bfcf772c90",
"detector_profile": "rf-detr-large-native-kb4-risk-shadow-v0.json",
"detector_profile_sha256": "dbf4da5dbad6c3c22b1280b46ffcad81719bd183c81c263a4859847d829019b6",
"detector_provider_id": "triton-rf-detr-large-coco-native-kb4-risk-fp16-shadow/v0",
"single_inference_pass": true
},
"tgs": {
"profile": "m49-tgs-full-shadow-v1.json",
"profile_sha256": "c2e07010aaee78259d36c057962d6bfb885349251ff7356d867e5813e632881c",
"candidate_id": "travel-tgs-only-gravity-aligned",
"linked_accepted_result_id": "m49-tgs-full-shadow-ef98de7db7596d48e8c8c0549ce68e6704ee03e87c8c4bcf1e3e748b7ccb032e",
"aos_allowed": false,
"gpu_allowed": false,
"states_remain_separate": true
}
},
"acceptance": {
"minimum_delivery_ratio": 1.0,
"reference_world_state_fps": 11.79902,
"maximum_world_state_fps_regression_fraction": 0.05,
"minimum_effective_world_state_fps": 11.209069,
"maximum_world_state_completion_p95_ms": 125.0,
"candidate_stage_p95_ms_max": 25.0,
"candidate_stage_p99_ms_max": 50.0,
"combined_output_age_p99_ms_max": 125.0,
"capacity_drop_count_max": 0,
"unaccounted_frame_count_max": 0
},
"telemetry": {
"sample_interval_seconds": 1.0,
"required_products": [
"graph_pipeline_timing",
"graph_queue_high_watermarks",
"tgs_stage_timing",
"host_container_cpu_memory",
"gpu_utilization_memory_power_temperature"
]
},
"invariants": {
"native_fisheye_raster_unchanged": true,
"camera_rectification_allowed": false,
"camera_resize_allowed": false,
"tgs_candidate_parameters_unchanged": true,
"tgs_modifies_reference_graph_state": false,
"missing_lidar_means_unobserved": true,
"future_frames_used": false,
"low_step_used": false,
"gauss_or_playcanvas_in_scope": false
},
"authority": {
"visual_quality_accepted": false,
"traversability_accepted": false,
"physical_free_space_accepted": false,
"commands_enabled": false,
"actuation_allowed": false,
"navigation_or_safety_accepted": false,
"production_accepted": false
}
}
@@ -67,6 +67,27 @@ AUTHORITY: Final = {
LOAD_PURPOSES: Final = ("production-rate", "reserve-gate", "limit-discovery")
def wait_for_shared_start(
*,
ready_file: Path,
start_file: Path,
timeout_seconds: float,
) -> None:
"""Join an external source-admission barrier after local warmup is complete."""
if timeout_seconds <= 0:
raise RuntimeError("shared-start timeout must be positive")
if ready_file.exists():
raise RuntimeError("shared-start ready file already exists")
ready_file.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
ready_file.write_text("ready\n", encoding="utf-8")
deadline = time.monotonic() + timeout_seconds
while not start_file.is_file():
if time.monotonic() >= deadline:
raise RuntimeError("shared-start barrier timed out")
time.sleep(0.01)
class GpuTelemetry:
def __init__(self, interval_seconds: float) -> None:
self.interval_seconds = interval_seconds
@@ -394,6 +415,9 @@ def main() -> int:
parser.add_argument("--runtime-artifact-sha256", required=True)
parser.add_argument("--runner-sha256", required=True)
parser.add_argument("--telemetry-interval-seconds", type=float, default=1.0)
parser.add_argument("--shared-start-ready-file", type=Path)
parser.add_argument("--shared-start-file", type=Path)
parser.add_argument("--shared-start-timeout-seconds", type=float, default=600.0)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--progress", type=Path, required=True)
parser.add_argument("--frame-ledger", type=Path, required=True)
@@ -404,6 +428,17 @@ def main() -> int:
raise RuntimeError("maximum frame count must be positive")
if arguments.telemetry_interval_seconds <= 0:
raise RuntimeError("telemetry interval must be positive")
shared_start_requested = (
arguments.shared_start_ready_file is not None or arguments.shared_start_file is not None
)
if shared_start_requested and (
arguments.shared_start_ready_file is None or arguments.shared_start_file is None
):
raise RuntimeError("shared-start ready and start files must be configured together")
if shared_start_requested and arguments.loops != 1:
raise RuntimeError("shared-start barrier requires exactly one loop")
if arguments.shared_start_timeout_seconds <= 0:
raise RuntimeError("shared-start timeout must be positive")
if arguments.source_rate_hz is not None and (
not np.isfinite(arguments.source_rate_hz) or arguments.source_rate_hz <= 0
):
@@ -506,6 +541,12 @@ def main() -> int:
)
detector_warmup = runtime.warm_up_detector()
source_prefetch = runtime.prepare_source()
if shared_start_requested:
wait_for_shared_start(
ready_file=arguments.shared_start_ready_file,
start_file=arguments.shared_start_file,
timeout_seconds=arguments.shared_start_timeout_seconds,
)
gc_policy = CyclicGcHotLoopPolicy()
with gc_policy:
runtime.mark_source_admission_started()
@@ -0,0 +1,450 @@
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[string]$ReleaseRoot,
[Parameter(Mandatory = $true)]
[string]$CandidateRoot,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[a-f0-9]{64}$")]
[string]$ExpectedArtifactSha256,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[A-Za-z0-9._-]{1,96}$")]
[string]$RunId,
[ValidateRange(1.0, 120.0)]
[double]$SourceRateHz = 12.0,
[string]$OutputRoot = (
"D:\NDC_MISSIONCORE\runtime\results\m49-tgs-integrated-graph-shadow"
)
)
$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue"
$TravelImageTag = "ndc/mission-core-m49-t3-travel:20260826"
$TravelImageId = "sha256:7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f"
$ParityImageTag = "ndc-mission-core-m48t-upstream-parity:1.9.4-cu130"
$ParityImageId = "sha256:ceb13548617e4bd3f619766bfdff00af3fa5160946b367828da6d2233dcdcba0"
$RuntimeImage = (
"nvcr.io/nvidia/tritonserver:26.06-py3@" +
"sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
)
function Assert-LastExitCode([string]$Operation) {
if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" }
}
function Get-Sha256([string]$Path) {
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
}
function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) {
if ($Create -and -not (Test-Path -LiteralPath $Path)) {
$null = New-Item -ItemType Directory -Path $Path
}
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
if (
-not $item.PSIsContainer -or
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
) { throw "$Label must be a real D: directory" }
return $item.FullName
}
function Resolve-DFile([string]$Path, [string]$Label) {
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
if (
$item.PSIsContainer -or
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
) { throw "$Label must be a real D: file" }
return $item.FullName
}
function Convert-ToDockerPath([string]$Path) { return ($Path -replace "\\", "/") }
function Get-Container([string]$Name) {
$rows = @(((& docker inspect $Name) | ConvertFrom-Json))
Assert-LastExitCode "Docker inspection for $Name"
if ($rows.Count -ne 1) { throw "Container identity for $Name is not unique" }
return $rows[0]
}
function Assert-Image([string]$Tag, [string]$ExpectedId) {
$rows = @(((& docker image inspect $Tag) | ConvertFrom-Json))
Assert-LastExitCode "Docker image inspection for $Tag"
if ($rows.Count -ne 1 -or [string]$rows[0].Id -cne $ExpectedId) {
throw "Pinned image identity changed for $Tag"
}
}
function Remove-ExactContainer([string]$Name) {
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$Name$") {
& docker rm --force $Name *> $null
}
}
function Wait-Healthy([string]$Name) {
foreach ($attempt in 1..60) {
Start-Sleep -Seconds 2
$container = Get-Container $Name
if (-not $container.State.Running) {
& docker logs $Name
throw "$Name stopped during startup"
}
if ($container.State.Health.Status -ceq "healthy") { return }
}
throw "$Name did not become healthy"
}
function Wait-SharedReady(
[string]$GraphReady,
[string]$TgsReady,
[string]$GraphName,
[string]$TgsName
) {
$deadline = [DateTimeOffset]::UtcNow.AddMinutes(10)
while (-not ((Test-Path -LiteralPath $GraphReady) -and (Test-Path -LiteralPath $TgsReady))) {
foreach ($name in @($GraphName, $TgsName)) {
$container = Get-Container $name
if (-not $container.State.Running) {
& docker logs $name
throw "$name stopped before shared-start readiness"
}
}
if ([DateTimeOffset]::UtcNow -ge $deadline) {
throw "M49 integrated shared-start readiness timed out"
}
Start-Sleep -Milliseconds 250
}
}
if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
throw "M49 integrated TGS graph shadow is pinned to Worker 006"
}
$release = Resolve-DDirectory $ReleaseRoot "M49 integrated release root" $false
$payload = Resolve-DDirectory (Join-Path $release "payload") "M49 integrated payload" $false
$candidate = Resolve-DDirectory $CandidateRoot "M49 native candidate root" $false
$output = Resolve-DDirectory $OutputRoot "M49 integrated output root" $true
$runCandidate = Join-Path $output $RunId
if (Test-Path -LiteralPath $runCandidate) { throw "M49 integrated output already exists" }
$null = New-Item -ItemType Directory -Path $runCandidate
$runOutput = Resolve-DDirectory $runCandidate "M49 integrated run output" $false
foreach ($directory in @("bin", "control", "graph", "tgs")) {
$null = New-Item -ItemType Directory -Path (Join-Path $runOutput $directory)
}
$releaseDocument = Get-Content -LiteralPath (Join-Path $payload "release.json") -Raw | ConvertFrom-Json
if (
$releaseDocument.schema_version -cne "missioncore.m49-tgs-integrated-graph-worker-release/v1" -or
$releaseDocument.worker_id -cne "worker-006" -or
$releaseDocument.transition -cne "m49-tgs-native-risk-integrated-shadow/v1"
) { throw "M49 integrated release contract changed" }
foreach ($property in $releaseDocument.files.PSObject.Properties) {
$path = Join-Path $payload $property.Name
if ((Get-Sha256 $path) -cne [string]$property.Value.sha256) {
throw "M49 integrated payload digest changed: $($property.Name)"
}
}
$wheelSha256 = [string]$releaseDocument.files."nodedc_mission_core-0.1.0-py3-none-any.whl".sha256
$runnerSha256 = [string]$releaseDocument.files."run_m48s_reference_graph_shadow_worker.py".sha256
$source = [ordered]@{
CameraIndex = (
"D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d" +
"\input\camera\sensor.camera.right\epoch-1\index.jsonl"
)
SourcePack = (
"D:\NDC_MISSIONCORE\runtime\derived" +
"\e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b" +
"\lidar-pack.npz"
)
LocalSurface = (
"D:\NDC_MISSIONCORE\runtime\derived" +
"\k1-local-surface-23762244c8bdb97de26fb721ac957d7a00bc9a63571ac4cfa4be19c4effc7d55" +
"\local-surface.npz"
)
Video = (
"D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs" +
"\right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8.mp4"
)
Mask = (
"D:\NDC_MISSIONCORE\runtime\inputs\e2" +
"\valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2" +
"\mask.png"
)
}
foreach ($entry in $source.GetEnumerator()) {
$null = Resolve-DFile $entry.Value "M49 source $($entry.Key)"
}
if ((Get-Sha256 $source.SourcePack) -cne [string]$releaseDocument.source_pack_sha256) {
throw "RAVNOVES00 source pack digest changed"
}
$nativeConfig = Resolve-DFile (
(Join-Path $payload "rf_detr_large_native_kb4_config.pbtxt")
) "native RF-DETR config"
$nativeEngine = Resolve-DFile (
(Join-Path $candidate "rf-detr-native-uint8.plan")
) "native RF-DETR engine"
if ((Get-Sha256 $nativeEngine) -cne "b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695") {
throw "native RF-DETR engine SHA-256 changed"
}
$modelRoot = Join-Path $runOutput "triton-models"
$modelDirectory = Join-Path $modelRoot "rf_detr_large_native_kb4"
$modelVersionDirectory = Join-Path $modelDirectory "1"
$null = New-Item -ItemType Directory -Path $modelVersionDirectory
Copy-Item -LiteralPath $nativeConfig -Destination (Join-Path $modelDirectory "config.pbtxt")
Copy-Item -LiteralPath $nativeEngine -Destination (Join-Path $modelVersionDirectory "model.plan")
$media = Resolve-DDirectory (
"D:\NDC_MISSIONCORE\runtime\derived\perception-e15-media-pyav180-lz445-v1"
) "PyAV dependency" $false
$opencv = Resolve-DDirectory (
"D:\NDC_MISSIONCORE\runtime\derived\perception-e3-opencv413092-v1\packages"
) "OpenCV dependency" $false
$pillow = Resolve-DDirectory (
"D:\NDC_MISSIONCORE\runtime\derived\perception-p0-env-v1"
) "Pillow dependency" $false
Assert-Image $TravelImageTag $TravelImageId
Assert-Image $ParityImageTag $ParityImageId
& docker image inspect $RuntimeImage *> $null
Assert-LastExitCode "pinned runtime image inspection"
$os = Get-CimInstance Win32_OperatingSystem
$freeMemoryGiB = [double]$os.FreePhysicalMemory / 1MB
if ($freeMemoryGiB -lt 24.0) {
throw ("M49 integrated shadow requires 24 GiB free memory; observed {0:N2} GiB" -f $freeMemoryGiB)
}
$canonicalBefore = Get-Container "ndc-mission-core-triton"
if (-not $canonicalBefore.State.Running -or $canonicalBefore.State.Health.Status -cne "healthy") {
throw "Canonical Mission Core Triton must remain healthy"
}
$canonicalId = [string]$canonicalBefore.Id
$prepareName = "ndc-mission-core-m49-integrated-prepare-$RunId"
$compileName = "ndc-mission-core-m49-integrated-compile-$RunId"
$tritonName = "ndc-mission-core-m49-integrated-triton-$RunId"
$graphName = "ndc-mission-core-m49-integrated-graph-$RunId"
$tgsName = "ndc-mission-core-m49-integrated-tgs-$RunId"
$analyzeName = "ndc-mission-core-m49-integrated-analyze-$RunId"
$evidenceName = "ndc-mission-core-m49-integrated-evidence-$RunId"
$containers = @($prepareName, $compileName, $tritonName, $graphName, $tgsName, $analyzeName, $evidenceName)
foreach ($name in $containers) {
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
throw "M49 integrated container name already exists: $name"
}
}
$started = [DateTimeOffset]::UtcNow
try {
& docker run --rm --name $prepareName --network none --cpus 8 --memory 16g `
--entrypoint python3 `
--volume ((Convert-ToDockerPath $source.SourcePack) + ":/source/lidar-pack.npz:ro") `
--volume ((Convert-ToDockerPath $payload) + ":/release:ro") `
--volume ((Convert-ToDockerPath (Join-Path $runOutput "tgs")) + ":/tgs") `
$ParityImageTag /release/prepare_tgs_full_shadow_inputs.py `
--source-pack /source/lidar-pack.npz `
--config /release/m49-tgs-full-shadow-v1.json `
--output-root /tgs/inputs
Assert-LastExitCode "M49 integrated TGS input preparation"
& docker run --rm --name $compileName --network none --cpus 8 --memory 8g `
--entrypoint /bin/bash `
--volume ((Convert-ToDockerPath $payload) + ":/release:ro") `
--volume ((Convert-ToDockerPath (Join-Path $runOutput "bin")) + ":/out") `
$TravelImageTag /release/build_tgs_full_shadow_binary.sh /out/run_tgs_full_shadow
Assert-LastExitCode "M49 integrated TGS binary build"
& docker create --name $tritonName `
--read-only --security-opt "no-new-privileges:true" --cap-drop ALL `
--pids-limit 512 --shm-size 1g --gpus all `
--tmpfs "/tmp:rw,noexec,nosuid,size=2g" `
--health-cmd "curl --fail --silent http://127.0.0.1:8000/v2/health/ready" `
--health-interval 5s --health-timeout 3s --health-start-period 20s --health-retries 24 `
-v ((Convert-ToDockerPath $modelRoot) + ":/models:ro") `
$RuntimeImage tritonserver --model-repository=/models `
--model-control-mode=explicit --load-model=rf_detr_large_native_kb4 `
--disable-auto-complete-config --strict-readiness=true --exit-on-error=true `
--allow-http=true --allow-grpc=false --allow-metrics=false *> $null
Assert-LastExitCode "M49 integrated Triton creation"
& docker start $tritonName *> $null
Assert-LastExitCode "M49 integrated Triton start"
Wait-Healthy $tritonName
$dockerRelease = Convert-ToDockerPath $payload
$dockerRun = Convert-ToDockerPath $runOutput
$rate = [string]::Format([Globalization.CultureInfo]::InvariantCulture, "{0:R}", $SourceRateHz)
$graphArguments = @(
"create", "--name", $graphName,
"--network", ("container:{0}" -f $tritonName),
"--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
"--pids-limit", "256", "--gpus", "all",
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
"-e", "PYTHONDONTWRITEBYTECODE=1",
"-e", "PYTHONPATH=/release/nodedc_mission_core-0.1.0-py3-none-any.whl:/opt/media:/opt/opencv:/opt/pillow",
"-v", ("{0}:/release:ro" -f $dockerRelease),
"-v", ("{0}:/shared:rw" -f $dockerRun),
"-v", ((Convert-ToDockerPath $media) + ":/opt/media:ro"),
"-v", ((Convert-ToDockerPath $opencv) + ":/opt/opencv:ro"),
"-v", ((Convert-ToDockerPath $pillow) + ":/opt/pillow:ro"),
"-v", ((Convert-ToDockerPath $source.CameraIndex) + ":/source/camera-index.jsonl:ro"),
"-v", ((Convert-ToDockerPath $source.SourcePack) + ":/source/source-pack.npz:ro"),
"-v", ((Convert-ToDockerPath $source.LocalSurface) + ":/source/local-surface.npz:ro"),
"-v", ((Convert-ToDockerPath $source.Video) + ":/source/right.mp4:ro"),
"-v", ((Convert-ToDockerPath $source.Mask) + ":/source/mask.png:ro"),
"--entrypoint", "python3", $RuntimeImage,
"/release/run_m48s_reference_graph_shadow_worker.py",
"--graph-config", "/release/m48n-rf-detr-native-reference-graph-shadow-v0.json",
"--baseline-profile", "/release/m4-recorded-realtime-baseline-v1.json",
"--detector-profile", "/release/rf-detr-large-native-kb4-risk-shadow-v0.json",
"--geometry-profile", "/release/m4-geometry-association-v1.json",
"--temporal-motion-profile", "/release/m4-temporal-motion-v1.json",
"--rolling-map-profile", "/release/m4-rolling-local-map-v1.json",
"--threat-profile", "/release/m4-replay-threat-v3.json",
"--camera-index", "/source/camera-index.jsonl",
"--source-pack", "/source/source-pack.npz",
"--local-surface", "/source/local-surface.npz",
"--video", "/source/right.mp4",
"--valid-fov-mask", "/source/mask.png",
"--triton-origin", "http://127.0.0.1:8000",
"--loops", "1", "--maximum-frames", "4489", "--source-rate-hz", $rate,
"--minimum-delivery-ratio", "1.0",
"--minimum-effective-world-state-fps", "11.209069",
"--maximum-world-state-completion-p95-ms", "125.0",
"--load-purpose", "reserve-gate",
"--runtime-artifact-sha256", $wheelSha256,
"--runner-sha256", $runnerSha256,
"--shared-start-ready-file", "/shared/control/graph.ready",
"--shared-start-file", "/shared/control/start.signal",
"--output", "/shared/graph/result.json",
"--progress", "/shared/graph/progress.jsonl",
"--frame-ledger", "/shared/graph/frames.jsonl"
)
& docker @graphArguments *> $null
Assert-LastExitCode "M49 integrated graph creation"
& docker create --name $tgsName --network none --cpus 16 --memory 24g `
--read-only --security-opt "no-new-privileges:true" --cap-drop ALL `
--pids-limit 256 --tmpfs "/tmp:rw,noexec,nosuid,size=1g" `
-e ("M49_SOURCE_RATE_HZ={0}" -f $rate) `
--entrypoint /bin/bash `
--volume ($dockerRelease + ":/release:ro") `
--volume ($dockerRun + ":/shared:rw") `
$TravelImageTag /release/run_tgs_integrated_shadow.sh *> $null
Assert-LastExitCode "M49 integrated TGS creation"
& docker start $graphName *> $null
Assert-LastExitCode "M49 integrated graph start"
& docker start $tgsName *> $null
Assert-LastExitCode "M49 integrated TGS start"
$graphReady = Join-Path $runOutput "control\graph.ready"
$tgsReady = Join-Path $runOutput "control\tgs.ready"
Wait-SharedReady $graphReady $tgsReady $graphName $tgsName
[DateTimeOffset]::UtcNow.ToString("o") | Set-Content -LiteralPath (
Join-Path $runOutput "control\start.signal"
) -Encoding utf8
$telemetryPath = Join-Path $runOutput "container-telemetry.jsonl"
while ($true) {
$graphState = Get-Container $graphName
$tgsState = Get-Container $tgsName
$running = @()
if ($graphState.State.Running) { $running += $graphName }
if ($tgsState.State.Running) { $running += $tgsName }
if ((Get-Container $tritonName).State.Running) { $running += $tritonName }
if ($running.Count -gt 0) {
$stats = @((& docker stats --no-stream --format "{{json .}}" @running))
Assert-LastExitCode "M49 integrated container telemetry"
foreach ($line in $stats) {
$value = $line | ConvertFrom-Json
$role = if ($value.Name -ceq $graphName) {
"graph"
} elseif ($value.Name -ceq $tgsName) {
"tgs"
} elseif ($value.Name -ceq $tritonName) {
"triton"
} else {
throw "Unknown M49 telemetry container"
}
[ordered]@{
observed_utc = [DateTimeOffset]::UtcNow.ToString("o")
role = $role
name = [string]$value.Name
cpu_percent = [string]$value.CPUPerc
memory_usage = [string]$value.MemUsage
memory_percent = [string]$value.MemPerc
pids = [string]$value.PIDs
} | ConvertTo-Json -Compress | Out-File -LiteralPath $telemetryPath -Encoding utf8 -Append
}
}
if (-not $graphState.State.Running -and -not $tgsState.State.Running) { break }
Start-Sleep -Seconds 1
}
$graphExit = [int](Get-Container $graphName).State.ExitCode
$tgsExit = [int](Get-Container $tgsName).State.ExitCode
(& docker logs $graphName 2>&1) | Set-Content -LiteralPath (Join-Path $runOutput "graph.log") -Encoding utf8
(& docker logs $tgsName 2>&1) | Set-Content -LiteralPath (Join-Path $runOutput "tgs.log") -Encoding utf8
if ($graphExit -ne 0) { throw "M49 integrated graph failed with exit code $graphExit" }
if ($tgsExit -ne 0) { throw "M49 integrated TGS failed with exit code $tgsExit" }
& docker run --rm --name $analyzeName --network none --cpus 8 --memory 16g `
--entrypoint python3 `
--volume ($dockerRelease + ":/release:ro") `
--volume ($dockerRun + ":/shared:rw") `
$ParityImageTag /release/build_tgs_full_shadow_evidence.py `
--run-root /shared/tgs `
--config /release/m49-tgs-full-shadow-v1.json `
--output-root /shared/tgs/evidence
Assert-LastExitCode "M49 integrated TGS evidence analysis"
& docker run --rm --name $evidenceName --network none --cpus 4 --memory 8g `
--entrypoint python3 `
--volume ($dockerRelease + ":/release:ro") `
--volume ($dockerRun + ":/shared:rw") `
$ParityImageTag /release/build_tgs_integrated_graph_evidence.py `
--profile /release/m49-tgs-integrated-graph-shadow-v1.json `
--graph-result /shared/graph/result.json `
--graph-frames /shared/graph/frames.jsonl `
--tgs-result /shared/tgs/evidence/result.json `
--tgs-timing /shared/tgs/tgs-full-timing.tsv `
--telemetry /shared/container-telemetry.jsonl `
--output /shared/result.json `
--release-sha256 $ExpectedArtifactSha256
Assert-LastExitCode "M49 integrated evidence gate"
} finally {
foreach ($name in $containers) { Remove-ExactContainer $name }
$canonicalAfter = Get-Container "ndc-mission-core-triton"
if (
[string]$canonicalAfter.Id -cne $canonicalId -or
-not $canonicalAfter.State.Running -or
$canonicalAfter.State.Health.Status -cne "healthy"
) { throw "Canonical Mission Core Triton changed during M49 integrated shadow" }
}
$completed = [DateTimeOffset]::UtcNow
$resultPath = Join-Path $runOutput "result.json"
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
throw "M49 integrated result is missing"
}
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
$summary = [ordered]@{
schema_version = "missioncore.m49-tgs-integrated-graph-worker-summary/v1"
worker_id = "worker-006"
run_id = $RunId
code_revision = [string]$releaseDocument.code_revision
source_rate_hz = $SourceRateHz
started_utc = $started.ToString("o")
completed_utc = $completed.ToString("o")
wall_seconds = [math]::Round(($completed - $started).TotalSeconds, 6)
free_memory_gib_before = [math]::Round($freeMemoryGiB, 6)
result_id = [string]$result.result_id
result_status = [string]$result.status
canonical_triton_id = $canonicalId
canonical_triton_health = "healthy"
gauss_or_playcanvas_action = "none"
durable_worker_action = "none"
navigation_or_actuation_allowed = $false
}
$summary | ConvertTo-Json -Depth 3 | Set-Content -LiteralPath (
Join-Path $runOutput "worker-summary.json"
) -Encoding utf8
$summary | ConvertTo-Json -Depth 3
@@ -0,0 +1,15 @@
#!/usr/bin/env bash
set -euo pipefail
readonly TARGET=${1:?target binary path is required}
test -f /release/run_tgs_full_shadow.cpp
test ! -e "${TARGET}"
mkdir -p "$(dirname "${TARGET}")"
g++ -std=c++17 -O3 -DNDEBUG -pthread \
-I/opt/travel/src/TRAVEL/cpp/travel/core \
-I/usr/include/eigen3 \
/release/run_tgs_full_shadow.cpp \
-o "${TARGET}"
chmod 0755 "${TARGET}"
sha256sum "${TARGET}"
@@ -0,0 +1,339 @@
#!/usr/bin/env python3
"""Seal the synchronized TGS plus native RF-DETR reference-graph shadow."""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
import math
import re
from collections import defaultdict
from pathlib import Path
from typing import Any
import numpy as np
PROFILE_SCHEMA = "missioncore.m49-tgs-integrated-graph-shadow-profile/v1"
GRAPH_SCHEMA = "missioncore.m48s-reference-graph-shadow-load/v5"
TGS_SCHEMA = "missioncore.m49-tgs-full-shadow-result/v1"
RESULT_SCHEMA = "missioncore.m49-tgs-integrated-graph-shadow-result/v1"
FRAME_COUNT = 4_489
class IntegratedShadowError(RuntimeError):
"""The integrated shadow evidence is incomplete or incompatible."""
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def canonical_json(value: object) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
def load_json(path: Path, label: str) -> dict[str, Any]:
try:
value = json.loads(path.read_text(encoding="utf-8-sig"))
except (OSError, json.JSONDecodeError) as exc:
raise IntegratedShadowError(f"{label} is unreadable") from exc
if not isinstance(value, dict):
raise IntegratedShadowError(f"{label} is not an object")
return value
def distribution(values: list[float]) -> dict[str, float]:
if not values:
return {"mean": 0.0, "p50": 0.0, "p95": 0.0, "p99": 0.0, "maximum": 0.0}
array = np.asarray(values, dtype=np.float64)
return {
"mean": round(float(array.mean()), 6),
"p50": round(float(np.percentile(array, 50)), 6),
"p95": round(float(np.percentile(array, 95)), 6),
"p99": round(float(np.percentile(array, 99)), 6),
"maximum": round(float(array.max()), 6),
}
def graph_completion_ages(path: Path) -> list[float]:
ages: list[float] = []
with path.open("r", encoding="utf-8") as stream:
for expected, line in enumerate(stream):
row = json.loads(line)
sequence = row.get("source_envelope", {}).get("sequence")
if sequence != expected:
raise IntegratedShadowError("graph frame ledger sequence changed")
age_ns = row.get("completion_age_ns")
if not isinstance(age_ns, int) or age_ns < 0:
raise IntegratedShadowError("graph completion age is invalid")
ages.append(age_ns / 1_000_000.0)
if len(ages) != FRAME_COUNT:
raise IntegratedShadowError("graph frame ledger is incomplete")
return ages
def tgs_completion_ages(path: Path) -> list[float]:
ages: list[float] = []
with path.open("r", encoding="utf-8", newline="") as stream:
for expected, row in enumerate(csv.DictReader(stream, delimiter="\t")):
if int(row["timeline_frame_index"]) != expected:
raise IntegratedShadowError("TGS timing sequence changed")
value = float(row["completion_age_ms"])
if not math.isfinite(value) or value < 0:
raise IntegratedShadowError("TGS completion age is invalid")
ages.append(value)
if len(ages) != FRAME_COUNT:
raise IntegratedShadowError("TGS timing ledger is incomplete")
return ages
_SIZE = re.compile(r"^\s*([0-9.]+)\s*([kmgt]?i?b)\s*$", re.IGNORECASE)
def size_mib(value: str) -> float:
match = _SIZE.fullmatch(value)
if match is None:
raise IntegratedShadowError("container memory telemetry is invalid")
number = float(match.group(1))
unit = match.group(2).lower()
scale = {
"b": 1.0 / (1024.0 * 1024.0),
"kb": 1.0 / 1024.0,
"kib": 1.0 / 1024.0,
"mb": 1.0,
"mib": 1.0,
"gb": 1024.0,
"gib": 1024.0,
"tb": 1024.0 * 1024.0,
"tib": 1024.0 * 1024.0,
}[unit]
return number * scale
def host_telemetry(path: Path) -> dict[str, object]:
samples: dict[str, list[dict[str, float]]] = defaultdict(list)
with path.open("r", encoding="utf-8-sig") as stream:
for line in stream:
row = json.loads(line)
role = row.get("role")
if role not in {"graph", "tgs", "triton"}:
raise IntegratedShadowError("container telemetry role changed")
cpu_text = row.get("cpu_percent")
memory_text = row.get("memory_usage")
memory_percent_text = row.get("memory_percent")
if not all(
isinstance(value, str) for value in (cpu_text, memory_text, memory_percent_text)
):
raise IntegratedShadowError("container telemetry row is incomplete")
used_text = memory_text.split("/", 1)[0].strip()
samples[role].append(
{
"cpu_percent": float(cpu_text.rstrip("%")),
"memory_used_mib": size_mib(used_text),
"memory_percent": float(memory_percent_text.rstrip("%")),
}
)
if any(not samples[role] for role in ("graph", "tgs", "triton")):
raise IntegratedShadowError("container telemetry does not cover every runtime role")
return {
role: {
"sample_count": len(rows),
"cpu_percent": distribution([row["cpu_percent"] for row in rows]),
"memory_used_mib": distribution([row["memory_used_mib"] for row in rows]),
"memory_percent": distribution([row["memory_percent"] for row in rows]),
}
for role, rows in sorted(samples.items())
}
def build(
*,
profile_path: Path,
graph_result_path: Path,
graph_frames_path: Path,
tgs_result_path: Path,
tgs_timing_path: Path,
telemetry_path: Path,
output_path: Path,
release_sha256: str,
) -> dict[str, object]:
if output_path.exists():
raise IntegratedShadowError("integrated result already exists")
profile = load_json(profile_path, "integrated profile")
graph = load_json(graph_result_path, "reference graph result")
tgs = load_json(tgs_result_path, "TGS result")
if profile.get("schema_version") != PROFILE_SCHEMA:
raise IntegratedShadowError("integrated profile schema changed")
if graph.get("schema_version") != GRAPH_SCHEMA:
raise IntegratedShadowError("reference graph result schema changed")
if tgs.get("schema_version") != TGS_SCHEMA:
raise IntegratedShadowError("TGS result schema changed")
if len(release_sha256) != 64 or any(
value not in "0123456789abcdef" for value in release_sha256
):
raise IntegratedShadowError("release SHA-256 is invalid")
graph_ages = graph_completion_ages(graph_frames_path)
tgs_ages = tgs_completion_ages(tgs_timing_path)
combined_ages = [
max(graph_age, tgs_age) for graph_age, tgs_age in zip(graph_ages, tgs_ages, strict=True)
]
combined = distribution(combined_ages)
telemetry = host_telemetry(telemetry_path)
acceptance_profile = profile["acceptance"]
execution = graph.get("execution", {})
graph_metrics = graph.get("metrics", {})
tgs_performance = tgs.get("performance", {})
effective_fps = float(execution.get("effective_world_state_fps", 0.0))
reference_fps = float(acceptance_profile["reference_world_state_fps"])
fps_regression = max(0.0, (reference_fps - effective_fps) / reference_fps)
graph_p95 = float(graph_metrics.get("world_state_completion_age_ms", {}).get("p95", math.inf))
tgs_p95 = float(tgs_performance.get("candidate_tgs_ms", {}).get("p95", math.inf))
tgs_p99 = float(tgs_performance.get("candidate_tgs_ms", {}).get("p99", math.inf))
tgs_drops = int(tgs_performance.get("capacity_drop_count", -1))
terminal = execution.get("terminal_outcomes", {})
superseded = int(terminal.get("superseded", 0)) if isinstance(terminal, dict) else -1
gpu_samples = int(graph_metrics.get("gpu", {}).get("sample_count", 0))
graph_inputs = graph.get("identity", {}).get("inputs", {})
checks = {
"frozen_graph_identity": (
graph_inputs.get("graph_config")
== profile["stages"]["reference_graph"]["graph_config_sha256"]
and graph_inputs.get("detector_profile")
== profile["stages"]["reference_graph"]["detector_profile_sha256"]
),
"frozen_tgs_identity": (
tgs.get("config_sha256") == profile["stages"]["tgs"]["profile_sha256"]
),
"requested_source_rate_preserved": (
execution.get("requested_source_rate_hz")
== profile["source"]["requested_source_rate_hz"]
),
"all_graph_frames_delivered": (
execution.get("admitted_frames") == FRAME_COUNT
and execution.get("delivered_world_states") == FRAME_COUNT
),
"all_tgs_frames_accounted": tgs.get("timeline", {}).get("frame_count") == FRAME_COUNT,
"exact_sequence_join": len(combined_ages) == FRAME_COUNT,
"minimum_delivery_ratio": float(execution.get("delivery_ratio", 0.0))
>= float(acceptance_profile["minimum_delivery_ratio"]),
"maximum_world_state_fps_regression": fps_regression
<= float(acceptance_profile["maximum_world_state_fps_regression_fraction"]),
"minimum_effective_world_state_fps": effective_fps
>= float(acceptance_profile["minimum_effective_world_state_fps"]),
"maximum_world_state_completion_p95_ms": graph_p95
<= float(acceptance_profile["maximum_world_state_completion_p95_ms"]),
"candidate_stage_p95_ms": tgs_p95
<= float(acceptance_profile["candidate_stage_p95_ms_max"]),
"candidate_stage_p99_ms": tgs_p99
<= float(acceptance_profile["candidate_stage_p99_ms_max"]),
"combined_output_age_p99_ms": combined["p99"]
<= float(acceptance_profile["combined_output_age_p99_ms_max"]),
"zero_capacity_drops": tgs_drops <= int(acceptance_profile["capacity_drop_count_max"])
and superseded <= int(acceptance_profile["capacity_drop_count_max"]),
"reference_graph_integrity": graph.get("evidence_integrity_gate_passed") is True,
"tgs_integrity": tgs.get("status") == "passed",
"host_resource_telemetry_complete": all(
telemetry[role]["sample_count"] > 0 for role in ("graph", "tgs", "triton")
),
"gpu_telemetry_complete": gpu_samples > 0,
"authority_remains_false": all(value is False for value in profile["authority"].values()),
}
files = {
label: {"bytes": path.stat().st_size, "sha256": sha256_file(path)}
for label, path in (
("graph-result.json", graph_result_path),
("graph-frames.jsonl", graph_frames_path),
("tgs-result.json", tgs_result_path),
("tgs-timing.tsv", tgs_timing_path),
("container-telemetry.jsonl", telemetry_path),
)
}
document: dict[str, object] = {
"schema_version": RESULT_SCHEMA,
"profile_id": profile["profile_id"],
"status": "passed" if all(checks.values()) else "failed",
"source": {
"source_id": profile["source"]["source_id"],
"source_pack_sha256": profile["source"]["source_pack_sha256"],
"requested_source_rate_hz": profile["source"]["requested_source_rate_hz"],
"joined_frame_count": len(combined_ages),
},
"identity": {
"release_sha256": release_sha256,
"profile_sha256": sha256_file(profile_path),
"graph_config_sha256": profile["stages"]["reference_graph"]["graph_config_sha256"],
"tgs_profile_sha256": profile["stages"]["tgs"]["profile_sha256"],
"linked_accepted_tgs_result_id": profile["stages"]["tgs"]["linked_accepted_result_id"],
},
"performance": {
"effective_world_state_fps": effective_fps,
"reference_world_state_fps": reference_fps,
"world_state_fps_regression_fraction": round(fps_regression, 9),
"world_state_completion_age_ms": graph_metrics.get("world_state_completion_age_ms"),
"tgs_candidate_stage_ms": tgs_performance.get("candidate_tgs_ms"),
"tgs_completion_age_ms": tgs_performance.get("completion_age_ms"),
"combined_output_age_ms": combined,
"gpu": graph_metrics.get("gpu"),
"host_containers": telemetry,
},
"accounting": {
"graph_admitted": execution.get("admitted_frames"),
"graph_delivered": execution.get("delivered_world_states"),
"graph_terminal_outcomes": terminal,
"tgs_timeline_frames": tgs.get("timeline", {}).get("frame_count"),
"tgs_available_lidar_frames": tgs.get("timeline", {}).get(
"available_lidar_frame_count"
),
"tgs_capacity_drops": tgs_drops,
},
"checks": checks,
"integrated_runtime_gate_passed": all(checks.values()),
"visual_quality_accepted": False,
"traversability_accepted": False,
"production_accepted": False,
"authority": profile["authority"],
"files": files,
}
identity = hashlib.sha256(canonical_json(document)).hexdigest()
document["result_id"] = f"m49-tgs-integrated-graph-shadow-{identity}"
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8")
return document
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--profile", type=Path, required=True)
parser.add_argument("--graph-result", type=Path, required=True)
parser.add_argument("--graph-frames", type=Path, required=True)
parser.add_argument("--tgs-result", type=Path, required=True)
parser.add_argument("--tgs-timing", type=Path, required=True)
parser.add_argument("--telemetry", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--release-sha256", required=True)
arguments = parser.parse_args()
result = build(
profile_path=arguments.profile,
graph_result_path=arguments.graph_result,
graph_frames_path=arguments.graph_frames,
tgs_result_path=arguments.tgs_result,
tgs_timing_path=arguments.tgs_timing,
telemetry_path=arguments.telemetry,
output_path=arguments.output,
release_sha256=arguments.release_sha256,
)
print(
json.dumps({"result_id": result["result_id"], "status": result["status"]}, sort_keys=True)
)
return 0 if result["status"] == "passed" else 2
if __name__ == "__main__":
raise SystemExit(main())
@@ -1,4 +1,5 @@
#include <chrono>
#include <cmath>
#include <filesystem>
#include <fstream>
#include <iomanip>
@@ -76,11 +77,38 @@ double milliseconds(Clock::duration duration) {
return std::chrono::duration<double, std::milli>(duration).count();
}
void waitForSharedStart(const std::string& ready_path, const std::string& start_path) {
if (ready_path.empty() != start_path.empty()) {
throw std::runtime_error("shared-start paths must be configured together");
}
if (ready_path.empty()) {
return;
}
if (std::filesystem::exists(ready_path)) {
throw std::runtime_error("shared-start ready file already exists");
}
{
std::ofstream ready(ready_path);
ready << "ready\n";
if (!ready) {
throw std::runtime_error("cannot publish TGS shared-start readiness");
}
}
const auto deadline = Clock::now() + std::chrono::minutes(10);
while (!std::filesystem::is_regular_file(start_path)) {
if (Clock::now() >= deadline) {
throw std::runtime_error("TGS shared-start barrier timed out");
}
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
}
} // namespace
int main(int argc, char** argv) {
if (argc != 5) {
std::cerr << "Usage: run_tgs_full_shadow <sequence_dir> <schedule.tsv> <output_dir> <timing.tsv>\n";
if (argc != 5 && argc != 8) {
std::cerr << "Usage: run_tgs_full_shadow <sequence_dir> <schedule.tsv> <output_dir>"
" <timing.tsv> [target_rate_hz ready_file start_file]\n";
return 1;
}
try {
@@ -89,6 +117,19 @@ int main(int argc, char** argv) {
const std::string output_dir = argv[3];
const std::string timing_path = argv[4];
const auto schedule = readSchedule(schedule_path);
const double target_rate_hz = argc == 8 ? std::stod(argv[5]) : 0.0;
if (target_rate_hz < 0.0 || !std::isfinite(target_rate_hz)) {
throw std::runtime_error("invalid TGS target rate");
}
const double source_duration_seconds =
schedule.back().session_seconds - schedule.front().session_seconds;
if (!(source_duration_seconds > 0.0)) {
throw std::runtime_error("invalid TGS source duration");
}
const double recorded_rate_hz =
static_cast<double>(schedule.size() - 1) / source_duration_seconds;
const double pacing_scale =
target_rate_hz > 0.0 ? recorded_rate_hz / target_rate_hz : 1.0;
KittiLoader loader(sequence_dir);
if (loader.size() != 3928) {
throw std::runtime_error("full-shadow available LiDAR frame count changed");
@@ -103,12 +144,14 @@ int main(int argc, char** argv) {
<< "\ttgs_ms\tstage_wall_ms\tqueue_delay_ms\tcompletion_age_ms\tcapacity_drop\n";
timing << std::fixed << std::setprecision(6);
waitForSharedStart(argc == 8 ? argv[6] : "", argc == 8 ? argv[7] : "");
const double first_source_seconds = schedule.front().session_seconds;
const auto run_started = Clock::now();
std::size_t expected_slot = 0;
for (const auto& row : schedule) {
const auto target = run_started + std::chrono::duration_cast<Clock::duration>(
std::chrono::duration<double>(row.session_seconds - first_source_seconds));
std::chrono::duration<double>(
(row.session_seconds - first_source_seconds) * pacing_scale));
const auto before_wait = Clock::now();
if (before_wait < target) {
std::this_thread::sleep_until(target);
@@ -173,6 +216,9 @@ int main(int argc, char** argv) {
throw std::runtime_error("full-shadow available frame accounting changed");
}
std::cout << "[TGS-FULL] complete timeline=4489 available=3928\n";
std::cout << "[TGS-FULL] recorded_rate_hz=" << recorded_rate_hz
<< " target_rate_hz=" << (target_rate_hz > 0.0 ? target_rate_hz : recorded_rate_hz)
<< "\n";
return 0;
} catch (const std::exception& error) {
std::cerr << "[TGS-FULL] " << error.what() << '\n';
@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -euo pipefail
readonly BINARY=/shared/bin/run_tgs_full_shadow
readonly INPUT_ROOT=/shared/tgs/inputs
readonly OUTPUT_ROOT=/shared/tgs/outputs/causal_rolling_1s
readonly TIMING_PATH=/shared/tgs/tgs-full-timing.tsv
readonly READY_FILE=/shared/control/tgs.ready
readonly START_FILE=/shared/control/start.signal
readonly SOURCE_RATE_HZ=${M49_SOURCE_RATE_HZ:-12.0}
test -x "${BINARY}"
test -f "${INPUT_ROOT}/input-manifest.json"
test -f "${INPUT_ROOT}/schedule.tsv"
test ! -e /shared/tgs/outputs
test ! -e "${TIMING_PATH}"
test ! -e "${READY_FILE}"
mkdir -p "${OUTPUT_ROOT}"
exec /usr/bin/time -v "${BINARY}" \
"${INPUT_ROOT}/profiles/causal_rolling_1s" \
"${INPUT_ROOT}/schedule.tsv" \
"${OUTPUT_ROOT}" \
"${TIMING_PATH}" \
"${SOURCE_RATE_HZ}" \
"${READY_FILE}" \
"${START_FILE}"
@@ -0,0 +1,257 @@
#!/usr/bin/env python3
"""Build a clean-revision Worker 006 release for the integrated M4.9 shadow."""
from __future__ import annotations
import argparse
import gzip
import hashlib
import io
import json
import os
import re
import subprocess
import tarfile
import tempfile
from pathlib import Path
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
WHEEL_NAME = "nodedc_mission_core-0.1.0-py3-none-any.whl"
PATCH_ID = re.compile(r"^[A-Za-z0-9._-]{1,96}$")
SOURCES = (
Path("experiments/perception/worker/Invoke-M49TgsIntegratedGraphShadow.ps1"),
Path("experiments/perception/run_m48s_reference_graph_shadow_worker.py"),
Path("experiments/perception/worker/rf_detr_large_native_kb4_config.pbtxt"),
Path("experiments/perception/worker/m49_t3_travel/prepare_tgs_fail_closed_inputs.py"),
Path("experiments/perception/worker/m49_t3_travel/prepare_tgs_full_shadow_inputs.py"),
Path("experiments/perception/worker/m49_t3_travel/run_tgs_full_shadow.cpp"),
Path("experiments/perception/worker/m49_t3_travel/build_tgs_full_shadow_binary.sh"),
Path("experiments/perception/worker/m49_t3_travel/run_tgs_integrated_shadow.sh"),
Path("experiments/perception/worker/m49_t3_travel/build_tgs_full_shadow_evidence.py"),
Path("experiments/perception/worker/m49_t3_travel/build_tgs_integrated_graph_evidence.py"),
Path("config/perception/m49-tgs-integrated-graph-shadow-v1.json"),
Path("config/perception/m49-tgs-full-shadow-v1.json"),
Path("config/perception/m48n-rf-detr-native-reference-graph-shadow-v0.json"),
Path("config/perception/m4-recorded-realtime-baseline-v1.json"),
Path("config/perception/rf-detr-large-native-kb4-risk-shadow-v0.json"),
Path("config/perception/m4-geometry-association-v1.json"),
Path("config/perception/m4-temporal-motion-v1.json"),
Path("config/perception/m4-rolling-local-map-v1.json"),
Path("config/perception/m4-replay-threat-v3.json"),
)
class ArtifactBuildError(RuntimeError):
"""The integrated Worker release cannot be built from its declared revision."""
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def git_revision() -> str:
result = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=REPOSITORY_ROOT,
check=True,
capture_output=True,
text=True,
)
revision = result.stdout.strip()
if re.fullmatch(r"[a-f0-9]{40}", revision) is None:
raise ArtifactBuildError("Git revision is not a full SHA-1")
return revision
def materialize_revision(revision: str, destination: Path) -> None:
archive_path = destination.parent / "source.tar"
subprocess.run(
["git", "archive", "--format=tar", "--output", str(archive_path), revision],
cwd=REPOSITORY_ROOT,
check=True,
capture_output=True,
)
destination.mkdir()
root = destination.resolve()
with tarfile.open(archive_path, "r:") as archive:
for member in archive.getmembers():
target = (destination / member.name).resolve()
if target != root and root not in target.parents:
raise ArtifactBuildError("Git archive contains an unsafe path")
archive.extractall(destination)
def build_wheel(source_root: Path, output: Path) -> Path:
environment = os.environ.copy()
environment["SOURCE_DATE_EPOCH"] = "0"
result = subprocess.run(
["uv", "build", "--wheel", "--out-dir", str(output)],
cwd=source_root,
env=environment,
check=False,
capture_output=True,
text=True,
)
if result.returncode != 0:
detail = (result.stderr or result.stdout).strip()
raise ArtifactBuildError(f"wheel build failed: {detail}")
wheel = output / WHEEL_NAME
if not wheel.is_file() or wheel.is_symlink():
raise ArtifactBuildError("expected Worker wheel was not built")
return wheel
def tar_info(path: Path, arcname: str) -> tarfile.TarInfo:
info = tarfile.TarInfo(arcname)
info.uid = info.gid = 0
info.uname = info.gname = "root"
info.mtime = 0
if path.is_dir():
info.type = tarfile.DIRTYPE
info.mode = 0o755
else:
info.type = tarfile.REGTYPE
info.mode = 0o755 if path.suffix in {".sh", ".ps1", ".py"} else 0o644
info.size = path.stat().st_size
return info
def write_archive(stage: Path, target: Path) -> None:
members = [stage / "manifest.env", stage / "files.txt", stage / "payload"]
members.extend(sorted((stage / "payload").rglob("*")))
target.parent.mkdir(parents=True, exist_ok=True)
with (
target.open("wb") as raw,
gzip.GzipFile(filename="", mode="wb", fileobj=raw, compresslevel=9, mtime=0) as compressed,
tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive,
):
for path in members:
info = tar_info(path, path.relative_to(stage).as_posix())
if path.is_file():
with path.open("rb") as stream:
archive.addfile(info, stream)
else:
archive.addfile(info, io.BytesIO())
def build_artifact(
patch_id: str,
output_directory: Path,
*,
revision: str | None = None,
source_root: Path | None = None,
) -> dict[str, object]:
if PATCH_ID.fullmatch(patch_id) is None:
raise ArtifactBuildError("patch id is invalid")
selected_revision = revision or git_revision()
if re.fullmatch(r"[a-f0-9]{40}", selected_revision) is None:
raise ArtifactBuildError("artifact revision is invalid")
with tempfile.TemporaryDirectory(prefix="mission-core-m49-integrated-") as directory:
stage = Path(directory)
snapshot = source_root
if snapshot is None:
snapshot = stage / "source"
materialize_revision(selected_revision, snapshot)
sources = tuple(snapshot / relative for relative in SOURCES)
if any(path.is_symlink() or not path.is_file() for path in sources):
raise ArtifactBuildError("release input is not a regular file")
payload = stage / "payload"
payload.mkdir()
wheel = build_wheel(snapshot, stage / "wheel")
copied: list[Path] = []
for source in sources:
destination = payload / source.name
if destination.exists():
raise ArtifactBuildError("release payload file names are not unique")
destination.write_bytes(source.read_bytes())
copied.append(destination)
wheel_destination = payload / WHEEL_NAME
wheel_destination.write_bytes(wheel.read_bytes())
copied.append(wheel_destination)
release = {
"schema_version": "missioncore.m49-tgs-integrated-graph-worker-release/v1",
"patch_id": patch_id,
"transition": "m49-tgs-native-risk-integrated-shadow/v1",
"code_revision": selected_revision,
"worker_id": "worker-006",
"source_pack_sha256": (
"0685d24219d8236caf8b7f1685e93f6d6b59e7fd015a768d88a92bbe8b154944"
),
"expected_frames": 4489,
"requested_source_rate_hz": 12.0,
"native_engine_sha256": (
"b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695"
),
"images": {
"travel": "sha256:7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f",
"parity": "sha256:ceb13548617e4bd3f619766bfdff00af3fa5160946b367828da6d2233dcdcba0",
"runtime": (
"sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
),
},
"authority": {
"visual_quality_accepted": False,
"traversability_accepted": False,
"physical_free_space_accepted": False,
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
"production_accepted": False,
},
"scope": {
"gauss_or_playcanvas_action": "none",
"durable_worker_action": "none",
"canonical_triton_action": "none",
},
"files": {
path.name: {"sha256": sha256_file(path), "bytes": path.stat().st_size}
for path in sorted(copied)
},
}
release_path = payload / "release.json"
release_path.write_text(
json.dumps(release, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
payload_files = sorted((*release["files"], release_path.name))
(stage / "manifest.env").write_text(
f"id={patch_id}\ncomponent=mission-core-worker\ntype=shadow-release\n",
encoding="utf-8",
)
(stage / "files.txt").write_text("\n".join(payload_files) + "\n", encoding="utf-8")
target = output_directory.resolve() / f"nodedc-{patch_id}.tgz"
write_archive(stage, target)
return {
"ok": True,
"patch_id": patch_id,
"artifact": str(target),
"sha256": sha256_file(target),
"code_revision": selected_revision,
"wheel_sha256": release["files"][WHEEL_NAME]["sha256"],
"payload_files": payload_files,
"transition": release["transition"],
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("patch_id")
parser.add_argument(
"--output-directory",
type=Path,
default=REPOSITORY_ROOT / ".runtime/worker-artifacts",
)
arguments = parser.parse_args()
try:
result = build_artifact(arguments.patch_id, arguments.output_directory)
except (ArtifactBuildError, OSError, subprocess.SubprocessError) as exc:
parser.error(str(exc))
print(json.dumps(result, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+31
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import importlib.util
import time
from pathlib import Path
from threading import Thread
from unittest.mock import patch
from k1link.perception.detector import DetectorFrameTiming
@@ -157,3 +158,33 @@ def test_cyclic_gc_policy_collects_outside_hot_loop_and_restores_state() -> None
"pre_collected": 3,
"post_collected": 0,
}
def test_shared_start_barrier_publishes_readiness_and_waits_for_release(
tmp_path: Path,
) -> None:
ready = tmp_path / "graph.ready"
start = tmp_path / "start.signal"
completed: list[bool] = []
thread = Thread(
target=lambda: (
RUNNER.wait_for_shared_start(
ready_file=ready,
start_file=start,
timeout_seconds=1.0,
),
completed.append(True),
)
)
thread.start()
deadline = time.monotonic() + 1.0
while not ready.exists() and time.monotonic() < deadline:
time.sleep(0.005)
assert ready.read_text(encoding="utf-8") == "ready\n"
assert completed == []
start.write_text("start\n", encoding="utf-8")
thread.join(timeout=1.0)
assert completed == [True]
@@ -0,0 +1,208 @@
from __future__ import annotations
import importlib.util
import json
import tarfile
from pathlib import Path
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
EVIDENCE_PATH = (
REPOSITORY_ROOT
/ "experiments/perception/worker/m49_t3_travel/build_tgs_integrated_graph_evidence.py"
)
ARTIFACT_PATH = REPOSITORY_ROOT / "scripts/build_m49_tgs_integrated_graph_worker_artifact.py"
def load_module(name: str, path: Path):
spec = importlib.util.spec_from_file_location(name, path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
EVIDENCE = load_module("m49_tgs_integrated_evidence", EVIDENCE_PATH)
ARTIFACT = load_module("m49_tgs_integrated_artifact", ARTIFACT_PATH)
def test_integrated_gate_joins_all_frames_and_preserves_false_authority(tmp_path: Path) -> None:
profile_path = tmp_path / "profile.json"
profile_path.write_text(
json.dumps(
{
"schema_version": EVIDENCE.PROFILE_SCHEMA,
"profile_id": "test",
"source": {
"source_id": "RAVNOVES00",
"source_pack_sha256": "a" * 64,
"requested_source_rate_hz": 12.0,
},
"stages": {
"reference_graph": {
"graph_config_sha256": "b" * 64,
"detector_profile_sha256": "z" * 64,
},
"tgs": {
"profile_sha256": "c" * 64,
"linked_accepted_result_id": "m49-tgs-full-shadow-" + "d" * 64,
},
},
"acceptance": {
"minimum_delivery_ratio": 1.0,
"reference_world_state_fps": 11.79902,
"maximum_world_state_fps_regression_fraction": 0.05,
"minimum_effective_world_state_fps": 11.209069,
"maximum_world_state_completion_p95_ms": 125.0,
"candidate_stage_p95_ms_max": 25.0,
"candidate_stage_p99_ms_max": 50.0,
"combined_output_age_p99_ms_max": 125.0,
"capacity_drop_count_max": 0,
},
"authority": {
"visual_quality_accepted": False,
"traversability_accepted": False,
"physical_free_space_accepted": False,
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
"production_accepted": False,
},
}
),
encoding="utf-8",
)
graph_result = tmp_path / "graph-result.json"
graph_result.write_text(
json.dumps(
{
"schema_version": EVIDENCE.GRAPH_SCHEMA,
"execution": {
"admitted_frames": EVIDENCE.FRAME_COUNT,
"delivered_world_states": EVIDENCE.FRAME_COUNT,
"effective_world_state_fps": 11.75,
"delivery_ratio": 1.0,
"terminal_outcomes": {"delivered": EVIDENCE.FRAME_COUNT},
"requested_source_rate_hz": 12.0,
},
"identity": {
"inputs": {
"graph_config": "b" * 64,
"detector_profile": "z" * 64,
}
},
"metrics": {
"world_state_completion_age_ms": {"p95": 40.0},
"gpu": {"sample_count": 2},
},
"evidence_integrity_gate_passed": True,
}
),
encoding="utf-8",
)
graph_frames = tmp_path / "graph-frames.jsonl"
graph_frames.write_text(
"".join(
json.dumps(
{
"source_envelope": {"sequence": index},
"completion_age_ns": 40_000_000,
}
)
+ "\n"
for index in range(EVIDENCE.FRAME_COUNT)
),
encoding="utf-8",
)
tgs_result = tmp_path / "tgs-result.json"
tgs_result.write_text(
json.dumps(
{
"schema_version": EVIDENCE.TGS_SCHEMA,
"status": "passed",
"config_sha256": "c" * 64,
"timeline": {
"frame_count": EVIDENCE.FRAME_COUNT,
"available_lidar_frame_count": 3928,
},
"performance": {
"candidate_tgs_ms": {"p95": 2.0, "p99": 3.0},
"completion_age_ms": {"p99": 5.0},
"capacity_drop_count": 0,
},
}
),
encoding="utf-8",
)
tgs_timing = tmp_path / "tgs-timing.tsv"
tgs_timing.write_text(
"timeline_frame_index\tcompletion_age_ms\n"
+ "".join(f"{index}\t5.0\n" for index in range(EVIDENCE.FRAME_COUNT)),
encoding="utf-8",
)
telemetry = tmp_path / "telemetry.jsonl"
telemetry.write_text(
"".join(
json.dumps(
{
"role": role,
"cpu_percent": "10.0%",
"memory_usage": "1GiB / 64GiB",
"memory_percent": "1.56%",
}
)
+ "\n"
for role in ("graph", "tgs", "triton")
),
encoding="utf-8",
)
output = tmp_path / "result.json"
result = EVIDENCE.build(
profile_path=profile_path,
graph_result_path=graph_result,
graph_frames_path=graph_frames,
tgs_result_path=tgs_result,
tgs_timing_path=tgs_timing,
telemetry_path=telemetry,
output_path=output,
release_sha256="e" * 64,
)
assert result["status"] == "passed"
assert result["source"]["joined_frame_count"] == EVIDENCE.FRAME_COUNT
assert result["performance"]["combined_output_age_ms"]["p99"] == 40.0
assert result["checks"]["authority_remains_false"] is True
assert result["production_accepted"] is False
def test_worker_artifact_is_deterministic_and_excludes_gauss(monkeypatch, tmp_path: Path) -> None:
def fake_wheel(_source_root: Path, output: Path) -> Path:
output.mkdir(parents=True, exist_ok=True)
wheel = output / ARTIFACT.WHEEL_NAME
wheel.write_bytes(b"clean committed wheel\n")
return wheel
monkeypatch.setattr(ARTIFACT, "build_wheel", fake_wheel)
revision = "f" * 40
first = ARTIFACT.build_artifact(
"mission-core-m49-integrated-unit-001",
tmp_path / "first",
revision=revision,
source_root=REPOSITORY_ROOT,
)
second = ARTIFACT.build_artifact(
"mission-core-m49-integrated-unit-001",
tmp_path / "second",
revision=revision,
source_root=REPOSITORY_ROOT,
)
assert Path(first["artifact"]).read_bytes() == Path(second["artifact"]).read_bytes()
with tarfile.open(first["artifact"], "r:gz") as archive:
names = set(archive.getnames())
release_stream = archive.extractfile("payload/release.json")
assert release_stream is not None
release = json.loads(release_stream.read())
assert not any("gauss" in name.lower() or "playcanvas" in name.lower() for name in names)
assert release["scope"]["gauss_or_playcanvas_action"] == "none"
assert all(value is False for value in release["authority"].values())