SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): integrate vegetation policy review

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-28 11:22:00 +03:00
parent 30080c51aa
commit c4c2392c79
17 changed files with 2388 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/perception/worker/Invoke-M49TgsIntegratedGraphShadow.ps1
+190 -15
View File
@@ -12,6 +12,10 @@ param(
[string]$RunId,
[ValidateRange(1.0, 120.0)]
[double]$SourceRateHz = 12.0,
[switch]$VegetationLoadGate,
[string]$VegetationAssetRoot = (
"D:\NDC_MISSIONCORE\datasets\vegetation-v1\observed-2026-08-27"
),
[string]$OutputRoot = (
"D:\NDC_MISSIONCORE\runtime\results\m49-tgs-integrated-graph-shadow"
)
@@ -23,6 +27,8 @@ $TravelImageTag = "ndc/mission-core-m49-t3-travel:20260826"
$TravelImageId = "sha256:7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f"
$ParityImageTag = "ndc-mission-core-m48t-upstream-parity:1.9.4-cu130"
$ParityImageId = "sha256:ceb13548617e4bd3f619766bfdff00af3fa5160946b367828da6d2233dcdcba0"
$VegetationImageTag = "ndc/mission-core-lab-v1-goose:sg3.2.0-cu117-v1"
$VegetationImageId = "sha256:591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd"
$RuntimeImage = (
"nvcr.io/nvidia/tritonserver:26.06-py3@" +
"sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
@@ -98,12 +104,20 @@ function Wait-Healthy([string]$Name) {
function Wait-SharedReady(
[string]$GraphReady,
[string]$TgsReady,
[string]$VegetationReady,
[string]$GraphName,
[string]$TgsName
[string]$TgsName,
[string]$VegetationName
) {
$deadline = [DateTimeOffset]::UtcNow.AddMinutes(10)
while (-not ((Test-Path -LiteralPath $GraphReady) -and (Test-Path -LiteralPath $TgsReady))) {
foreach ($name in @($GraphName, $TgsName)) {
$requiredFiles = @($GraphReady, $TgsReady)
$requiredContainers = @($GraphName, $TgsName)
if (-not [string]::IsNullOrWhiteSpace($VegetationReady)) {
$requiredFiles += $VegetationReady
$requiredContainers += $VegetationName
}
while ($requiredFiles.Where({ -not (Test-Path -LiteralPath $_) }).Count -gt 0) {
foreach ($name in $requiredContainers) {
$container = Get-Container $name
if (-not $container.State.Running) {
& docker logs $name
@@ -128,15 +142,25 @@ $runCandidate = Join-Path $output $RunId
if (Test-Path -LiteralPath $runCandidate) { throw "M49 integrated output already exists" }
$null = New-Item -ItemType Directory -Path $runCandidate
$runOutput = Resolve-DDirectory $runCandidate "M49 integrated run output" $false
foreach ($directory in @("bin", "control", "graph", "tgs")) {
foreach ($directory in @("bin", "control", "graph", "tgs", "vegetation")) {
$null = New-Item -ItemType Directory -Path (Join-Path $runOutput $directory)
}
$releaseDocument = Get-Content -LiteralPath (Join-Path $payload "release.json") -Raw | ConvertFrom-Json
$expectedReleaseSchema = if ($VegetationLoadGate) {
"missioncore.lab-v1-vegetation-integrated-worker-release/v1"
} else {
"missioncore.m49-tgs-integrated-graph-worker-release/v1"
}
$expectedTransition = if ($VegetationLoadGate) {
"lab-v1-vegetation-m49-integrated-shadow/v1"
} else {
"m49-tgs-native-risk-integrated-shadow/v1"
}
if (
$releaseDocument.schema_version -cne "missioncore.m49-tgs-integrated-graph-worker-release/v1" -or
$releaseDocument.schema_version -cne $expectedReleaseSchema -or
$releaseDocument.worker_id -cne "worker-006" -or
$releaseDocument.transition -cne "m49-tgs-native-risk-integrated-shadow/v1"
$releaseDocument.transition -cne $expectedTransition
) { throw "M49 integrated release contract changed" }
foreach ($property in $releaseDocument.files.PSObject.Properties) {
$path = Join-Path $payload $property.Name
@@ -146,6 +170,9 @@ foreach ($property in $releaseDocument.files.PSObject.Properties) {
}
$wheelSha256 = [string]$releaseDocument.files."nodedc_mission_core-0.1.0-py3-none-any.whl".sha256
$runnerSha256 = [string]$releaseDocument.files."run_m48s_reference_graph_shadow_worker.py".sha256
$vegetationRunnerSha256 = if ($VegetationLoadGate) {
[string]$releaseDocument.files."run_vegetation_integrated_load.py".sha256
} else { "" }
$source = [ordered]@{
CameraIndex = (
@@ -179,6 +206,23 @@ if ((Get-Sha256 $source.SourcePack) -cne [string]$releaseDocument.source_pack_sh
throw "RAVNOVES00 source pack digest changed"
}
$vegetation = $null
if ($VegetationLoadGate) {
$vegetationRoot = Resolve-DDirectory $VegetationAssetRoot "vegetation asset root" $false
$vegetation = [ordered]@{
Dataset = Resolve-DDirectory (
(Join-Path $vegetationRoot "goose-2d\validation")
) "GOOSE validation root" $false
Checkpoint = Resolve-DFile (
(Join-Path $vegetationRoot "models\goose\ddrnet_class_512.pth")
) "DDRNet checkpoint"
}
if (
(Get-Sha256 $vegetation.Checkpoint) -cne
"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"
) { throw "DDRNet checkpoint SHA-256 changed" }
}
$nativeConfig = Resolve-DFile (
(Join-Path $payload "rf_detr_large_native_kb4_config.pbtxt")
) "native RF-DETR config"
@@ -207,12 +251,17 @@ $pillow = Resolve-DDirectory (
Assert-Image $TravelImageTag $TravelImageId
Assert-Image $ParityImageTag $ParityImageId
if ($VegetationLoadGate) { Assert-Image $VegetationImageTag $VegetationImageId }
& docker image inspect $RuntimeImage *> $null
Assert-LastExitCode "pinned runtime image inspection"
$os = Get-CimInstance Win32_OperatingSystem
$freeMemoryGiB = [double]$os.FreePhysicalMemory / 1MB
if ($freeMemoryGiB -lt 24.0) {
throw ("M49 integrated shadow requires 24 GiB free memory; observed {0:N2} GiB" -f $freeMemoryGiB)
$requiredMemoryGiB = if ($VegetationLoadGate) { 32.0 } else { 24.0 }
if ($freeMemoryGiB -lt $requiredMemoryGiB) {
throw (
"M49 integrated shadow requires {0:N0} GiB free memory; observed {1:N2} GiB" -f
$requiredMemoryGiB, $freeMemoryGiB
)
}
$canonicalBefore = Get-Container "ndc-mission-core-triton"
if (-not $canonicalBefore.State.Running -or $canonicalBefore.State.Health.Status -cne "healthy") {
@@ -225,9 +274,12 @@ $compileName = "ndc-mission-core-m49-integrated-compile-$RunId"
$tritonName = "ndc-mission-core-m49-integrated-triton-$RunId"
$graphName = "ndc-mission-core-m49-integrated-graph-$RunId"
$tgsName = "ndc-mission-core-m49-integrated-tgs-$RunId"
$vegetationName = "ndc-mission-core-m49-integrated-vegetation-$RunId"
$analyzeName = "ndc-mission-core-m49-integrated-analyze-$RunId"
$evidenceName = "ndc-mission-core-m49-integrated-evidence-$RunId"
$vegetationEvidenceName = "ndc-mission-core-m49-integrated-vegetation-evidence-$RunId"
$containers = @($prepareName, $compileName, $tritonName, $graphName, $tgsName, $analyzeName, $evidenceName)
if ($VegetationLoadGate) { $containers += @($vegetationName, $vegetationEvidenceName) }
foreach ($name in $containers) {
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
throw "M49 integrated container name already exists: $name"
@@ -236,6 +288,24 @@ foreach ($name in $containers) {
$started = [DateTimeOffset]::UtcNow
try {
if ($VegetationLoadGate) {
$vegetationFrames = Join-Path $runOutput "vegetation\input-frames"
$null = New-Item -ItemType Directory -Path $vegetationFrames
& ffmpeg -hide_banner -loglevel error -i $source.Video -map 0:v:0 -fps_mode passthrough (
Join-Path $vegetationFrames "frame-%06d.png"
)
Assert-LastExitCode "RAVNOVES full-video frame extraction"
$extractedFrames = @(
Get-ChildItem -LiteralPath $vegetationFrames -File -Filter "frame-*.png" |
Sort-Object Name
)
if (
$extractedFrames.Count -ne 4489 -or
$extractedFrames[0].Name -cne "frame-000001.png" -or
$extractedFrames[-1].Name -cne "frame-004489.png"
) { throw "RAVNOVES full-video frame sequence changed" }
}
& docker run --rm --name $prepareName --network none --cpus 8 --memory 16g `
--entrypoint python3 `
--volume ((Convert-ToDockerPath $source.SourcePack) + ":/source/lidar-pack.npz:ro") `
@@ -332,24 +402,71 @@ try {
$TravelImageTag /release/run_tgs_integrated_shadow.sh *> $null
Assert-LastExitCode "M49 integrated TGS creation"
if ($VegetationLoadGate) {
$dockerVegetationDataset = Convert-ToDockerPath $vegetation.Dataset
$dockerVegetationCheckpoint = Convert-ToDockerPath $vegetation.Checkpoint
& docker create --name $vegetationName --network none --cpus 8 --memory 10g `
--gpus all --read-only --security-opt "no-new-privileges:true" --cap-drop ALL `
--pids-limit 512 --tmpfs "/tmp:rw,noexec,nosuid,size=2g" `
-e "HOME=/tmp" `
--entrypoint conda `
--volume ($dockerRelease + ":/release:ro") `
--volume ($dockerRun + ":/shared:rw") `
--volume ($dockerVegetationDataset + ":/data/goose:ro") `
--volume ($dockerVegetationCheckpoint + ":/models/candidate.pth:ro") `
$VegetationImageTag run --no-capture-output --name goose python `
/release/run_vegetation_integrated_load.py `
--config /release/lab-v1-goose-vegetation-benchmark-v1.json `
--policy /release/lab-v1-vegetation-mission-policy-v1.json `
--provider-map /release/lab-v1-vegetation-provider-label-map-v1.json `
--checkpoint /models/candidate.pth `
--dataset-root /data/goose `
--frames-root /shared/vegetation/input-frames `
--source-rate-hz $rate `
--minimum-effective-fps 11.209069 `
--maximum-completion-p95-ms 125.0 `
--shared-start-ready-file /shared/control/vegetation.ready `
--shared-start-file /shared/control/start.signal `
--frame-ledger /shared/vegetation/frames.jsonl `
--output /shared/vegetation/result.json `
--release-sha256 $ExpectedArtifactSha256 *> $null
Assert-LastExitCode "M49 integrated vegetation creation"
}
& docker start $graphName *> $null
Assert-LastExitCode "M49 integrated graph start"
& docker start $tgsName *> $null
Assert-LastExitCode "M49 integrated TGS start"
if ($VegetationLoadGate) {
& docker start $vegetationName *> $null
Assert-LastExitCode "M49 integrated vegetation start"
}
$graphReady = Join-Path $runOutput "control\graph.ready"
$tgsReady = Join-Path $runOutput "control\tgs.ready"
Wait-SharedReady $graphReady $tgsReady $graphName $tgsName
$vegetationReady = if ($VegetationLoadGate) {
Join-Path $runOutput "control\vegetation.ready"
} else { "" }
Wait-SharedReady $graphReady $tgsReady $vegetationReady $graphName $tgsName $vegetationName
[DateTimeOffset]::UtcNow.ToString("o") | Set-Content -LiteralPath (
Join-Path $runOutput "control\start.signal"
) -Encoding utf8
$telemetryPath = Join-Path $runOutput "container-telemetry.jsonl"
$m49TelemetryPath = if ($VegetationLoadGate) {
Join-Path $runOutput "m49-container-telemetry.jsonl"
} else { $telemetryPath }
while ($true) {
$graphState = Get-Container $graphName
$tgsState = Get-Container $tgsName
$vegetationState = if ($VegetationLoadGate) {
Get-Container $vegetationName
} else { $null }
$running = @()
if ($graphState.State.Running) { $running += $graphName }
if ($tgsState.State.Running) { $running += $tgsName }
if ($VegetationLoadGate -and $vegetationState.State.Running) {
$running += $vegetationName
}
if ((Get-Container $tritonName).State.Running) { $running += $tritonName }
if ($running.Count -gt 0) {
$stats = @((& docker stats --no-stream --format "{{json .}}" @running))
@@ -362,10 +479,12 @@ try {
"tgs"
} elseif ($value.Name -ceq $tritonName) {
"triton"
} elseif ($VegetationLoadGate -and $value.Name -ceq $vegetationName) {
"vegetation"
} else {
throw "Unknown M49 telemetry container"
}
[ordered]@{
$telemetryRow = [ordered]@{
observed_utc = [DateTimeOffset]::UtcNow.ToString("o")
role = $role
name = [string]$value.Name
@@ -373,23 +492,46 @@ try {
memory_usage = [string]$value.MemUsage
memory_percent = [string]$value.MemPerc
pids = [string]$value.PIDs
} | ConvertTo-Json -Compress | Out-File -LiteralPath $telemetryPath -Encoding utf8 -Append
} | ConvertTo-Json -Compress
$telemetryRow | Out-File -LiteralPath $telemetryPath -Encoding utf8 -Append
if ($VegetationLoadGate -and $role -cne "vegetation") {
$telemetryRow | Out-File -LiteralPath $m49TelemetryPath -Encoding utf8 -Append
}
}
}
if (-not $graphState.State.Running -and -not $tgsState.State.Running) { break }
$vegetationStopped = -not $VegetationLoadGate -or -not $vegetationState.State.Running
if (
-not $graphState.State.Running -and
-not $tgsState.State.Running -and
$vegetationStopped
) { break }
Start-Sleep -Seconds 1
}
$graphExit = [int](Get-Container $graphName).State.ExitCode
$tgsExit = [int](Get-Container $tgsName).State.ExitCode
$vegetationExit = if ($VegetationLoadGate) {
[int](Get-Container $vegetationName).State.ExitCode
} else { 0 }
$previousErrorAction = $ErrorActionPreference
$ErrorActionPreference = "Continue"
$graphLogs = & docker logs $graphName 2>&1
$tgsLogs = & docker logs $tgsName 2>&1
$vegetationLogs = if ($VegetationLoadGate) {
& docker logs $vegetationName 2>&1
} else { @() }
$ErrorActionPreference = $previousErrorAction
$graphLogs | Set-Content -LiteralPath (Join-Path $runOutput "graph.log") -Encoding utf8
$tgsLogs | Set-Content -LiteralPath (Join-Path $runOutput "tgs.log") -Encoding utf8
if ($VegetationLoadGate) {
$vegetationLogs | Set-Content -LiteralPath (
Join-Path $runOutput "vegetation.log"
) -Encoding utf8
}
if ($graphExit -ne 0) { throw "M49 integrated graph failed with exit code $graphExit" }
if ($tgsExit -ne 0) { throw "M49 integrated TGS failed with exit code $tgsExit" }
if ($vegetationExit -ne 0) {
throw "M49 integrated vegetation failed with exit code $vegetationExit"
}
& docker run --rm --name $analyzeName --network none --cpus 8 --memory 16g `
--entrypoint python3 `
@@ -401,6 +543,12 @@ try {
--output-root /shared/tgs/evidence
Assert-LastExitCode "M49 integrated TGS evidence analysis"
$m49ResultPath = if ($VegetationLoadGate) {
"/shared/m49-result.json"
} else { "/shared/result.json" }
$dockerM49TelemetryPath = if ($VegetationLoadGate) {
"/shared/m49-container-telemetry.jsonl"
} else { "/shared/container-telemetry.jsonl" }
& docker run --rm --name $evidenceName --network none --cpus 4 --memory 8g `
--entrypoint python3 `
--volume ($dockerRelease + ":/release:ro") `
@@ -411,11 +559,33 @@ try {
--graph-frames /shared/graph/frames.jsonl `
--tgs-result /shared/tgs/evidence/result.json `
--tgs-timing /shared/tgs/tgs-full-timing.tsv `
--telemetry /shared/container-telemetry.jsonl `
--output /shared/result.json `
--telemetry $dockerM49TelemetryPath `
--output $m49ResultPath `
--release-sha256 $ExpectedArtifactSha256
Assert-LastExitCode "M49 integrated evidence gate"
if ($VegetationLoadGate) {
& docker run --rm --name $vegetationEvidenceName --network none --cpus 4 --memory 8g `
--entrypoint python3 `
--volume ($dockerRelease + ":/release:ro") `
--volume ($dockerRun + ":/shared:rw") `
$ParityImageTag /release/build_vegetation_integrated_graph_evidence.py `
--profile /release/lab-v1-vegetation-integrated-shadow-v1.json `
--m49-result /shared/m49-result.json `
--graph-frames /shared/graph/frames.jsonl `
--tgs-timing /shared/tgs/tgs-full-timing.tsv `
--vegetation-result /shared/vegetation/result.json `
--vegetation-frames /shared/vegetation/frames.jsonl `
--telemetry /shared/container-telemetry.jsonl `
--output /shared/result.json `
--release-sha256 $ExpectedArtifactSha256
Assert-LastExitCode "M49 integrated vegetation evidence gate"
}
} finally {
$vegetationFrames = Join-Path $runOutput "vegetation\input-frames"
if (Test-Path -LiteralPath $vegetationFrames -PathType Container) {
Remove-Item -LiteralPath $vegetationFrames -Recurse -Force
}
foreach ($name in $containers) { Remove-ExactContainer $name }
$canonicalAfter = Get-Container "ndc-mission-core-triton"
if (
@@ -432,7 +602,11 @@ if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
}
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
$summary = [ordered]@{
schema_version = "missioncore.m49-tgs-integrated-graph-worker-summary/v1"
schema_version = if ($VegetationLoadGate) {
"missioncore.lab-v1-vegetation-integrated-worker-summary/v1"
} else {
"missioncore.m49-tgs-integrated-graph-worker-summary/v1"
}
worker_id = "worker-006"
run_id = $RunId
code_revision = [string]$releaseDocument.code_revision
@@ -443,6 +617,7 @@ $summary = [ordered]@{
free_memory_gib_before = [math]::Round($freeMemoryGiB, 6)
result_id = [string]$result.result_id
result_status = [string]$result.status
vegetation_load_gate = [bool]$VegetationLoadGate
canonical_triton_id = $canonicalId
canonical_triton_health = "healthy"
gauss_or_playcanvas_action = "none"
experiments/perception/worker/lab_v1_vegetation_goose/run_vegetation_integrated_load.py
+280
View File
@@ -0,0 +1,280 @@
#!/usr/bin/env python3
"""Run source-paced DDRNet beside the frozen M4 graph and TGS shadow."""
from __future__ import annotations
import argparse
import json
import math
import platform
import statistics
import time
from pathlib import Path
from typing import Any
import torch
from PIL import Image
from run_goose_vegetation_benchmark import (
infer,
load_mapping,
load_model,
percentile,
preprocess,
read_json,
sha256,
stable_digest,
validate_contracts,
)
SCHEMA = "missioncore.lab-v1-vegetation-integrated-load/v1"
FRAME_SCHEMA = "missioncore.lab-v1-vegetation-integrated-frame/v1"
FRAME_COUNT = 4_489
AUTHORITY = {
"ground_truth": False,
"candidate_accepted": False,
"camera_semantics_can_clear_rigid_geometry": False,
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
"production_accepted": False,
}
class IntegratedLoadError(RuntimeError):
"""The bounded integrated-load contract is incomplete or changed."""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--config", type=Path, required=True)
parser.add_argument("--policy", type=Path, required=True)
parser.add_argument("--provider-map", type=Path, required=True)
parser.add_argument("--checkpoint", type=Path, required=True)
parser.add_argument("--dataset-root", type=Path, required=True)
parser.add_argument("--frames-root", type=Path, required=True)
parser.add_argument("--source-rate-hz", type=float, required=True)
parser.add_argument("--minimum-effective-fps", type=float, required=True)
parser.add_argument("--maximum-completion-p95-ms", type=float, required=True)
parser.add_argument("--shared-start-ready-file", type=Path, required=True)
parser.add_argument("--shared-start-file", type=Path, required=True)
parser.add_argument("--shared-start-timeout-seconds", type=float, default=600.0)
parser.add_argument("--frame-ledger", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--release-sha256", required=True)
return parser.parse_args()
def wait_for_shared_start(ready_file: Path, start_file: Path, timeout_seconds: float) -> None:
if ready_file.exists():
raise IntegratedLoadError("shared-start ready file already exists")
ready_file.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
ready_file.write_text("ready\n", encoding="utf-8")
deadline = time.monotonic() + timeout_seconds
while not start_file.is_file():
if time.monotonic() >= deadline:
raise IntegratedLoadError("shared-start barrier timed out")
time.sleep(0.01)
def distribution(values: list[float]) -> dict[str, float]:
return {
"mean": round(statistics.fmean(values), 6),
"p50": round(percentile(values, 0.50), 6),
"p95": round(percentile(values, 0.95), 6),
"p99": round(percentile(values, 0.99), 6),
"maximum": round(max(values), 6),
}
def exact_frames(root: Path) -> list[Path]:
if root.is_symlink() or not root.is_dir():
raise IntegratedLoadError("RAVNOVES frame root is unavailable")
frames = sorted(root.glob("frame-*.png"))
expected = [f"frame-{sequence + 1:06d}.png" for sequence in range(FRAME_COUNT)]
if len(frames) != FRAME_COUNT or [frame.name for frame in frames] != expected:
raise IntegratedLoadError("RAVNOVES full-video frame sequence changed")
return frames
def validate_sha256(value: str, label: str) -> None:
if len(value) != 64 or any(character not in "0123456789abcdef" for character in value):
raise IntegratedLoadError(f"{label} SHA-256 is invalid")
def run() -> int:
args = parse_args()
if not torch.cuda.is_available():
raise IntegratedLoadError("CUDA is required for Worker 006 qualification")
if (
not math.isfinite(args.source_rate_hz)
or args.source_rate_hz <= 0
or args.minimum_effective_fps <= 0
or args.maximum_completion_p95_ms <= 0
or args.shared_start_timeout_seconds <= 0
):
raise IntegratedLoadError("integrated-load thresholds must be positive and finite")
validate_sha256(args.release_sha256, "release")
if args.output.exists() or args.frame_ledger.exists():
raise IntegratedLoadError("integrated-load output already exists")
config = read_json(args.config, "benchmark config")
policy = read_json(args.policy, "mission policy")
provider_map = read_json(args.provider_map, "provider map")
candidate = validate_contracts(config, policy, provider_map, "ddrnet")
if args.checkpoint.is_symlink() or not args.checkpoint.is_file():
raise IntegratedLoadError("DDRNet checkpoint is unavailable")
if args.checkpoint.stat().st_size != candidate["checkpoint_size_bytes"]:
raise IntegratedLoadError("DDRNet checkpoint size changed")
checkpoint_sha256 = sha256(args.checkpoint)
if checkpoint_sha256 != candidate["checkpoint_sha256"]:
raise IntegratedLoadError("DDRNet checkpoint digest changed")
mapping_path = args.dataset_root / config["dataset"]["mapping_relative_path"]
load_mapping(mapping_path, config["dataset"]["mapping_sha256"])
frames = exact_frames(args.frames_root)
torch.cuda.empty_cache()
model, model_name, architecture_failures = load_model("ddrnet", args.checkpoint)
with Image.open(frames[0]) as image:
warmup_tensor, _ = preprocess(image.convert("RGB"))
warmup_latencies_ms = [infer(model, warmup_tensor)[1] for _ in range(3)]
torch.cuda.reset_peak_memory_stats()
wait_for_shared_start(
args.shared_start_ready_file,
args.shared_start_file,
args.shared_start_timeout_seconds,
)
interval_ns = 1_000_000_000.0 / args.source_rate_hz
start_ns = time.monotonic_ns()
started_utc_ns = time.time_ns()
completion_ages_ms: list[float] = []
stage_latencies_ms: list[float] = []
inference_latencies_ms: list[float] = []
late_deadline_count = 0
args.frame_ledger.parent.mkdir(parents=True, exist_ok=True)
with args.frame_ledger.open("x", encoding="utf-8") as ledger:
for sequence, frame in enumerate(frames):
scheduled_ns = start_ns + round(sequence * interval_ns)
remaining_ns = scheduled_ns - time.monotonic_ns()
if remaining_ns > 0:
time.sleep(remaining_ns / 1_000_000_000.0)
admitted_ns = time.monotonic_ns()
with Image.open(frame) as image:
source = image.convert("RGB")
if source.size != (
config["ravnoves"]["expected_width"],
config["ravnoves"]["expected_height"],
):
raise IntegratedLoadError("RAVNOVES video frame dimensions changed")
tensor, _ = preprocess(source)
_, inference_ms = infer(model, tensor)
completed_ns = time.monotonic_ns()
completion_age_ms = (completed_ns - scheduled_ns) / 1_000_000.0
stage_ms = (completed_ns - admitted_ns) / 1_000_000.0
completion_ages_ms.append(completion_age_ms)
stage_latencies_ms.append(stage_ms)
inference_latencies_ms.append(inference_ms)
if sequence + 1 < FRAME_COUNT and completed_ns > start_ns + round(
(sequence + 1) * interval_ns
):
late_deadline_count += 1
row = {
"schema_version": FRAME_SCHEMA,
"sequence": sequence,
"frame_name": frame.name,
"scheduled_monotonic_ns": scheduled_ns,
"admitted_monotonic_ns": admitted_ns,
"completed_monotonic_ns": completed_ns,
"completion_age_ms": round(completion_age_ms, 6),
"stage_ms": round(stage_ms, 6),
"inference_ms": round(inference_ms, 6),
}
ledger.write(json.dumps(row, sort_keys=True, separators=(",", ":")) + "\n")
if sequence % 64 == 0:
ledger.flush()
completed_ns = time.monotonic_ns()
wall_seconds = (completed_ns - start_ns) / 1_000_000_000.0
effective_fps = FRAME_COUNT / wall_seconds
completion = distribution(completion_ages_ms)
checks = {
"all_frames_accounted": len(completion_ages_ms) == FRAME_COUNT,
"minimum_effective_fps": effective_fps >= args.minimum_effective_fps,
"maximum_completion_p95_ms": completion["p95"]
<= args.maximum_completion_p95_ms,
"zero_capacity_drops": len(completion_ages_ms) == FRAME_COUNT,
"authority_remains_false": all(value is False for value in AUTHORITY.values()),
}
result: dict[str, Any] = {
"schema_version": SCHEMA,
"worker_id": "worker-006",
"source": {
"source_id": config["ravnoves"]["source_id"],
"frame_count": FRAME_COUNT,
"requested_source_rate_hz": args.source_rate_hz,
"raw_fisheye_immutable": True,
"ground_truth_available": False,
},
"candidate": {
"candidate_id": candidate["candidate_id"],
"candidate_key": "ddrnet",
"loaded_model_name": model_name,
"architecture_probe_failures": architecture_failures,
"checkpoint_size_bytes": args.checkpoint.stat().st_size,
"checkpoint_sha256": checkpoint_sha256,
},
"execution": {
"run_mode": "source-paced-integrated-shadow/v1",
"started_utc_ns": started_utc_ns,
"wall_seconds": round(wall_seconds, 6),
"effective_fps": round(effective_fps, 6),
"frame_count": FRAME_COUNT,
"capacity_drop_count": 0,
"deadline_miss_count": late_deadline_count,
"frame_ledger": {
"path": args.frame_ledger.name,
"rows": FRAME_COUNT,
"sha256": sha256(args.frame_ledger),
},
},
"timing": {
"prewarm_inference_count": len(warmup_latencies_ms),
"prewarm_latency_ms_first": round(warmup_latencies_ms[0], 6),
"prewarm_latency_ms_last": round(warmup_latencies_ms[-1], 6),
"completion_age_ms": completion,
"stage_ms": distribution(stage_latencies_ms),
"inference_ms": distribution(inference_latencies_ms),
},
"resource": {
"gpu_name": torch.cuda.get_device_name(0),
"peak_allocated_vram_bytes": int(torch.cuda.max_memory_allocated()),
"peak_reserved_vram_bytes": int(torch.cuda.max_memory_reserved()),
"torch_version": torch.__version__,
"cuda_runtime_version": torch.version.cuda,
"python_version": platform.python_version(),
},
"identity": {
"release_sha256": args.release_sha256,
"config_sha256": sha256(args.config),
"policy_sha256": sha256(args.policy),
"provider_map_sha256": sha256(args.provider_map),
"runner_sha256": sha256(Path(__file__)),
},
"predeclared_thresholds": {
"minimum_effective_fps": args.minimum_effective_fps,
"maximum_completion_p95_ms": args.maximum_completion_p95_ms,
"capacity_drop_count_max": 0,
},
"checks": checks,
"integrated_load_gate_passed": all(checks.values()),
"authority": AUTHORITY,
}
result["result_id"] = f"lab-v1-vegetation-integrated-{stable_digest(result)}"
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps({"result_id": result["result_id"], "passed": all(checks.values())}))
return 0 if all(checks.values()) else 2
if __name__ == "__main__":
raise SystemExit(run())
experiments/perception/worker/m49_t3_travel/build_vegetation_integrated_graph_evidence.py
+366
View File
@@ -0,0 +1,366 @@
#!/usr/bin/env python3
"""Seal the synchronized RF-DETR, TGS and DDRNet Worker 006 load gate."""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
import math
import re
from collections import defaultdict
from pathlib import Path
from typing import Any
import numpy as np
PROFILE_SCHEMA = "missioncore.lab-v1-vegetation-integrated-shadow-profile/v1"
M49_SCHEMA = "missioncore.m49-tgs-integrated-graph-shadow-result/v1"
VEGETATION_SCHEMA = "missioncore.lab-v1-vegetation-integrated-load/v1"
RESULT_SCHEMA = "missioncore.lab-v1-vegetation-integrated-shadow-result/v1"
FRAME_COUNT = 4_489
class VegetationIntegratedError(RuntimeError):
"""The synchronized three-layer load evidence is incomplete."""
def canonical_json(value: object) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def load_json(path: Path, label: str) -> dict[str, Any]:
try:
value = json.loads(path.read_text(encoding="utf-8-sig"))
except (OSError, json.JSONDecodeError) as exc:
raise VegetationIntegratedError(f"{label} is unreadable") from exc
if not isinstance(value, dict):
raise VegetationIntegratedError(f"{label} is not an object")
return value
def distribution(values: list[float]) -> dict[str, float]:
if not values:
raise VegetationIntegratedError("timing distribution is empty")
array = np.asarray(values, dtype=np.float64)
return {
"mean": round(float(array.mean()), 6),
"p50": round(float(np.percentile(array, 50)), 6),
"p95": round(float(np.percentile(array, 95)), 6),
"p99": round(float(np.percentile(array, 99)), 6),
"maximum": round(float(array.max()), 6),
}
def graph_completion_ages(path: Path) -> list[float]:
values: list[float] = []
with path.open("r", encoding="utf-8") as stream:
for expected, line in enumerate(stream):
row = json.loads(line)
if row.get("source_envelope", {}).get("sequence") != expected:
raise VegetationIntegratedError("graph frame sequence changed")
age = row.get("completion_age_ns")
if not isinstance(age, int) or age < 0:
raise VegetationIntegratedError("graph completion age is invalid")
values.append(age / 1_000_000.0)
if len(values) != FRAME_COUNT:
raise VegetationIntegratedError("graph frame ledger is incomplete")
return values
def tgs_completion_ages(path: Path) -> list[float]:
values: list[float] = []
with path.open("r", encoding="utf-8", newline="") as stream:
for expected, row in enumerate(csv.DictReader(stream, delimiter="\t")):
if int(row["timeline_frame_index"]) != expected:
raise VegetationIntegratedError("TGS timing sequence changed")
age = float(row["completion_age_ms"])
if not math.isfinite(age) or age < 0:
raise VegetationIntegratedError("TGS completion age is invalid")
values.append(age)
if len(values) != FRAME_COUNT:
raise VegetationIntegratedError("TGS timing ledger is incomplete")
return values
def vegetation_completion_ages(path: Path) -> list[float]:
values: list[float] = []
with path.open("r", encoding="utf-8") as stream:
for expected, line in enumerate(stream):
row = json.loads(line)
if row.get("schema_version") != "missioncore.lab-v1-vegetation-integrated-frame/v1":
raise VegetationIntegratedError("vegetation frame schema changed")
if row.get("sequence") != expected:
raise VegetationIntegratedError("vegetation frame sequence changed")
age = row.get("completion_age_ms")
if not isinstance(age, (int, float)) or not math.isfinite(age) or age < 0:
raise VegetationIntegratedError("vegetation completion age is invalid")
values.append(float(age))
if len(values) != FRAME_COUNT:
raise VegetationIntegratedError("vegetation frame ledger is incomplete")
return values
_SIZE = re.compile(r"^\s*([0-9.]+)\s*([kmgt]?i?b)\s*$", re.IGNORECASE)
def size_mib(value: str) -> float:
match = _SIZE.fullmatch(value)
if match is None:
raise VegetationIntegratedError("container memory telemetry is invalid")
number = float(match.group(1))
scale = {
"b": 1.0 / (1024.0 * 1024.0),
"kb": 1.0 / 1024.0,
"kib": 1.0 / 1024.0,
"mb": 1.0,
"mib": 1.0,
"gb": 1024.0,
"gib": 1024.0,
"tb": 1024.0 * 1024.0,
"tib": 1024.0 * 1024.0,
}[match.group(2).lower()]
return number * scale
def host_telemetry(path: Path) -> dict[str, object]:
roles = ("graph", "tgs", "triton", "vegetation")
samples: dict[str, list[dict[str, float]]] = defaultdict(list)
with path.open("r", encoding="utf-8-sig") as stream:
for line in stream:
row = json.loads(line)
role = row.get("role")
if role not in roles:
raise VegetationIntegratedError("container telemetry role changed")
cpu = row.get("cpu_percent")
memory = row.get("memory_usage")
memory_percent = row.get("memory_percent")
if not all(isinstance(value, str) for value in (cpu, memory, memory_percent)):
raise VegetationIntegratedError("container telemetry row is incomplete")
assert isinstance(cpu, str) and isinstance(memory, str)
assert isinstance(memory_percent, str)
samples[role].append(
{
"cpu_percent": float(cpu.rstrip("%")),
"memory_used_mib": size_mib(memory.split("/", 1)[0].strip()),
"memory_percent": float(memory_percent.rstrip("%")),
}
)
if any(not samples[role] for role in roles):
raise VegetationIntegratedError("container telemetry does not cover every runtime role")
return {
role: {
"sample_count": len(samples[role]),
"cpu_percent": distribution([row["cpu_percent"] for row in samples[role]]),
"memory_used_mib": distribution(
[row["memory_used_mib"] for row in samples[role]]
),
"memory_percent": distribution(
[row["memory_percent"] for row in samples[role]]
),
}
for role in roles
}
def build(
*,
profile_path: Path,
m49_result_path: Path,
graph_frames_path: Path,
tgs_timing_path: Path,
vegetation_result_path: Path,
vegetation_frames_path: Path,
telemetry_path: Path,
output_path: Path,
release_sha256: str,
) -> dict[str, object]:
if output_path.exists():
raise VegetationIntegratedError("integrated vegetation result already exists")
if len(release_sha256) != 64 or any(
character not in "0123456789abcdef" for character in release_sha256
):
raise VegetationIntegratedError("release SHA-256 is invalid")
profile = load_json(profile_path, "integrated vegetation profile")
m49 = load_json(m49_result_path, "M49 integrated result")
vegetation = load_json(vegetation_result_path, "vegetation load result")
if profile.get("schema_version") != PROFILE_SCHEMA:
raise VegetationIntegratedError("integrated vegetation profile schema changed")
if m49.get("schema_version") != M49_SCHEMA:
raise VegetationIntegratedError("M49 integrated result schema changed")
if vegetation.get("schema_version") != VEGETATION_SCHEMA:
raise VegetationIntegratedError("vegetation load result schema changed")
graph_ages = graph_completion_ages(graph_frames_path)
tgs_ages = tgs_completion_ages(tgs_timing_path)
vegetation_ages = vegetation_completion_ages(vegetation_frames_path)
combined_ages = [
max(graph, tgs, semantic)
for graph, tgs, semantic in zip(
graph_ages, tgs_ages, vegetation_ages, strict=True
)
]
combined = distribution(combined_ages)
telemetry = host_telemetry(telemetry_path)
acceptance = profile["acceptance"]
vegetation_execution = vegetation.get("execution", {})
vegetation_timing = vegetation.get("timing", {})
vegetation_identity = vegetation.get("identity", {})
vegetation_candidate = vegetation.get("candidate", {})
m49_performance = m49.get("performance", {})
m49_accounting = m49.get("accounting", {})
checks = {
"base_m49_runtime_passed": (
m49.get("status") == "passed"
and m49.get("integrated_runtime_gate_passed") is True
and m49.get("identity", {}).get("profile_sha256")
== profile["stages"]["m49_graph_tgs"]["profile_sha256"]
),
"vegetation_identity_frozen": (
vegetation_candidate.get("candidate_key") == "ddrnet"
and vegetation_candidate.get("checkpoint_sha256")
== profile["stages"]["vegetation"]["checkpoint_sha256"]
and vegetation_identity.get("config_sha256")
== profile["stages"]["vegetation"]["config_sha256"]
and vegetation_identity.get("policy_sha256")
== profile["stages"]["vegetation"]["policy_sha256"]
and vegetation_identity.get("provider_map_sha256")
== profile["stages"]["vegetation"]["provider_map_sha256"]
),
"requested_source_rate_preserved": (
vegetation.get("source", {}).get("requested_source_rate_hz")
== profile["source"]["requested_source_rate_hz"]
),
"exact_three_layer_sequence_join": len(combined_ages) == FRAME_COUNT,
"all_graph_frames_delivered": (
m49_accounting.get("graph_admitted") == FRAME_COUNT
and m49_accounting.get("graph_delivered") == FRAME_COUNT
),
"all_tgs_frames_accounted": m49_accounting.get("tgs_timeline_frames")
== FRAME_COUNT,
"all_vegetation_frames_accounted": vegetation_execution.get("frame_count")
== FRAME_COUNT,
"minimum_graph_world_state_fps": float(
m49_performance.get("effective_world_state_fps", 0.0)
)
>= float(acceptance["minimum_graph_world_state_fps"]),
"minimum_vegetation_fps": float(vegetation_execution.get("effective_fps", 0.0))
>= float(acceptance["minimum_vegetation_fps"]),
"maximum_vegetation_completion_p95_ms": float(
vegetation_timing.get("completion_age_ms", {}).get("p95", math.inf)
)
<= float(acceptance["maximum_vegetation_completion_p95_ms"]),
"maximum_combined_output_age_p99_ms": combined["p99"]
<= float(acceptance["maximum_combined_output_age_p99_ms"]),
"zero_capacity_drops": (
int(m49_accounting.get("tgs_capacity_drops", -1)) == 0
and int(vegetation_execution.get("capacity_drop_count", -1)) == 0
),
"host_resource_telemetry_complete": all(
telemetry[role]["sample_count"] > 0
for role in ("graph", "tgs", "triton", "vegetation")
),
"authority_remains_false": (
all(value is False for value in profile["authority"].values())
and all(value is False for value in vegetation.get("authority", {}).values())
),
}
files = {
label: {"bytes": path.stat().st_size, "sha256": sha256_file(path)}
for label, path in (
("m49-result.json", m49_result_path),
("graph-frames.jsonl", graph_frames_path),
("tgs-timing.tsv", tgs_timing_path),
("vegetation-result.json", vegetation_result_path),
("vegetation-frames.jsonl", vegetation_frames_path),
("container-telemetry.jsonl", telemetry_path),
)
}
document: dict[str, object] = {
"schema_version": RESULT_SCHEMA,
"profile_id": profile["profile_id"],
"status": "passed" if all(checks.values()) else "failed",
"source": {
"source_id": profile["source"]["source_id"],
"requested_source_rate_hz": profile["source"]["requested_source_rate_hz"],
"joined_frame_count": len(combined_ages),
"ground_truth_available": False,
},
"identity": {
"release_sha256": release_sha256,
"profile_sha256": sha256_file(profile_path),
"m49_result_id": m49.get("result_id"),
"vegetation_result_id": vegetation.get("result_id"),
},
"performance": {
"graph_tgs": m49_performance,
"vegetation": {
"effective_fps": vegetation_execution.get("effective_fps"),
"completion_age_ms": vegetation_timing.get("completion_age_ms"),
"stage_ms": vegetation_timing.get("stage_ms"),
"inference_ms": vegetation_timing.get("inference_ms"),
"resource": vegetation.get("resource"),
},
"three_layer_output_age_ms": combined,
"host_containers": telemetry,
},
"accounting": {
"graph_frames": m49_accounting.get("graph_delivered"),
"tgs_frames": m49_accounting.get("tgs_timeline_frames"),
"vegetation_frames": vegetation_execution.get("frame_count"),
"capacity_drop_count": int(m49_accounting.get("tgs_capacity_drops", 0))
+ int(vegetation_execution.get("capacity_drop_count", 0)),
},
"checks": checks,
"integrated_runtime_gate_passed": all(checks.values()),
"visual_quality_accepted": False,
"route_truth_available": False,
"production_accepted": False,
"authority": profile["authority"],
"files": files,
}
identity = hashlib.sha256(canonical_json(document)).hexdigest()
document["result_id"] = f"lab-v1-vegetation-integrated-shadow-{identity}"
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8")
return document
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--profile", type=Path, required=True)
parser.add_argument("--m49-result", type=Path, required=True)
parser.add_argument("--graph-frames", type=Path, required=True)
parser.add_argument("--tgs-timing", type=Path, required=True)
parser.add_argument("--vegetation-result", type=Path, required=True)
parser.add_argument("--vegetation-frames", type=Path, required=True)
parser.add_argument("--telemetry", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--release-sha256", required=True)
arguments = parser.parse_args()
result = build(
profile_path=arguments.profile,
m49_result_path=arguments.m49_result,
graph_frames_path=arguments.graph_frames,
tgs_timing_path=arguments.tgs_timing,
vegetation_result_path=arguments.vegetation_result,
vegetation_frames_path=arguments.vegetation_frames,
telemetry_path=arguments.telemetry,
output_path=arguments.output,
release_sha256=arguments.release_sha256,
)
print(json.dumps({"result_id": result["result_id"], "status": result["status"]}))
return 0 if result["status"] == "passed" else 2
if __name__ == "__main__":
raise SystemExit(main())