feat(perception): qualify inline temporal stability
This commit is contained in:
DCCONSTRUCTIONS
@@ -9,6 +9,7 @@ param(
|
||||
[Parameter(Mandatory = $true)] [string]$ValidFovRoot,
|
||||
[Parameter(Mandatory = $true)] [string]$ProjectionPackRoot,
|
||||
[Parameter(Mandatory = $true)] [string]$PackageRoot,
|
||||
[string]$StabilityProfilePath = "",
|
||||
[switch]$PreflightOnly,
|
||||
[switch]$PersistentService,
|
||||
[switch]$TokenStdin,
|
||||
@@ -84,6 +85,10 @@ $liveProfile = Resolve-DFile $LiveProfilePath "LAB E15 live profile"
|
||||
$e14Profile = Resolve-DFile $E14ProfilePath "Accepted E14 profile"
|
||||
$detectorProfile = Resolve-DFile $DetectorProfilePath "Detector profile"
|
||||
$semanticProfile = Resolve-DFile $SemanticProfilePath "Semantic profile"
|
||||
$stabilityProfile = $null
|
||||
if ($StabilityProfilePath) {
|
||||
$stabilityProfile = Resolve-DFile $StabilityProfilePath "LAB E23 stability profile"
|
||||
}
|
||||
$validFov = Resolve-DDirectory $ValidFovRoot "Valid-FOV root"
|
||||
$projectionPack = Resolve-DDirectory $ProjectionPackRoot "E15 projection pack"
|
||||
$package = Resolve-DDirectory $PackageRoot "Mission Core package root"
|
||||
@@ -97,6 +102,9 @@ foreach ($path in @($liveProfile, $e14Profile, $detectorProfile, $semanticProfil
|
||||
throw "Runner and profiles must share one immutable mount"
|
||||
}
|
||||
}
|
||||
if ($stabilityProfile -and (Split-Path $stabilityProfile -Parent) -ne $runnerRoot) {
|
||||
throw "Runner and LAB E23 stability profile must share one immutable mount"
|
||||
}
|
||||
foreach ($dependency in @(
|
||||
"e10_fusion_runtime.py",
|
||||
"e15_shadow_runtime.py",
|
||||
@@ -115,6 +123,9 @@ foreach ($dependency in @(
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $package "k1link\compute\live_perception.py") -PathType Leaf)) {
|
||||
throw "Mission Core package mount lacks live perception synchronization"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $package "k1link\compute\inline_temporal.py") -PathType Leaf)) {
|
||||
throw "Mission Core package mount lacks inline temporal state"
|
||||
}
|
||||
|
||||
$job = Get-Content -LiteralPath (Join-Path $jobDirectory "job.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
@@ -130,6 +141,17 @@ if (
|
||||
[bool]$live.authority.navigation_or_safety_accepted -or
|
||||
$live.transport.pyav_version -ne "18.0.0"
|
||||
) { throw "LAB E15 replay-shadow authority contract changed" }
|
||||
if ($stabilityProfile) {
|
||||
$stability = Get-Content -LiteralPath $stabilityProfile -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$stability.schema_version -ne "missioncore.e23-inline-temporal-profile/v1" -or
|
||||
$stability.mode -ne "inline-shadow-qualification" -or
|
||||
$stability.stage -ne "warm-worker-after-fusion-before-result-publication" -or
|
||||
$stability.source.calibration_sha256 -ne $live.source.calibration_sha256 -or
|
||||
[bool]$stability.authority.commands_enabled -or
|
||||
[bool]$stability.authority.navigation_or_safety_accepted
|
||||
) { throw "LAB E23 inline temporal authority contract changed" }
|
||||
}
|
||||
$projection = Get-Content -LiteralPath (Join-Path $projectionPack "manifest.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$projection.schema_version -ne "missioncore.e15-live-projection-pack/v1" -or
|
||||
@@ -137,6 +159,12 @@ if (
|
||||
$projection.identity.calibration_slot -ne "camera_1" -or
|
||||
$projection.identity.calibration_sha256 -ne $live.source.calibration_sha256
|
||||
) { throw "LAB E15 projection pack binding changed" }
|
||||
$packageManifest = Get-Content -LiteralPath (Join-Path $package "manifest.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$packageManifest.schema_version -ne "missioncore.e15-worker-package/v1" -or
|
||||
$packageManifest.package_id -ne (Split-Path $package -Leaf) -or
|
||||
$packageManifest.identity_sha256 -notmatch "^[a-f0-9]{64}$"
|
||||
) { throw "LAB E15 worker package manifest changed" }
|
||||
$mediaManifest = Get-Content -LiteralPath (Join-Path $mediaRuntime "manifest.json") -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$mediaManifest.schema_version -ne "missioncore.e15-media-runtime/v1" -or
|
||||
@@ -165,6 +193,7 @@ $liveProfileName = Split-Path $liveProfile -Leaf
|
||||
$e14ProfileName = Split-Path $e14Profile -Leaf
|
||||
$detectorProfileName = Split-Path $detectorProfile -Leaf
|
||||
$semanticProfileName = Split-Path $semanticProfile -Leaf
|
||||
$stabilityProfileName = if ($stabilityProfile) { Split-Path $stabilityProfile -Leaf } else { $null }
|
||||
$projectionMount = "/" + (Split-Path $projectionPack -Leaf)
|
||||
$packageMount = "/" + (Split-Path $package -Leaf)
|
||||
$orchestratorSha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
@@ -200,6 +229,11 @@ $commonRunnerArgs = @(
|
||||
"--environment", "/environment",
|
||||
"--worker-package", $packageMount
|
||||
)
|
||||
if ($stabilityProfileName) {
|
||||
$commonRunnerArgs += @(
|
||||
"--stability-profile", ("/runner/{0}" -f $stabilityProfileName)
|
||||
)
|
||||
}
|
||||
|
||||
if ($PersistentService) {
|
||||
if ($PreflightOnly -or $TokenStdin) {
|
||||
@@ -214,6 +248,14 @@ if ($PersistentService) {
|
||||
throw "Persistent output root must be a direct child of its guarded D: parent"
|
||||
}
|
||||
$runnerSha256 = (Get-FileHash -LiteralPath $runner -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$liveProfileSha256 = (Get-FileHash -LiteralPath $liveProfile -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$e14ProfileSha256 = (Get-FileHash -LiteralPath $e14Profile -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$detectorProfileSha256 = (Get-FileHash -LiteralPath $detectorProfile -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$semanticProfileSha256 = (Get-FileHash -LiteralPath $semanticProfile -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$stabilityProfileSha256 = if ($stabilityProfile) {
|
||||
(Get-FileHash -LiteralPath $stabilityProfile -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
else { "none" }
|
||||
$existingContainerId = docker ps -a --filter ("name=^{0}$" -f $PersistentContainer) --format "{{.ID}}"
|
||||
Assert-LastExitCode "Persistent worker container lookup"
|
||||
if ($existingContainerId) {
|
||||
@@ -221,7 +263,14 @@ if ($PersistentService) {
|
||||
Assert-LastExitCode "Persistent worker label inspection"
|
||||
if (
|
||||
$labels.'missioncore.role' -ne "perception-persistent-worker" -or
|
||||
$labels.'missioncore.runner.sha256' -ne $runnerSha256
|
||||
$labels.'missioncore.runner.sha256' -ne $runnerSha256 -or
|
||||
$labels.'missioncore.live.sha256' -ne $liveProfileSha256 -or
|
||||
$labels.'missioncore.e14.sha256' -ne $e14ProfileSha256 -or
|
||||
$labels.'missioncore.detector.sha256' -ne $detectorProfileSha256 -or
|
||||
$labels.'missioncore.semantic.sha256' -ne $semanticProfileSha256 -or
|
||||
$labels.'missioncore.stability.sha256' -ne $stabilityProfileSha256 -or
|
||||
$labels.'missioncore.worker-package.identity' -ne $packageManifest.identity_sha256 -or
|
||||
$labels.'missioncore.projection.identity' -ne $projection.identity_sha256
|
||||
) { throw "Existing persistent worker has a different immutable identity" }
|
||||
$running = docker inspect --format "{{.State.Running}}" $PersistentContainer
|
||||
Assert-LastExitCode "Persistent worker state inspection"
|
||||
@@ -258,6 +307,13 @@ if ($PersistentService) {
|
||||
"run", "--detach", "--name", $PersistentContainer,
|
||||
"--label", "missioncore.role=perception-persistent-worker",
|
||||
"--label", ("missioncore.runner.sha256={0}" -f $runnerSha256),
|
||||
"--label", ("missioncore.live.sha256={0}" -f $liveProfileSha256),
|
||||
"--label", ("missioncore.e14.sha256={0}" -f $e14ProfileSha256),
|
||||
"--label", ("missioncore.detector.sha256={0}" -f $detectorProfileSha256),
|
||||
"--label", ("missioncore.semantic.sha256={0}" -f $semanticProfileSha256),
|
||||
"--label", ("missioncore.stability.sha256={0}" -f $stabilityProfileSha256),
|
||||
"--label", ("missioncore.worker-package.identity={0}" -f $packageManifest.identity_sha256),
|
||||
"--label", ("missioncore.projection.identity={0}" -f $projection.identity_sha256),
|
||||
"--gpus", "all", "--network", ("container:{0}" -f $TritonContainer),
|
||||
"--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL",
|
||||
"--pids-limit", "512", "--shm-size", "4g",
|
||||
|
||||
@@ -212,6 +212,7 @@ def semantic_worker(
|
||||
completed: list[SemanticResult],
|
||||
failures: list[BaseException],
|
||||
stop_after_results: int | None,
|
||||
transform_target: Any | None = None,
|
||||
) -> None:
|
||||
try:
|
||||
while (envelope := queue.take()) is not None:
|
||||
@@ -225,6 +226,14 @@ def semantic_worker(
|
||||
raise RuntimeError("LAB E10 EoMT emitted an unknown category")
|
||||
target = target_lut[semantic].copy()
|
||||
target[~valid_mask] = 0
|
||||
if transform_target is not None:
|
||||
target = transform_target(target)
|
||||
if (
|
||||
not isinstance(target, np.ndarray)
|
||||
or target.dtype != np.uint8
|
||||
or target.shape != valid_mask.shape
|
||||
):
|
||||
raise RuntimeError("semantic target transform returned an invalid mask")
|
||||
for name, value in measured.items():
|
||||
latency[name].append(float(value))
|
||||
finished = time.perf_counter()
|
||||
|
||||
@@ -19,7 +19,7 @@ import struct
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections import Counter, deque
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
@@ -83,6 +83,15 @@ from run_recorded_perception_epoch import (
|
||||
_write_json,
|
||||
)
|
||||
|
||||
from k1link.compute.inline_temporal import (
|
||||
PIPELINE_ID as INLINE_TEMPORAL_PIPELINE_ID,
|
||||
)
|
||||
from k1link.compute.inline_temporal import (
|
||||
StreamingSemanticStabilizer,
|
||||
TemporalStabilizer,
|
||||
read_inline_profile,
|
||||
stabilize_world_state,
|
||||
)
|
||||
from k1link.compute.live_perception import (
|
||||
LIVE_RESULT_MAX_PAYLOAD_BYTES,
|
||||
LiveSensorSynchronizer,
|
||||
@@ -119,6 +128,7 @@ def arguments() -> argparse.Namespace:
|
||||
command.add_argument("--cache", type=Path, required=True)
|
||||
command.add_argument("--environment", type=Path, required=True)
|
||||
command.add_argument("--worker-package", type=Path, required=True)
|
||||
command.add_argument("--stability-profile", type=Path)
|
||||
if name in {"run", "serve"}:
|
||||
command.add_argument("--host", default="host.docker.internal")
|
||||
command.add_argument("--port", type=int, default=18012)
|
||||
@@ -172,8 +182,7 @@ def read_live_profile(path: Path) -> tuple[dict[str, Any], str]:
|
||||
or source.get("source_id") != "sensor.camera.right"
|
||||
or source.get("resolution") != [800, 600]
|
||||
or source.get("calibration_slot") != "camera_1"
|
||||
or authority
|
||||
!= {"commands_enabled": False, "navigation_or_safety_accepted": False}
|
||||
or authority != {"commands_enabled": False, "navigation_or_safety_accepted": False}
|
||||
or transport.get("wire_schema") != "missioncore.live-perception-wire/v1"
|
||||
or transport.get("camera_media") != "persistent-fmp4-pyav"
|
||||
):
|
||||
@@ -190,9 +199,7 @@ def read_live_profile(path: Path) -> tuple[dict[str, Any], str]:
|
||||
if (
|
||||
not 100 <= float(scheduling.get("semantic_ttl_ms", 0)) <= 5000
|
||||
or not 0 <= float(scheduling.get("sensor_wait_ms", -1)) <= 100
|
||||
or not 1_048_576
|
||||
<= int(transport.get("maximum_media_buffer_bytes", 0))
|
||||
<= 64 * 1024 * 1024
|
||||
or not 1_048_576 <= int(transport.get("maximum_media_buffer_bytes", 0)) <= 64 * 1024 * 1024
|
||||
or not 2 <= int(transport.get("camera_metadata_capacity", 0)) <= 128
|
||||
or not 1 <= int(temporal.get("buffer_capacity_per_modality", 0)) <= 256
|
||||
or not 0.1 <= float(temporal.get("retention_seconds", 0)) <= 30
|
||||
@@ -294,14 +301,13 @@ def read_worker_package(root: Path) -> dict[str, Any]:
|
||||
manifest.get("schema_version") != WORKER_PACKAGE_SCHEMA
|
||||
or not isinstance(identity, dict)
|
||||
or identity.get("schema_version") != WORKER_PACKAGE_SCHEMA
|
||||
or identity.get("classification")
|
||||
!= "minimal-live-worker-import-projection"
|
||||
or identity.get("classification") != "minimal-live-worker-import-projection"
|
||||
or not isinstance(identity_sha256, str)
|
||||
or hashlib.sha256(canonical_json(identity)).hexdigest() != identity_sha256
|
||||
or manifest.get("package_id") != f"e15-worker-package-{identity_sha256}"
|
||||
or resolved.name != manifest.get("package_id")
|
||||
or not isinstance(artifacts, list)
|
||||
or len(artifacts) != 11
|
||||
or len(artifacts) != 12
|
||||
):
|
||||
raise RuntimeError("LAB E15 worker package identity is invalid")
|
||||
expected = set()
|
||||
@@ -321,9 +327,7 @@ def read_worker_package(root: Path) -> dict[str, Any]:
|
||||
raise RuntimeError("LAB E15 worker package artifact changed")
|
||||
expected.add(relative)
|
||||
actual = {
|
||||
path.relative_to(resolved).as_posix()
|
||||
for path in resolved.rglob("*")
|
||||
if path.is_file()
|
||||
path.relative_to(resolved).as_posix() for path in resolved.rglob("*") if path.is_file()
|
||||
}
|
||||
if actual != expected | {"manifest.json"}:
|
||||
raise RuntimeError("LAB E15 worker package file set changed")
|
||||
@@ -386,7 +390,7 @@ class _RuntimeTelemetry:
|
||||
self._started = time.perf_counter()
|
||||
self._previous_wall = self._started
|
||||
self._previous_cpu = time.process_time()
|
||||
self.samples: list[dict[str, Any]] = []
|
||||
self.samples: deque[dict[str, Any]] = deque(maxlen=4096)
|
||||
|
||||
def __enter__(self) -> _RuntimeTelemetry:
|
||||
self._thread.start()
|
||||
@@ -498,6 +502,7 @@ class _RuntimeTelemetry:
|
||||
self._sample()
|
||||
|
||||
def summary(self) -> dict[str, Any]:
|
||||
samples = list(self.samples)
|
||||
numeric_fields = (
|
||||
"process_cpu_percent",
|
||||
"process_rss_mib",
|
||||
@@ -506,25 +511,26 @@ class _RuntimeTelemetry:
|
||||
"cgroup_memory_current_mib",
|
||||
)
|
||||
summary: dict[str, Any] = {
|
||||
"sample_count": len(self.samples),
|
||||
"sample_count": len(samples),
|
||||
"sample_capacity": self.samples.maxlen,
|
||||
"interval_seconds": self._interval,
|
||||
}
|
||||
for field in numeric_fields:
|
||||
values = [
|
||||
float(sample[field])
|
||||
for sample in self.samples
|
||||
for sample in samples
|
||||
if isinstance(sample.get(field), int | float)
|
||||
]
|
||||
summary[field] = _percentiles(values)
|
||||
if self.samples:
|
||||
quarter = max(1, len(self.samples) // 4)
|
||||
early = [float(value["process_rss_mib"]) for value in self.samples[:quarter]]
|
||||
late = [float(value["process_rss_mib"]) for value in self.samples[-quarter:]]
|
||||
if samples:
|
||||
quarter = max(1, len(samples) // 4)
|
||||
early = [float(value["process_rss_mib"]) for value in samples[:quarter]]
|
||||
late = [float(value["process_rss_mib"]) for value in samples[-quarter:]]
|
||||
summary["rss_growth_mib"] = round(
|
||||
float(_percentiles(late)["p95"]) - float(_percentiles(early)["p95"]),
|
||||
6,
|
||||
)
|
||||
summary["final_queues"] = self.samples[-1]["queues"]
|
||||
summary["final_queues"] = samples[-1]["queues"]
|
||||
else:
|
||||
summary["rss_growth_mib"] = 0.0
|
||||
summary["final_queues"] = {}
|
||||
@@ -627,9 +633,7 @@ def _receiver(
|
||||
if state.last_ingress_sequence is not None:
|
||||
if sequence <= state.last_ingress_sequence:
|
||||
raise ShadowRuntimeError("shadow ingress sequence is not increasing")
|
||||
state.ingress_sequence_gaps += max(
|
||||
0, sequence - state.last_ingress_sequence - 1
|
||||
)
|
||||
state.ingress_sequence_gaps += max(0, sequence - state.last_ingress_sequence - 1)
|
||||
if state.first_ingress_sequence is None:
|
||||
state.first_ingress_sequence = sequence
|
||||
state.last_ingress_sequence = sequence
|
||||
@@ -661,9 +665,7 @@ def _receiver(
|
||||
f"expected at least {expected_camera_sequence}, "
|
||||
f"got {camera_source_sequence}"
|
||||
)
|
||||
state.camera_sequence_gaps += (
|
||||
camera_source_sequence - expected_camera_sequence
|
||||
)
|
||||
state.camera_sequence_gaps += camera_source_sequence - expected_camera_sequence
|
||||
state.last_camera_source_sequence = camera_source_sequence
|
||||
decoder.feed_segment(
|
||||
CameraFragmentMetadata(
|
||||
@@ -686,9 +688,7 @@ def _receiver(
|
||||
),
|
||||
processing_started_monotonic_ns=time.monotonic_ns(),
|
||||
)
|
||||
sensor_decode_ms[modality].append(
|
||||
(time.perf_counter() - decode_started) * 1000
|
||||
)
|
||||
sensor_decode_ms[modality].append((time.perf_counter() - decode_started) * 1000)
|
||||
if modality == "lidar" and isinstance(normalized, DecodedPointCloudView):
|
||||
synchronizer.publish_point_cloud(normalized)
|
||||
elif modality == "pose" and isinstance(normalized, DecodedPoseView):
|
||||
@@ -707,9 +707,7 @@ def _receiver(
|
||||
sender_thread.join(timeout=5)
|
||||
if sender_thread.is_alive():
|
||||
assert state.failures is not None
|
||||
state.failures.append(
|
||||
ShadowRuntimeError("live result publisher did not stop")
|
||||
)
|
||||
state.failures.append(ShadowRuntimeError("live result publisher did not stop"))
|
||||
if stream is not None:
|
||||
with suppress(Exception), write_lock:
|
||||
_send_client_frame(stream, 0x8, b"")
|
||||
@@ -760,6 +758,12 @@ def _common(args: argparse.Namespace) -> dict[str, Any]:
|
||||
if dependency["identity"]["profile_sha256"] != semantic_sha256:
|
||||
raise RuntimeError("LAB E15 semantic dependency identity changed")
|
||||
worker_package = read_worker_package(args.worker_package)
|
||||
stability = None
|
||||
stability_sha256 = None
|
||||
if args.stability_profile is not None:
|
||||
stability, stability_sha256 = read_inline_profile(args.stability_profile)
|
||||
if stability["source"] != live["source"] or stability["authority"] != live["authority"]:
|
||||
raise RuntimeError("LAB E23 inline temporal source binding changed")
|
||||
return {
|
||||
"job": job,
|
||||
"live": live,
|
||||
@@ -777,6 +781,8 @@ def _common(args: argparse.Namespace) -> dict[str, Any]:
|
||||
"detector_files": detector_files,
|
||||
"dependency": dependency,
|
||||
"worker_package": worker_package,
|
||||
"stability": stability,
|
||||
"stability_sha256": stability_sha256,
|
||||
}
|
||||
|
||||
|
||||
@@ -804,6 +810,7 @@ def preflight(args: argparse.Namespace) -> int:
|
||||
"e14_profile_sha256": common["e14_sha256"],
|
||||
"projection_pack": common["projection_manifest"]["pack_id"],
|
||||
"worker_package": common["worker_package"]["package_id"],
|
||||
"stability_profile_sha256": common["stability_sha256"],
|
||||
"pyav": av.__version__,
|
||||
"lz4": getattr(lz4, "__version__", importlib.metadata.version("lz4")),
|
||||
"detector_files": common["detector_files"],
|
||||
@@ -840,8 +847,7 @@ def _load_models(args: argparse.Namespace, common: dict[str, Any]) -> _LoadedMod
|
||||
)
|
||||
infer_semantic = e9._semantic_infer_factory(processor, semantic_model, device)
|
||||
target_names = {
|
||||
int(key): str(value)
|
||||
for key, value in common["semantic"]["target_taxonomy"].items()
|
||||
int(key): str(value) for key, value in common["semantic"]["target_taxonomy"].items()
|
||||
}
|
||||
warm = np.zeros((600, 800, 3), dtype=np.uint8)
|
||||
_infer(
|
||||
@@ -894,6 +900,10 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
infer_semantic = loaded.infer_semantic
|
||||
semantic_files = loaded.semantic_files
|
||||
target_lut = loaded.target_lut
|
||||
stability = common["stability"]
|
||||
temporal_stabilizer = TemporalStabilizer(stability) if stability is not None else None
|
||||
semantic_stabilizer = StreamingSemanticStabilizer(stability) if stability is not None else None
|
||||
temporal_world_memory: dict[int, dict[str, Any]] = {}
|
||||
tracker = TwoStageTracker(detector["tracking"])
|
||||
linear_sum_assignment(np.zeros((1, 1), dtype=np.float64))
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
@@ -988,16 +998,19 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
"source_epoch_age_ms_unqualified",
|
||||
)
|
||||
}
|
||||
latency["temporal_2d_3d_ms"] = deque(maxlen=4096)
|
||||
status_counts: Counter[str] = Counter()
|
||||
fusion_state_counts: Counter[str] = Counter()
|
||||
rejection_counts: Counter[str] = Counter()
|
||||
fused_frames = 0
|
||||
accepted_cuboids = 0
|
||||
stabilized_cuboids = 0
|
||||
detector_failures = 0
|
||||
history = distance_history(int(e14["association"]["distance_history_frames"]))
|
||||
projector = WorldStateProjector(float(e14["world_state"]["velocity_history_limit_s"]))
|
||||
completion_tracker = CuboidCompletionTracker(e14["cuboid_completion"])
|
||||
semantic_path = output / "semantic-frames.jsonl"
|
||||
raw_fusion_path = output / "raw-fusion-frames.jsonl"
|
||||
fusion_path = output / "fusion-frames.jsonl"
|
||||
world_path = output / "world-state.jsonl"
|
||||
gpu_path = output / "gpu-telemetry.jsonl"
|
||||
@@ -1007,6 +1020,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
|
||||
with (
|
||||
semantic_path.open("x", encoding="utf-8", newline="\n") as semantic_stream,
|
||||
raw_fusion_path.open("x", encoding="utf-8", newline="\n") as raw_fusion_stream,
|
||||
fusion_path.open("x", encoding="utf-8", newline="\n") as fusion_stream,
|
||||
world_path.open("x", encoding="utf-8", newline="\n") as world_stream,
|
||||
gpu_path.open("x", encoding="utf-8", newline="\n") as gpu_stream,
|
||||
@@ -1029,6 +1043,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
},
|
||||
) as runtime_telemetry,
|
||||
):
|
||||
|
||||
class SemanticResultStream:
|
||||
def __init__(self) -> None:
|
||||
self.count = 0
|
||||
@@ -1067,6 +1082,9 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
"completed": completed_semantics,
|
||||
"failures": semantic_errors,
|
||||
"stop_after_results": None,
|
||||
"transform_target": (
|
||||
None if semantic_stabilizer is None else semantic_stabilizer.update
|
||||
),
|
||||
},
|
||||
name="lab-e15-semantic",
|
||||
daemon=True,
|
||||
@@ -1115,18 +1133,14 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
while (envelope := detector_queue.take()) is not None:
|
||||
started = time.perf_counter()
|
||||
latency["decode_age_ms"].append(float(envelope.decode_ms))
|
||||
latency["queue_wait_ms"].append(
|
||||
max(0.0, (started - envelope.decoded_monotonic) * 1000)
|
||||
)
|
||||
latency["queue_wait_ms"].append(max(0.0, (started - envelope.decoded_monotonic) * 1000))
|
||||
try:
|
||||
detector_started = time.perf_counter()
|
||||
tensor = _preprocess(envelope.image, valid_mask, detector)
|
||||
output_tensor, _request_ms = _infer(args.triton_url, detector["model"], tensor)
|
||||
detections, _rejected = _detections(output_tensor, detector, valid_mask)
|
||||
tracks = tracker.update(detections, envelope.frame_index)
|
||||
latency["detector_ms"].append(
|
||||
(time.perf_counter() - detector_started) * 1000
|
||||
)
|
||||
latency["detector_ms"].append((time.perf_counter() - detector_started) * 1000)
|
||||
|
||||
frame_seconds = float(envelope.timeline["session_seconds"])
|
||||
current_semantic = latest.snapshot()
|
||||
@@ -1141,9 +1155,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
int(envelope.timeline["captured_at_epoch_ns"]),
|
||||
wait_seconds=float(scheduling["sensor_wait_ms"]) / 1000,
|
||||
)
|
||||
latency["sensor_wait_ms"].append(
|
||||
(time.perf_counter() - sensor_wait_started) * 1000
|
||||
)
|
||||
latency["sensor_wait_ms"].append((time.perf_counter() - sensor_wait_started) * 1000)
|
||||
fusions = ()
|
||||
points_lidar = np.empty((0, 3), dtype=np.float64)
|
||||
if binding.state != "fused-ready":
|
||||
@@ -1194,18 +1206,12 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
rejection_counts[item.status] += 1
|
||||
clearance_started = time.perf_counter()
|
||||
clearance_state = clearance(points_lidar, e14["world_state"]["clearance"])
|
||||
latency["clearance_ms"].append(
|
||||
(time.perf_counter() - clearance_started) * 1000
|
||||
)
|
||||
latency["clearance_ms"].append((time.perf_counter() - clearance_started) * 1000)
|
||||
world_started = time.perf_counter()
|
||||
result_age = max(
|
||||
0.0, (world_started - envelope.scheduled_monotonic) * 1000
|
||||
)
|
||||
result_age = max(0.0, (world_started - envelope.scheduled_monotonic) * 1000)
|
||||
if result_age >= 1000:
|
||||
health = "unavailable"
|
||||
elif result_age >= float(
|
||||
live["acceptance"]["maximum_p95_world_state_age_ms"]
|
||||
):
|
||||
elif result_age >= float(live["acceptance"]["maximum_p95_world_state_age_ms"]):
|
||||
health = "stale"
|
||||
elif fusion_state != "fused":
|
||||
health = "degraded"
|
||||
@@ -1229,37 +1235,58 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
"clock_qualification": "worker-ingress-only",
|
||||
},
|
||||
)
|
||||
latency["world_state_ms"].append(
|
||||
(time.perf_counter() - world_started) * 1000
|
||||
)
|
||||
result_age = max(
|
||||
0.0, (time.perf_counter() - envelope.scheduled_monotonic) * 1000
|
||||
raw_fusion_objects = [fusion_document(item) for item in fusions]
|
||||
if temporal_stabilizer is None:
|
||||
fusion_objects = raw_fusion_objects
|
||||
else:
|
||||
temporal_started = time.perf_counter()
|
||||
fusion_objects = temporal_stabilizer.update(
|
||||
frame_index=envelope.frame_index,
|
||||
session_seconds=frame_seconds,
|
||||
objects=raw_fusion_objects,
|
||||
)
|
||||
world = stabilize_world_state(
|
||||
world,
|
||||
fusion_objects,
|
||||
temporal_world_memory,
|
||||
)
|
||||
latency["temporal_2d_3d_ms"].append(
|
||||
(time.perf_counter() - temporal_started) * 1000
|
||||
)
|
||||
stabilized_cuboids += sum(
|
||||
str(item.get("cuboid_status", "")).startswith("accepted-")
|
||||
for item in fusion_objects
|
||||
)
|
||||
latency["world_state_ms"].append((time.perf_counter() - world_started) * 1000)
|
||||
result_age = max(0.0, (time.perf_counter() - envelope.scheduled_monotonic) * 1000)
|
||||
world["delivery"]["result_age_ms"] = result_age
|
||||
latency["world_state_age_ms"].append(result_age)
|
||||
latency["source_epoch_age_ms_unqualified"].append(
|
||||
(time.time_ns() - int(envelope.timeline["captured_at_epoch_ns"]))
|
||||
/ 1_000_000
|
||||
(time.time_ns() - int(envelope.timeline["captured_at_epoch_ns"])) / 1_000_000
|
||||
)
|
||||
fusion_row = {
|
||||
"schema_version": FUSION_SCHEMA,
|
||||
"frame_index": envelope.frame_index,
|
||||
"source_frame_index": int(envelope.timeline["source_frame_index"]),
|
||||
"session_seconds": frame_seconds,
|
||||
"fusion_state": fusion_state,
|
||||
"semantic_status": semantic_status,
|
||||
"semantic_source_frame_index": (
|
||||
None if current_semantic is None else current_semantic.source_frame_index
|
||||
),
|
||||
}
|
||||
raw_fusion_stream.write(
|
||||
json.dumps(
|
||||
{**fusion_row, "objects": raw_fusion_objects},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
fusion_objects = [fusion_document(item) for item in fusions]
|
||||
fusion_stream.write(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": FUSION_SCHEMA,
|
||||
"frame_index": envelope.frame_index,
|
||||
"source_frame_index": int(
|
||||
envelope.timeline["source_frame_index"]
|
||||
),
|
||||
"session_seconds": frame_seconds,
|
||||
"fusion_state": fusion_state,
|
||||
"semantic_status": semantic_status,
|
||||
"semantic_source_frame_index": (
|
||||
None
|
||||
if current_semantic is None
|
||||
else current_semantic.source_frame_index
|
||||
),
|
||||
"objects": fusion_objects,
|
||||
},
|
||||
{**fusion_row, "objects": fusion_objects},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
@@ -1315,18 +1342,9 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
receiver_thread.join(timeout=args.max_duration_seconds + 10)
|
||||
decoder_watcher.join(timeout=30)
|
||||
semantic_thread.join(timeout=30)
|
||||
if (
|
||||
receiver_thread.is_alive()
|
||||
or decoder_watcher.is_alive()
|
||||
or semantic_thread.is_alive()
|
||||
):
|
||||
if receiver_thread.is_alive() or decoder_watcher.is_alive() or semantic_thread.is_alive():
|
||||
raise RuntimeError("LAB E15 runtime thread did not stop")
|
||||
if (
|
||||
transport.failures
|
||||
or decoder_watch_failures
|
||||
or callback_failures
|
||||
or semantic_errors
|
||||
):
|
||||
if transport.failures or decoder_watch_failures or callback_failures or semantic_errors:
|
||||
failures = [
|
||||
*(transport.failures or []),
|
||||
*decoder_watch_failures,
|
||||
@@ -1334,11 +1352,15 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
*semantic_errors,
|
||||
]
|
||||
summary = "; ".join(
|
||||
f"{type(failure).__name__}: {str(failure)[:240]}"
|
||||
for failure in failures[:8]
|
||||
f"{type(failure).__name__}: {str(failure)[:240]}" for failure in failures[:8]
|
||||
)
|
||||
raise RuntimeError(f"LAB E15 runtime worker failed: {summary}")
|
||||
for stream in (semantic_stream, fusion_stream, world_stream):
|
||||
for stream in (
|
||||
semantic_stream,
|
||||
raw_fusion_stream,
|
||||
fusion_stream,
|
||||
world_stream,
|
||||
):
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
@@ -1354,22 +1376,23 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
detector_fps = int(detector_state["consumed"]) / max(source_span, run_wall, 1e-9)
|
||||
semantic_fps = int(semantic_state["consumed"]) / max(source_span, run_wall, 1e-9)
|
||||
semantic_scheduled = (
|
||||
(decoded_frame_count - 1)
|
||||
// int(scheduling["semantic_sample_every_frames"])
|
||||
(decoded_frame_count - 1) // int(scheduling["semantic_sample_every_frames"])
|
||||
) + 1
|
||||
fresh_coverage = status_counts["fresh"] / max(1, int(detector_state["consumed"]))
|
||||
fused_fraction = fused_frames / max(1, int(detector_state["consumed"]))
|
||||
latency_summary = {name: _percentiles(values) for name, values in latency.items()}
|
||||
semantic_summary = {
|
||||
name: _percentiles(values) for name, values in semantic_latency.items()
|
||||
}
|
||||
semantic_summary = {name: _percentiles(values) for name, values in semantic_latency.items()}
|
||||
sensor_decode_summary = {
|
||||
name: _percentiles(values) for name, values in sensor_decode_ms.items()
|
||||
}
|
||||
runtime_summary = runtime_telemetry.summary()
|
||||
temporal_track_summary = None if temporal_stabilizer is None else temporal_stabilizer.snapshot()
|
||||
temporal_semantic_summary = (
|
||||
None if semantic_stabilizer is None else semantic_stabilizer.snapshot()
|
||||
)
|
||||
acceptance = live["acceptance"]
|
||||
checks = {
|
||||
"minimum_camera_frames": decoded_frame_count
|
||||
>= int(acceptance["minimum_camera_frames"]),
|
||||
"minimum_camera_frames": decoded_frame_count >= int(acceptance["minimum_camera_frames"]),
|
||||
"camera_decoder_accounting": decoder.snapshot()["decoded_frames"]
|
||||
== transport.counts["camera-frame"],
|
||||
"detector_accounting": int(detector_state["consumed"])
|
||||
@@ -1394,13 +1417,10 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
<= float(acceptance["semantic_maximum_p95_completion_age_ms"]),
|
||||
"minimum_fresh_semantic_coverage": fresh_coverage
|
||||
>= float(acceptance["minimum_fresh_semantic_coverage"]),
|
||||
"minimum_fused_fraction": fused_fraction
|
||||
>= float(acceptance["minimum_fused_fraction"]),
|
||||
"minimum_fused_fraction": fused_fraction >= float(acceptance["minimum_fused_fraction"]),
|
||||
"maximum_p95_decode_age_ms": float(latency_summary["decode_age_ms"]["p95"])
|
||||
<= float(acceptance["maximum_p95_decode_age_ms"]),
|
||||
"maximum_p95_world_state_age_ms": float(
|
||||
latency_summary["world_state_age_ms"]["p95"]
|
||||
)
|
||||
"maximum_p95_world_state_age_ms": float(latency_summary["world_state_age_ms"]["p95"])
|
||||
<= float(acceptance["maximum_p95_world_state_age_ms"]),
|
||||
"zero_transport_gaps": (
|
||||
transport.ingress_sequence_gaps == 0
|
||||
@@ -1424,10 +1444,39 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
"authority_remains_shadow_only": live["authority"]
|
||||
== {"commands_enabled": False, "navigation_or_safety_accepted": False},
|
||||
}
|
||||
if stability is not None:
|
||||
temporal_acceptance = stability["acceptance"]
|
||||
assert temporal_track_summary is not None
|
||||
assert temporal_semantic_summary is not None
|
||||
checks.update(
|
||||
{
|
||||
"temporal_camera_processing_p95": float(latency_summary["temporal_2d_3d_ms"]["p95"])
|
||||
<= float(temporal_acceptance["maximum_camera_frame_processing_p95_ms"]),
|
||||
"temporal_semantic_processing_p95": float(
|
||||
temporal_semantic_summary["processing_ms"]["p95"]
|
||||
)
|
||||
<= float(temporal_acceptance["maximum_semantic_frame_processing_p95_ms"]),
|
||||
"temporal_track_state_bound": int(temporal_track_summary["peak_track_states"])
|
||||
<= int(temporal_acceptance["maximum_track_states_observed"]),
|
||||
"temporal_semantic_unsupported_change_reduction": float(
|
||||
temporal_semantic_summary["unsupported_change_reduction_fraction"]
|
||||
)
|
||||
>= float(
|
||||
temporal_acceptance["minimum_semantic_unsupported_change_reduction_fraction"]
|
||||
),
|
||||
"temporal_rss_growth_bound": float(runtime_summary["rss_growth_mib"])
|
||||
<= float(temporal_acceptance["maximum_rss_growth_mib"]),
|
||||
"temporal_authority_remains_shadow_only": stability["authority"]
|
||||
== {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
)
|
||||
accepted = all(checks.values())
|
||||
identity = {
|
||||
"schema_version": IDENTITY_SCHEMA,
|
||||
"pipeline": PIPELINE_ID,
|
||||
"pipeline": (PIPELINE_ID if stability is None else INLINE_TEMPORAL_PIPELINE_ID),
|
||||
"session_id": transport.session_id,
|
||||
"bootstrap_job_id": common["job"]["job_id"],
|
||||
"source_id": live["source"]["source_id"],
|
||||
@@ -1436,6 +1485,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
"e14_sha256": common["e14_sha256"],
|
||||
"detector_sha256": common["detector_sha256"],
|
||||
"semantic_sha256": common["semantic_sha256"],
|
||||
"stability_sha256": common["stability_sha256"],
|
||||
},
|
||||
"projection_pack": {
|
||||
"id": common["projection_manifest"]["pack_id"],
|
||||
@@ -1498,14 +1548,22 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
"fused_frames": fused_frames,
|
||||
"fused_fraction": fused_fraction,
|
||||
"accepted_cuboids": accepted_cuboids,
|
||||
"stabilized_cuboids": stabilized_cuboids,
|
||||
"state_counts": dict(fusion_state_counts),
|
||||
"rejection_counts": dict(rejection_counts),
|
||||
"synchronizer": synchronizer.snapshot(),
|
||||
"sensor_decode_ms": sensor_decode_summary,
|
||||
},
|
||||
"latency_ms": latency_summary,
|
||||
"temporal_stability": {
|
||||
"enabled": stability is not None,
|
||||
"profile_sha256": common["stability_sha256"],
|
||||
"tracking_2d_3d": temporal_track_summary,
|
||||
"semantic": temporal_semantic_summary,
|
||||
"world_memory_objects": len(temporal_world_memory),
|
||||
},
|
||||
"gpu_telemetry": gpu.summary(),
|
||||
"runtime_telemetry": runtime_telemetry.summary(),
|
||||
"runtime_telemetry": runtime_summary,
|
||||
"process_cpu_seconds": time.process_time() - process_cpu_started,
|
||||
"process_peak_rss_mib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024,
|
||||
"cuda_peak_memory_allocated_mib": torch.cuda.max_memory_allocated() / 2**20,
|
||||
@@ -1547,12 +1605,26 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
"Cross-host source epoch age is diagnostic and excluded from acceptance.",
|
||||
"COCO and Cityscapes models are not forest-domain or safety validated.",
|
||||
"Amodal cuboids infer unobserved volume from class priors.",
|
||||
*(
|
||||
[
|
||||
"E23 temporal outputs are inline bounded shadow diagnostics, "
|
||||
"not validated driving or safety authority."
|
||||
]
|
||||
if stability is not None
|
||||
else []
|
||||
),
|
||||
],
|
||||
}
|
||||
report_path = output / "run-report.json"
|
||||
_write_json(report_path, report)
|
||||
artifacts = [
|
||||
_artifact(semantic_path, "e15-semantic-frames", "application/x-ndjson", SEMANTIC_SCHEMA),
|
||||
_artifact(
|
||||
raw_fusion_path,
|
||||
"e23-raw-fusion-frames",
|
||||
"application/x-ndjson",
|
||||
FUSION_SCHEMA,
|
||||
),
|
||||
_artifact(fusion_path, "e15-fusion-frames", "application/x-ndjson", FUSION_SCHEMA),
|
||||
_artifact(world_path, "e15-world-state", "application/x-ndjson", WORLD_SCHEMA),
|
||||
_artifact(gpu_path, "worker-gpu-telemetry", "application/x-ndjson"),
|
||||
@@ -1590,6 +1662,8 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
"fused_fraction": fused_fraction,
|
||||
"world_state_age_p95_ms": latency_summary["world_state_age_ms"]["p95"],
|
||||
"accepted_cuboids": accepted_cuboids,
|
||||
"inline_temporal": stability is not None,
|
||||
"temporal_2d_3d_p95_ms": latency_summary["temporal_2d_3d_ms"]["p95"],
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user