"""Validate and project accepted LAB E10 integrated perception results.""" from __future__ import annotations import hashlib import json import os import re import shutil import stat import subprocess import tempfile import threading import time from collections.abc import Iterator from contextlib import contextmanager, suppress from dataclasses import dataclass from pathlib import Path from typing import Any, Protocol, TypeGuard import numpy as np import rerun as rr from rerun.components import FillMode from k1link.artifact_gateway import ( ArtifactGateway, ArtifactNotFound, ArtifactStoreUnavailable, ) from k1link.artifacts import write_json_atomic from k1link.sessions import SessionIntegrityError from .jobs import CameraComputeJob, validate_camera_compute_job from .results import ( RecordedPerceptionOverlayArtifact, RecordedPerceptionOverlayError, ) RESULT_SCHEMA = "missioncore.e10-integrated-perception-result/v1" IDENTITY_SCHEMA = "missioncore.e10-integrated-perception-identity/v1" REPORT_SCHEMA = "missioncore.e10-integrated-perception-report/v1" PACK_SCHEMA = "missioncore.e10-lidar-replay-pack/v1" SEMANTIC_SCHEMA = "missioncore.e10-semantic-frame/v1" FUSION_SCHEMA = "missioncore.e10-fusion-frame/v1" WORLD_SCHEMA = "missioncore.live-perception-world-state/v1" SESSION_TIMELINE = "session_time" MAX_SCAN = 512 MAX_JSON_BYTES = 64 * 1024 * 1024 MAX_LINE_BYTES = 4 * 1024 * 1024 MAX_SOURCE_BYTES = 512 * 1024 * 1024 OVERLAY_RENDERER_VERSION = "4" OVERLAY_ADMISSION_SCHEMA = "missioncore.e10-overlay-admission/v1" OVERLAY_CACHE_SCHEMA = "missioncore.e10-overlay-cache/v1" CUBOID_PRESENTATION_HOLD_NS = 500_000_000 _SAFE_RESULT_ID = re.compile(r"^e10-integrated-perception-[a-f0-9]{64}$") _SAFE_PACK_ID = re.compile(r"^e10-lidar-pack-[a-f0-9]{64}$") _SAFE_JOB_ID = re.compile(r"^recorded-camera-[a-f0-9]{24}$") _SAFE_RECORDING_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") _SHA256 = re.compile(r"^[a-f0-9]{64}$") @dataclass(frozen=True, slots=True) class IntegratedPerceptionResult: result_id: str result_root: Path job: CameraComputeJob pack_root: Path created_at_utc: str accepted: bool publication_scope: str source_start_frame_index: int frame_count: int timeline_start_seconds: float timeline_end_seconds: float semantic_path: Path fusion_path: Path world_path: Path arrays_path: Path report_path: Path @dataclass(frozen=True, slots=True) class _ResultDescriptor: result_id: str result_root: Path result_json_sha256: str session_id: str job_id: str created_at_utc: str @dataclass(frozen=True, slots=True) class _OverlayAdmission: session_id: str result_id: str result_json_sha256: str result_created_at_utc: str @dataclass(slots=True) class _FlightLock: lock: threading.Lock users: int = 0 @dataclass(frozen=True, slots=True) class _PresentedCuboid: observed_ns: int track_id: int label: str association_group: str distance_m: float | None support_points: int center: np.ndarray half_size: np.ndarray quaternion: np.ndarray color: np.ndarray class _CuboidPresentationState: """Hold accepted cuboids briefly for operator presentation only. The persisted world-state remains fail-closed and frame-exact. This bounded latest-at projection only prevents one rejected LiDAR association from visually clearing an otherwise stable tracked object for a single frame. """ def __init__(self, hold_ns: int = CUBOID_PRESENTATION_HOLD_NS) -> None: if hold_ns <= 0: raise ValueError("cuboid presentation hold must be positive") self._hold_ns = hold_ns self._latest: dict[int, _PresentedCuboid] = {} def update( self, timestamp_ns: int, objects: list[dict[str, Any]], centers: np.ndarray, half_sizes: np.ndarray, quaternions: np.ndarray, colors: np.ndarray, ) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, list[str]] | None: accepted = [ item for item in objects if str(item.get("cuboid_status", "")).startswith("accepted-") ] if not ( len(accepted) == len(centers) == len(half_sizes) == len(quaternions) == len(colors) ): raise RecordedPerceptionOverlayError( "integrated perception cuboid presentation arrays are inconsistent" ) for item, center, half_size, quaternion, color in zip( accepted, centers, half_sizes, quaternions, colors, strict=True, ): track_id = item.get("track_id") if not isinstance(track_id, int) or isinstance(track_id, bool): raise RecordedPerceptionOverlayError( "integrated perception cuboid track identity is invalid" ) distance_value = item.get("distance_smoothed_m") distance_m = ( float(distance_value) if isinstance(distance_value, int | float) and not isinstance(distance_value, bool) and np.isfinite(float(distance_value)) else None ) self._latest[track_id] = _PresentedCuboid( observed_ns=timestamp_ns, track_id=track_id, label=str(item.get("label", "object")), association_group=str(item.get("association_group", "object")), distance_m=distance_m, support_points=int(item["clustered_points"]), center=np.asarray(center).copy(), half_size=np.asarray(half_size).copy(), quaternion=np.asarray(quaternion).copy(), color=np.asarray(color).copy(), ) expired = [ track_id for track_id, cuboid in self._latest.items() if timestamp_ns - cuboid.observed_ns > self._hold_ns ] for track_id in expired: del self._latest[track_id] if not self._latest: return None presented = sorted(self._latest.values(), key=lambda cuboid: cuboid.track_id) presented_colors: list[np.ndarray] = [] labels: list[str] = [] for cuboid in presented: age_ns = max(0, timestamp_ns - cuboid.observed_ns) color = cuboid.color.copy() if age_ns > 0 and color.shape == (4,): fade = min(1.0, age_ns / self._hold_ns) color[3] = max(24, round(float(color[3]) * (1.0 - 0.55 * fade))) presented_colors.append(color) age_label = "" if age_ns == 0 else f" · hold {age_ns / 1_000_000:.0f} ms" distance_label = "" if cuboid.distance_m is None else f" · {cuboid.distance_m:.1f} m" labels.append( f"{cuboid.association_group} #{cuboid.track_id} {cuboid.label}" f"{distance_label} · {cuboid.support_points} pts{age_label}" ) return ( np.stack([cuboid.center for cuboid in presented]), np.stack([cuboid.half_size for cuboid in presented]), np.stack([cuboid.quaternion for cuboid in presented]), np.stack(presented_colors), labels, ) class _OverlayProvider(Protocol): def render( self, session_id: str, *, application_id: str, recording_id: str, ) -> bytes | None: ... def validate_integrated_perception_result( job_root: Path, result_root: Path, lidar_packs_root: Path, ) -> IntegratedPerceptionResult: job = validate_camera_compute_job(job_root) root = result_root.expanduser().resolve(strict=True) if not root.is_dir() or _SAFE_RESULT_ID.fullmatch(root.name) is None: raise SessionIntegrityError("integrated perception root is invalid") result = _read_object(root / "result.json", root) identity = result.get("identity") identity_sha256 = result.get("identity_sha256") if ( result.get("schema_version") != RESULT_SCHEMA or result.get("result_id") != root.name or not isinstance(identity, dict) or identity.get("schema_version") != IDENTITY_SCHEMA or not isinstance(identity_sha256, str) or _SHA256.fullmatch(identity_sha256) is None or root.name != f"e10-integrated-perception-{identity_sha256}" or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 or identity.get("job_id") != job.job_id or identity.get("input_sha256") != job.input_sha256 or identity.get("session_id") != job.session_id or identity.get("source_id") != job.source_id or result.get("ground_truth") is not False or result.get("publication_scope") not in { "recorded-integrated-realtime-qualification-only", "recorded-integrated-semantic-loss-negative-control-only", } or result.get("acceptance_state") not in {"accepted", "rejected"} ): raise SessionIntegrityError("integrated perception identity is inconsistent") selection = identity.get("selection") if not isinstance(selection, dict): raise SessionIntegrityError("integrated perception selection is missing") count = selection.get("frame_count") start = selection.get("source_start_frame_index") end = selection.get("source_end_frame_index") timeline_start = selection.get("timeline_start_seconds") timeline_end = selection.get("timeline_end_seconds") if ( not isinstance(count, int) or isinstance(count, bool) or count < 2 or not isinstance(start, int) or isinstance(start, bool) or not isinstance(end, int) or isinstance(end, bool) or end - start + 1 != count or start < 0 or end >= job.segment_count or not _finite(timeline_start) or not _finite(timeline_end) or float(timeline_end) <= float(timeline_start) or result.get("frames_processed") != count ): raise SessionIntegrityError("integrated perception selection is invalid") pack_id = identity.get("lidar_pack_id") if not isinstance(pack_id, str) or _SAFE_PACK_ID.fullmatch(pack_id) is None: raise SessionIntegrityError("integrated perception LiDAR pack binding is invalid") pack_root = lidar_packs_root.expanduser().resolve(strict=True) / pack_id _validate_pack(pack_root, job, identity, count, start, end) artifacts = _validate_artifacts(root, result.get("artifacts")) semantic_path = artifacts["e10-semantic-frames"] fusion_path = artifacts["e10-fusion-frames"] world_path = artifacts["e10-world-state"] arrays_path = artifacts["e10-transient-perception"] report_path = artifacts["e10-run-report"] semantic_rows = _read_rows(semantic_path, root, SEMANTIC_SCHEMA) fusion_rows = _read_rows(fusion_path, root, FUSION_SCHEMA) world_rows = _read_rows(world_path, root, WORLD_SCHEMA) if len(fusion_rows) != count or len(world_rows) != count or not semantic_rows: raise SessionIntegrityError("integrated perception frame counts are incomplete") for index, (fusion, world) in enumerate(zip(fusion_rows, world_rows, strict=True)): source_index = start + index if ( fusion.get("frame_index") != index or fusion.get("source_frame_index") != source_index or world.get("frame_index") != index or world.get("source_frame_index") != source_index or not isinstance(fusion.get("objects"), list) or not isinstance(world.get("objects"), list) or world.get("object_count") != len(world["objects"]) or not isinstance(world.get("delivery"), dict) ): raise SessionIntegrityError("integrated perception frame identity changed") semantic_indices: list[int] = [] for row in semantic_rows: frame_index = row.get("frame_index") if not isinstance(frame_index, int) or isinstance(frame_index, bool): raise SessionIntegrityError("integrated perception semantic timeline is invalid") semantic_indices.append(frame_index) if ( semantic_indices != sorted(set(semantic_indices)) or semantic_indices[0] != 0 or semantic_indices[-1] >= count ): raise SessionIntegrityError("integrated perception semantic timeline is invalid") _validate_arrays(arrays_path, count, semantic_rows, fusion_rows) report = _read_object(report_path, root) acceptance = report.get("acceptance") if ( report.get("schema_version") != REPORT_SCHEMA or report.get("result_id") != root.name or report.get("identity") != identity or report.get("ground_truth") is not False or not isinstance(acceptance, dict) or acceptance.get("accepted") is not (result.get("acceptance_state") == "accepted") or acceptance.get("navigation_or_safety_accepted") is not False ): raise SessionIntegrityError("integrated perception report is inconsistent") created = result.get("created_at_utc") if not isinstance(created, str) or not 1 <= len(created) <= 64: raise SessionIntegrityError("integrated perception creation time is invalid") return IntegratedPerceptionResult( result_id=root.name, result_root=root, job=job, pack_root=pack_root, created_at_utc=created, accepted=result.get("acceptance_state") == "accepted", publication_scope=str(result["publication_scope"]), source_start_frame_index=start, frame_count=count, timeline_start_seconds=float(timeline_start), timeline_end_seconds=float(timeline_end), semantic_path=semantic_path, fusion_path=fusion_path, world_path=world_path, arrays_path=arrays_path, report_path=report_path, ) class IntegratedPerceptionOverlayStore: """Serve the newest admitted E10 overlay without revalidating it on replay. Result validation is an admission concern. Once a complete overlay has been rendered and sealed, its cache and admission sidecars are sufficient for the presentation path. A cache miss still fails closed and validates the exact selected result before rendering. """ def __init__( self, *, jobs_root: Path, results_root: Path, lidar_packs_root: Path, cache_root: Path, ffmpeg_path: Path, artifact_gateway: ArtifactGateway | None = None, ) -> None: self.jobs_root = jobs_root.expanduser().absolute() self.results_root = results_root.expanduser().absolute() self.lidar_packs_root = lidar_packs_root.expanduser().absolute() self.cache_root = cache_root.expanduser().absolute() self.ffmpeg_path = ffmpeg_path.expanduser().absolute() self.artifact_gateway = artifact_gateway self._catalog_lock = threading.Lock() self._descriptor_catalog_generation: tuple[int, int] | None = None self._descriptors_by_session: dict[str, tuple[_ResultDescriptor, ...]] = {} self._latest_by_session: dict[str, IntegratedPerceptionResult | None] = {} self._flight_guard = threading.Lock() self._flights: dict[tuple[str, str], _FlightLock] = {} self._materialization_lock = threading.Lock() self._status_lock = threading.Lock() self._status_by_recording: dict[tuple[str, str], dict[str, Any]] = {} def render( self, session_id: str, *, application_id: str, recording_id: str, ) -> bytes | None: """Compatibility adapter for in-process consumers that require bytes.""" artifact = self.materialize( session_id, application_id=application_id, recording_id=recording_id, ) if artifact is None: return None try: payload = artifact.path.read_bytes() except OSError as exc: raise RecordedPerceptionOverlayError( "integrated perception cache became unavailable" ) from exc if ( len(payload) != artifact.byte_length or hashlib.sha256(payload).hexdigest() != artifact.sha256 ): raise RecordedPerceptionOverlayError( "integrated perception cache changed after validation" ) return payload def materialize( self, session_id: str, *, application_id: str, recording_id: str, ) -> RecordedPerceptionOverlayArtifact | None: """Return a verified file-backed overlay without copying it into heap.""" if application_id != "nodedc_mission_core_recorded": raise ValueError("integrated perception application id is invalid") if ( _SAFE_RECORDING_ID.fullmatch(session_id) is None or _SAFE_RECORDING_ID.fullmatch(recording_id) is None ): raise ValueError("integrated perception recording id is invalid") key = (session_id, recording_id) self._set_status(key, state="preparing", phase="cache-lookup") try: with self._single_flight(key): central = self._read_gateway_cache(session_id, recording_id) if central is not None: self._set_status( key, state="ready", phase="ready", byte_length=central.byte_length, ) return central cached = self._read_admitted_cache(session_id, recording_id) if cached is not None: self._set_status( key, state="ready", phase="ready", byte_length=cached.byte_length, ) return cached self._set_status(key, state="preparing", phase="queued") with self._materialization_lock: self._set_status(key, state="preparing", phase="artifact-validation") result = self._latest(session_id) if result is None: self._set_status(key, state="unavailable", phase="unavailable") return None self._write_admission(result) cache = _private_child( _private_child(_private_directory(self.cache_root), session_id), result.result_id, ) output = cache / f"{recording_id}.rrd" sidecar = output.with_suffix(".rrd.cache.json") cached = _read_cache_artifact( output, sidecar, result_id=result.result_id, recording_id=recording_id, ) if cached is not None: self._set_status( key, state="ready", phase="ready", byte_length=cached.byte_length, ) return cached self._set_status(key, state="preparing", phase="rendering") payload = _render( result, application_id=application_id, recording_id=recording_id, ffmpeg_path=self.ffmpeg_path, temporary_root=cache, ) self._set_status(key, state="preparing", phase="cache-write") temporary = output.with_name(f".{output.name}.{os.getpid()}.tmp") try: with temporary.open("xb") as stream: stream.write(payload) stream.flush() os.fsync(stream.fileno()) os.chmod(temporary, 0o600) os.replace(temporary, output) write_json_atomic( sidecar, { "schema_version": OVERLAY_CACHE_SCHEMA, "renderer_version": OVERLAY_RENDERER_VERSION, "session_id": session_id, "result_id": result.result_id, "result_created_at_utc": result.created_at_utc, "recording_id": recording_id, "byte_length": len(payload), "sha256": hashlib.sha256(payload).hexdigest(), }, ) finally: temporary.unlink(missing_ok=True) self._set_status( key, state="ready", phase="ready", byte_length=len(payload), ) published = _read_cache_artifact( output, sidecar, result_id=result.result_id, recording_id=recording_id, ) if published is None: raise RecordedPerceptionOverlayError( "integrated perception cache publication failed" ) return published except BaseException: self._set_status(key, state="error", phase="error") raise def _read_gateway_cache( self, session_id: str, recording_id: str, ) -> RecordedPerceptionOverlayArtifact | None: if self.artifact_gateway is None: return None role = f"integrated-overlay:{recording_id}" try: resolved = self.artifact_gateway.resolve_role("sessions", session_id, role) except ArtifactNotFound: return None except ArtifactStoreUnavailable as exc: raise RecordedPerceptionOverlayError( "central integrated perception artifact is unavailable " "and is not present in the local artifact cache" ) from exc expected_result_id = resolved.manifest.metadata.get("integrated-result-id") if ( resolved.member.media_type != "application/vnd.rerun.rrd" or expected_result_id is None or _SAFE_RESULT_ID.fullmatch(expected_result_id) is None ): raise RecordedPerceptionOverlayError( "central integrated perception artifact metadata is invalid" ) try: metadata = resolved.path.lstat() with resolved.path.open("rb") as stream: magic = stream.read(4) except OSError as exc: raise RecordedPerceptionOverlayError( "central integrated perception artifact became unavailable" ) from exc if ( stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode) or metadata.st_size != resolved.member.byte_length or magic != b"RRF2" ): raise RecordedPerceptionOverlayError( "central integrated perception artifact is invalid" ) return RecordedPerceptionOverlayArtifact( path=resolved.path.resolve(strict=True), byte_length=resolved.member.byte_length, sha256=resolved.member.sha256, ) def status(self, session_id: str, *, recording_id: str) -> dict[str, Any]: if ( _SAFE_RECORDING_ID.fullmatch(session_id) is None or _SAFE_RECORDING_ID.fullmatch(recording_id) is None ): raise ValueError("integrated perception recording id is invalid") key = (session_id, recording_id) with self._status_lock: value = self._status_by_recording.get(key) if value is None: return { "state": "idle", "phase": "idle", "elapsed_seconds": 0.0, "byte_length": None, } elapsed = max(0.0, time.monotonic() - float(value["started_monotonic"])) return { "state": value["state"], "phase": value["phase"], "elapsed_seconds": round(elapsed, 3), "byte_length": value.get("byte_length"), } def _set_status( self, key: tuple[str, str], *, state: str, phase: str, byte_length: int | None = None, ) -> None: with self._status_lock: previous = self._status_by_recording.get(key) started = ( float(previous["started_monotonic"]) if previous is not None and previous.get("state") == "preparing" else time.monotonic() ) self._status_by_recording[key] = { "state": state, "phase": phase, "started_monotonic": started, "byte_length": byte_length, } @contextmanager def _single_flight(self, key: tuple[str, str]) -> Iterator[None]: with self._flight_guard: flight = self._flights.get(key) if flight is None: flight = _FlightLock(lock=threading.Lock()) self._flights[key] = flight flight.users += 1 flight.lock.acquire() try: yield finally: flight.lock.release() with self._flight_guard: flight.users -= 1 if flight.users == 0: self._flights.pop(key, None) def _read_admitted_cache( self, session_id: str, recording_id: str, ) -> RecordedPerceptionOverlayArtifact | None: cache_root = _private_directory(self.cache_root) session_cache = _private_child(cache_root, session_id) admission = self._load_admission(session_cache, session_id) latest = self._latest_descriptor(session_id) if latest is None: return None if admission is None or admission.result_id != latest.result_id: recovered = self._recover_admission_from_cache( session_cache, latest, recording_id, ) if recovered is not None: _, artifact = recovered return artifact admission = None if admission is None or admission.result_id != latest.result_id: return None result_cache = _private_child(session_cache, admission.result_id) output = result_cache / f"{recording_id}.rrd" sidecar = output.with_suffix(".rrd.cache.json") return _read_cache_artifact( output, sidecar, result_id=admission.result_id, recording_id=recording_id, ) def _load_admission( self, session_cache: Path, session_id: str, ) -> _OverlayAdmission | None: path = session_cache / "admission.json" try: value = _read_object(path, session_cache) except (OSError, SessionIntegrityError): return None result_id = value.get("result_id") result_json_sha256 = value.get("result_json_sha256") created_at_utc = value.get("result_created_at_utc") if ( value.get("schema_version") != OVERLAY_ADMISSION_SCHEMA or value.get("renderer_version") != OVERLAY_RENDERER_VERSION or value.get("session_id") != session_id or not isinstance(result_id, str) or _SAFE_RESULT_ID.fullmatch(result_id) is None or not isinstance(result_json_sha256, str) or _SHA256.fullmatch(result_json_sha256) is None or not isinstance(created_at_utc, str) ): return None try: descriptor = _read_result_descriptor(self.results_root / result_id) except (OSError, SessionIntegrityError): return None if ( descriptor.session_id != session_id or descriptor.result_json_sha256 != result_json_sha256 or descriptor.created_at_utc != created_at_utc ): return None return _OverlayAdmission( session_id=session_id, result_id=result_id, result_json_sha256=result_json_sha256, result_created_at_utc=created_at_utc, ) def _recover_admission_from_cache( self, session_cache: Path, descriptor: _ResultDescriptor, recording_id: str, ) -> tuple[_OverlayAdmission, RecordedPerceptionOverlayArtifact] | None: cache = _private_child(session_cache, descriptor.result_id) output = cache / f"{recording_id}.rrd" sidecar = output.with_suffix(".rrd.cache.json") artifact = _read_cache_artifact( output, sidecar, result_id=descriptor.result_id, recording_id=recording_id, ) if artifact is None: return None admission = _OverlayAdmission( session_id=descriptor.session_id, result_id=descriptor.result_id, result_json_sha256=descriptor.result_json_sha256, result_created_at_utc=descriptor.created_at_utc, ) self._write_admission_document(session_cache, admission) return admission, artifact def _write_admission(self, result: IntegratedPerceptionResult) -> None: descriptor = _read_result_descriptor(result.result_root) if ( descriptor.session_id != result.job.session_id or descriptor.result_id != result.result_id or descriptor.created_at_utc != result.created_at_utc ): raise RecordedPerceptionOverlayError( "integrated perception admission identity changed" ) cache_root = _private_directory(self.cache_root) session_cache = _private_child(cache_root, descriptor.session_id) self._write_admission_document( session_cache, _OverlayAdmission( session_id=descriptor.session_id, result_id=descriptor.result_id, result_json_sha256=descriptor.result_json_sha256, result_created_at_utc=descriptor.created_at_utc, ), ) @staticmethod def _write_admission_document( session_cache: Path, admission: _OverlayAdmission, ) -> None: write_json_atomic( session_cache / "admission.json", { "schema_version": OVERLAY_ADMISSION_SCHEMA, "renderer_version": OVERLAY_RENDERER_VERSION, "session_id": admission.session_id, "result_id": admission.result_id, "result_json_sha256": admission.result_json_sha256, "result_created_at_utc": admission.result_created_at_utc, }, ) def _latest_descriptor(self, session_id: str) -> _ResultDescriptor | None: try: metadata = self.results_root.stat() entries = tuple(sorted(self.results_root.iterdir(), key=lambda path: path.name)) except FileNotFoundError: return None if len(entries) > MAX_SCAN: raise RecordedPerceptionOverlayError("integrated perception catalog is outside bounds") generation = (metadata.st_mtime_ns, len(entries)) with self._catalog_lock: if generation != self._descriptor_catalog_generation: self._descriptor_catalog_generation = generation self._descriptors_by_session.clear() self._latest_by_session.clear() existing = self._descriptors_by_session.get(session_id) if existing is not None: return existing[0] if existing else None matches: list[_ResultDescriptor] = [] for candidate in entries: if candidate.is_symlink() or _SAFE_RESULT_ID.fullmatch(candidate.name) is None: continue try: descriptor = _read_result_descriptor(candidate) except (OSError, SessionIntegrityError): continue if descriptor.session_id == session_id: matches.append(descriptor) ordered = tuple( sorted( matches, key=lambda value: (value.created_at_utc, value.result_id), reverse=True, ) ) self._descriptors_by_session[session_id] = ordered return ordered[0] if ordered else None def _latest(self, session_id: str) -> IntegratedPerceptionResult | None: latest_descriptor = self._latest_descriptor(session_id) if latest_descriptor is None: return None with self._catalog_lock: if session_id in self._latest_by_session: return self._latest_by_session[session_id] descriptors = self._descriptors_by_session.get(session_id, ()) for descriptor in descriptors: try: value = validate_integrated_perception_result( self.jobs_root / descriptor.job_id, descriptor.result_root, self.lidar_packs_root, ) except (OSError, SessionIntegrityError): continue if ( value.accepted and value.publication_scope == "recorded-integrated-realtime-qualification-only" ): with self._catalog_lock: self._latest_by_session[session_id] = value return value with self._catalog_lock: self._latest_by_session[session_id] = None return None def _render( result: IntegratedPerceptionResult, *, application_id: str, recording_id: str, ffmpeg_path: Path, temporary_root: Path, ) -> bytes: fusion_rows = _read_rows(result.fusion_path, result.result_root, FUSION_SCHEMA) with np.load(result.arrays_path, allow_pickle=False) as arrays: frame_times = arrays["frame_times_ns"] semantic_indices = arrays["semantic_frame_indices"] semantic_masks = arrays["semantic_masks"] semantic_by_frame = { int(index): mask for index, mask in zip(semantic_indices, semantic_masks, strict=True) } support_offsets = arrays["support_offsets"] support = arrays["support_points"] support_colors = arrays["support_colors"] box_offsets = arrays["box_offsets"] centers = arrays["box_centers"] half_sizes = arrays["box_half_sizes"] quaternions = arrays["box_quaternions"] box_colors = arrays["box_colors"] cuboid_presentation = _CuboidPresentationState() recording = rr.RecordingStream(application_id, recording_id=recording_id) stream = rr.binary_stream(recording) source_path: Path | None = None proxy_path: Path | None = None try: recording.log("/world", rr.ViewCoordinates.RIGHT_HAND_Z_UP, static=True) recording.log( "/world/perception/contract", rr.TextDocument( "LAB E10 integrated source-paced replay. Generic AI and host-arrival " "synchronization are diagnostic, not navigation or safety accepted." ), static=True, ) source_path = _camera_source_file( result.job, result.source_start_frame_index + result.frame_count, temporary_root ) source_end_frame_index = result.source_start_frame_index + result.frame_count - 1 proxy_path = _camera_proxy_file( source_path, result.source_start_frame_index, source_end_frame_index, ffmpeg_path, temporary_root, ) video = rr.AssetVideo(path=proxy_path) video_timestamps = video.read_frame_timestamps_nanos() if len(video_timestamps) != len(frame_times): raise RecordedPerceptionOverlayError( "integrated perception video frame count changed" ) recording.log( "/perception/camera/image", video, static=True, ) for index, timestamp in enumerate(frame_times): recording.set_time(SESSION_TIMELINE, duration=np.timedelta64(int(timestamp), "ns")) recording.log( "/perception/camera/image", rr.VideoFrameReference(nanoseconds=int(video_timestamps[index])), ) mask = semantic_by_frame.get(index) if mask is not None: recording.log( "/perception/camera/segmentation", rr.SegmentationImage(mask), ) objects = fusion_rows[index]["objects"] if objects: recording.log( "/perception/camera/detections", rr.Boxes2D( array=[item["bbox_xyxy"] for item in objects], array_format=rr.Box2DFormat.XYXY, labels=[ f"#{item['track_id']} {item['label']} · {float(item['score']):.0%}" for item in objects ], show_labels=True, ), ) else: recording.log("/perception/camera/detections", rr.Clear(recursive=False)) point_start, point_end = ( int(support_offsets[index]), int(support_offsets[index + 1]), ) if point_end > point_start: recording.log( "/world/perception/support", rr.Points3D( support[point_start:point_end], colors=support_colors[point_start:point_end], radii=rr.Radius.ui_points(3.0), ), ) else: recording.log("/world/perception/support", rr.Clear(recursive=False)) box_start, box_end = int(box_offsets[index]), int(box_offsets[index + 1]) presented_cuboids = cuboid_presentation.update( int(timestamp), objects, centers[box_start:box_end], half_sizes[box_start:box_end], quaternions[box_start:box_end], box_colors[box_start:box_end], ) if presented_cuboids is not None: ( presented_centers, presented_half_sizes, presented_quaternions, presented_colors, presented_labels, ) = presented_cuboids recording.log( "/world/perception/boxes3d", rr.Boxes3D( centers=presented_centers, half_sizes=presented_half_sizes, quaternions=presented_quaternions, colors=presented_colors, labels=presented_labels, fill_mode=FillMode.Solid, show_labels=True, ), ) else: recording.log("/world/perception/boxes3d", rr.Clear(recursive=False)) payload = stream.read(flush=True, flush_timeout_sec=300.0) except RecordedPerceptionOverlayError: raise except Exception as exc: raise RecordedPerceptionOverlayError( "failed to serialize integrated perception" ) from exc finally: with suppress(Exception): recording.disconnect() if source_path is not None: source_path.unlink(missing_ok=True) if proxy_path is not None: proxy_path.unlink(missing_ok=True) if payload is None or not payload.startswith(b"RRF2"): raise RecordedPerceptionOverlayError("integrated perception Rerun stream is invalid") return payload def _camera_source_file(job: CameraComputeJob, segment_count: int, root: Path) -> Path: epoch = job.job_root / "input" / "camera" / job.source_id / f"epoch-{job.codec_epoch}" paths = [epoch / "init.mp4"] + [ epoch / "segments" / f"{sequence}.m4s" for sequence in range(1, segment_count + 1) ] total = sum(path.stat().st_size for path in paths) if total <= 0 or total > MAX_SOURCE_BYTES: raise RecordedPerceptionOverlayError( "integrated perception camera source is outside bounds" ) descriptor, name = tempfile.mkstemp(prefix=".e10-camera-", suffix=".mp4", dir=root) path = Path(name) try: with os.fdopen(descriptor, "wb") as output: for source in paths: with source.open("rb") as stream: shutil.copyfileobj(stream, output, length=1024 * 1024) output.flush() os.fsync(output.fileno()) return path except BaseException: path.unlink(missing_ok=True) raise def _camera_proxy_file( source: Path, start_frame_index: int, end_frame_index: int, ffmpeg_path: Path, root: Path, ) -> Path: descriptor, name = tempfile.mkstemp(prefix=".e10-camera-proxy-", suffix=".mp4", dir=root) os.close(descriptor) path = Path(name) path.unlink(missing_ok=True) frame_filter = f"select=between(n\\,{start_frame_index}\\,{end_frame_index}),setpts=N/(10*TB)" try: completed = subprocess.run( [ str(ffmpeg_path), "-hide_banner", "-loglevel", "error", "-i", str(source), "-vf", frame_filter, "-an", "-c:v", "libx264", "-preset", "veryfast", "-crf", "28", "-g", "20", "-keyint_min", "20", "-pix_fmt", "yuv420p", "-movflags", "+faststart", str(path), ], check=False, capture_output=True, timeout=300, ) if completed.returncode != 0 or not path.is_file() or path.stat().st_size <= 0: raise RecordedPerceptionOverlayError( f"integrated perception video proxy failed: {completed.stderr[-1000:]!r}" ) os.chmod(path, 0o600) return path except BaseException: path.unlink(missing_ok=True) raise def _validate_pack( root: Path, job: CameraComputeJob, result_identity: dict[str, Any], count: int, start: int, end: int, ) -> None: if not root.is_dir() or _SAFE_PACK_ID.fullmatch(root.name) is None: raise SessionIntegrityError("integrated perception LiDAR pack root is invalid") manifest = _read_object(root / "manifest.json", root) identity = manifest.get("identity") identity_sha256 = manifest.get("identity_sha256") arrays_path = root / "lidar-pack.npz" artifact = manifest.get("artifact") if ( manifest.get("schema_version") != PACK_SCHEMA or manifest.get("pack_id") != root.name or not isinstance(identity, dict) or identity.get("schema_version") != PACK_SCHEMA or identity.get("job_id") != job.job_id or identity.get("input_sha256") != job.input_sha256 or identity.get("calibration_sha256") != result_identity["configuration"]["profile"]["source"]["calibration_sha256"] or identity.get("frame_count") != count or identity.get("source_start_frame_index") != start or identity.get("source_end_frame_index") != end or not isinstance(identity_sha256, str) or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 or root.name != f"e10-lidar-pack-{identity_sha256}" or not isinstance(artifact, dict) or artifact.get("path") != arrays_path.name or artifact.get("byte_length") != arrays_path.stat().st_size or artifact.get("sha256") != _sha256(arrays_path) ): raise SessionIntegrityError("integrated perception LiDAR pack is inconsistent") def _validate_artifacts(root: Path, raw: object) -> dict[str, Path]: expected = { "e10-semantic-frames": ("semantic-frames.jsonl", SEMANTIC_SCHEMA), "e10-fusion-frames": ("fusion-frames.jsonl", FUSION_SCHEMA), "e10-world-state": ("world-state.jsonl", WORLD_SCHEMA), "e10-transient-perception": ("transient-perception.npz", None), "worker-gpu-telemetry": ("gpu-telemetry.jsonl", None), "e10-run-report": ("run-report.json", REPORT_SCHEMA), } if not isinstance(raw, list) or len(raw) != len(expected): raise SessionIntegrityError("integrated perception artifact set is incomplete") result = {} for value in raw: kind = value.get("kind") if isinstance(value, dict) else None if not isinstance(kind, str) or kind not in expected or kind in result: raise SessionIntegrityError("integrated perception artifact descriptor is invalid") name, schema = expected[kind] path = root / name metadata = _confined_file(path, root) if ( value.get("path") != name or value.get("schema_version") != schema or value.get("byte_length") != metadata.st_size or value.get("sha256") != _sha256(path) ): raise SessionIntegrityError("integrated perception artifact identity changed") result[kind] = path return result def _validate_arrays( path: Path, count: int, semantic_rows: list[dict[str, Any]], fusion_rows: list[dict[str, Any]], ) -> None: with np.load(path, allow_pickle=False) as arrays: required = { "frame_times_ns", "semantic_frame_indices", "semantic_masks", "support_offsets", "support_points", "support_colors", "box_offsets", "box_centers", "box_half_sizes", "box_quaternions", "box_colors", } if set(arrays.files) != required: raise SessionIntegrityError("integrated perception array set changed") times = arrays["frame_times_ns"] semantic_indices = arrays["semantic_frame_indices"] masks = arrays["semantic_masks"] support_offsets = arrays["support_offsets"] support = arrays["support_points"] support_colors = arrays["support_colors"] box_offsets = arrays["box_offsets"] centers = arrays["box_centers"] half_sizes = arrays["box_half_sizes"] quaternions = arrays["box_quaternions"] colors = arrays["box_colors"] expected_boxes = sum( sum( str(item.get("cuboid_status", "")).startswith("accepted-") for item in row["objects"] ) for row in fusion_rows ) if ( times.dtype != np.int64 or times.shape != (count,) or np.any(np.diff(times) <= 0) or semantic_indices.dtype != np.int64 or semantic_indices.shape != (len(semantic_rows),) or not np.array_equal(semantic_indices, [row["frame_index"] for row in semantic_rows]) or masks.dtype != np.uint8 or masks.shape != (len(semantic_rows), 600, 800) or support_offsets.shape != (count + 1,) or support.shape[1:] != (3,) or support_colors.shape != support.shape or box_offsets.shape != (count + 1,) or centers.shape != (expected_boxes, 3) or half_sizes.shape != centers.shape or quaternions.shape != (expected_boxes, 4) or colors.shape != (expected_boxes, 4) or int(support_offsets[-1]) != support.shape[0] or int(box_offsets[-1]) != expected_boxes or np.any(np.diff(support_offsets) < 0) or np.any(np.diff(box_offsets) < 0) or not np.isfinite(support).all() or not np.isfinite(centers).all() or not np.isfinite(half_sizes).all() or np.any(half_sizes <= 0) ): raise SessionIntegrityError("integrated perception arrays are inconsistent") def _read_rows(path: Path, root: Path, schema: str) -> list[dict[str, Any]]: _confined_file(path, root) rows = [] with path.open(encoding="utf-8") as stream: for line in stream: if len(line.encode()) > MAX_LINE_BYTES: raise SessionIntegrityError("integrated perception row is oversized") value = json.loads(line) if not isinstance(value, dict) or value.get("schema_version") != schema: raise SessionIntegrityError("integrated perception row schema changed") rows.append(value) return rows def _read_result_descriptor(result_root: Path) -> _ResultDescriptor: root = result_root.expanduser().resolve(strict=True) if not root.is_dir() or _SAFE_RESULT_ID.fullmatch(root.name) is None: raise SessionIntegrityError("integrated perception result descriptor is invalid") result_path = root / "result.json" result = _read_object(result_path, root) identity = result.get("identity") identity_sha256 = result.get("identity_sha256") created_at_utc = result.get("created_at_utc") session_id = identity.get("session_id") if isinstance(identity, dict) else None job_id = identity.get("job_id") if isinstance(identity, dict) else None if ( result.get("schema_version") != RESULT_SCHEMA or result.get("result_id") != root.name or not isinstance(identity, dict) or identity.get("schema_version") != IDENTITY_SCHEMA or not isinstance(identity_sha256, str) or _SHA256.fullmatch(identity_sha256) is None or root.name != f"e10-integrated-perception-{identity_sha256}" or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 or result.get("acceptance_state") != "accepted" or result.get("publication_scope") != "recorded-integrated-realtime-qualification-only" or not isinstance(session_id, str) or _SAFE_RECORDING_ID.fullmatch(session_id) is None or not isinstance(job_id, str) or _SAFE_JOB_ID.fullmatch(job_id) is None or not isinstance(created_at_utc, str) or not 1 <= len(created_at_utc) <= 64 ): raise SessionIntegrityError("integrated perception result is not admitted") return _ResultDescriptor( result_id=root.name, result_root=root, result_json_sha256=_sha256(result_path), session_id=session_id, job_id=job_id, created_at_utc=created_at_utc, ) def _read_cache_artifact( output: Path, sidecar: Path, *, result_id: str, recording_id: str, ) -> RecordedPerceptionOverlayArtifact | None: try: value = _read_object(sidecar, sidecar.parent) metadata = _confined_file(output, sidecar.parent) with output.open("rb") as stream: magic = stream.read(4) digest = _sha256(output) except (OSError, SessionIntegrityError): return None if ( value.get("schema_version") != OVERLAY_CACHE_SCHEMA or value.get("renderer_version") != OVERLAY_RENDERER_VERSION or value.get("result_id") != result_id or value.get("recording_id") != recording_id or value.get("byte_length") != metadata.st_size or value.get("sha256") != digest or magic != b"RRF2" ): return None return RecordedPerceptionOverlayArtifact( path=output.resolve(strict=True), byte_length=metadata.st_size, sha256=digest, ) def _read_object(path: Path, root: Path) -> dict[str, Any]: metadata = _confined_file(path, root) if not 0 < metadata.st_size <= MAX_JSON_BYTES: raise SessionIntegrityError("integrated perception JSON is outside bounds") try: value = json.loads(path.read_text(encoding="utf-8-sig")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: raise SessionIntegrityError("integrated perception JSON is unavailable") from exc if not isinstance(value, dict): raise SessionIntegrityError("integrated perception JSON is not an object") return value def _confined_file(path: Path, root: Path) -> os.stat_result: try: resolved_root = root.resolve(strict=True) resolved = path.resolve(strict=True) metadata = path.lstat() except OSError as exc: raise SessionIntegrityError("integrated perception artifact is unavailable") from exc if ( stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode) or not resolved.is_relative_to(resolved_root) ): raise SessionIntegrityError("integrated perception artifact is not confined") return metadata def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: while chunk := stream.read(1024 * 1024): digest.update(chunk) return digest.hexdigest() def _canonical_json(value: object) -> bytes: return json.dumps( value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False, ).encode() def _finite(value: object) -> TypeGuard[int | float]: return ( isinstance(value, int | float) and not isinstance(value, bool) and bool(np.isfinite(value)) ) def _private_directory(path: Path) -> Path: path.mkdir(mode=0o700, parents=True, exist_ok=True) if path.is_symlink() or not path.is_dir(): raise RecordedPerceptionOverlayError("integrated perception cache root is invalid") with suppress(OSError): os.chmod(path, 0o700) return path.resolve(strict=True) def _private_child(root: Path, name: str) -> Path: child = root / name child.mkdir(mode=0o700, exist_ok=True) if child.is_symlink() or not child.is_dir() or child.resolve(strict=True).parent != root: raise RecordedPerceptionOverlayError("integrated perception cache child is invalid") with suppress(OSError): os.chmod(child, 0o700) return child.resolve(strict=True)