feat(perception): integrate calibrated operator pipeline
Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,925 @@
|
||||
"""Validate and project accepted LAB E10 integrated perception results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol, TypeGuard
|
||||
|
||||
import numpy as np
|
||||
import rerun as rr
|
||||
from rerun.components import FillMode
|
||||
|
||||
from k1link.artifacts import write_json_atomic
|
||||
from k1link.sessions import SessionIntegrityError
|
||||
|
||||
from .jobs import CameraComputeJob, validate_camera_compute_job
|
||||
from .results import RecordedPerceptionOverlayError
|
||||
|
||||
RESULT_SCHEMA = "missioncore.e10-integrated-perception-result/v1"
|
||||
IDENTITY_SCHEMA = "missioncore.e10-integrated-perception-identity/v1"
|
||||
REPORT_SCHEMA = "missioncore.e10-integrated-perception-report/v1"
|
||||
PACK_SCHEMA = "missioncore.e10-lidar-replay-pack/v1"
|
||||
SEMANTIC_SCHEMA = "missioncore.e10-semantic-frame/v1"
|
||||
FUSION_SCHEMA = "missioncore.e10-fusion-frame/v1"
|
||||
WORLD_SCHEMA = "missioncore.live-perception-world-state/v1"
|
||||
SESSION_TIMELINE = "session_time"
|
||||
MAX_SCAN = 512
|
||||
MAX_JSON_BYTES = 64 * 1024 * 1024
|
||||
MAX_LINE_BYTES = 4 * 1024 * 1024
|
||||
MAX_SOURCE_BYTES = 512 * 1024 * 1024
|
||||
OVERLAY_RENDERER_VERSION = "4"
|
||||
CUBOID_PRESENTATION_HOLD_NS = 500_000_000
|
||||
|
||||
_SAFE_RESULT_ID = re.compile(r"^e10-integrated-perception-[a-f0-9]{64}$")
|
||||
_SAFE_PACK_ID = re.compile(r"^e10-lidar-pack-[a-f0-9]{64}$")
|
||||
_SAFE_RECORDING_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class IntegratedPerceptionResult:
|
||||
result_id: str
|
||||
result_root: Path
|
||||
job: CameraComputeJob
|
||||
pack_root: Path
|
||||
created_at_utc: str
|
||||
accepted: bool
|
||||
publication_scope: str
|
||||
source_start_frame_index: int
|
||||
frame_count: int
|
||||
timeline_start_seconds: float
|
||||
timeline_end_seconds: float
|
||||
semantic_path: Path
|
||||
fusion_path: Path
|
||||
world_path: Path
|
||||
arrays_path: Path
|
||||
report_path: Path
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PresentedCuboid:
|
||||
observed_ns: int
|
||||
track_id: int
|
||||
label: str
|
||||
association_group: str
|
||||
distance_m: float
|
||||
support_points: int
|
||||
center: np.ndarray
|
||||
half_size: np.ndarray
|
||||
quaternion: np.ndarray
|
||||
color: np.ndarray
|
||||
|
||||
|
||||
class _CuboidPresentationState:
|
||||
"""Hold accepted cuboids briefly for operator presentation only.
|
||||
|
||||
The persisted world-state remains fail-closed and frame-exact. This bounded
|
||||
latest-at projection only prevents one rejected LiDAR association from
|
||||
visually clearing an otherwise stable tracked object for a single frame.
|
||||
"""
|
||||
|
||||
def __init__(self, hold_ns: int = CUBOID_PRESENTATION_HOLD_NS) -> None:
|
||||
if hold_ns <= 0:
|
||||
raise ValueError("cuboid presentation hold must be positive")
|
||||
self._hold_ns = hold_ns
|
||||
self._latest: dict[int, _PresentedCuboid] = {}
|
||||
|
||||
def update(
|
||||
self,
|
||||
timestamp_ns: int,
|
||||
objects: list[dict[str, Any]],
|
||||
centers: np.ndarray,
|
||||
half_sizes: np.ndarray,
|
||||
quaternions: np.ndarray,
|
||||
colors: np.ndarray,
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, list[str]] | None:
|
||||
accepted = [
|
||||
item
|
||||
for item in objects
|
||||
if str(item.get("cuboid_status", "")).startswith("accepted-")
|
||||
]
|
||||
if not (
|
||||
len(accepted)
|
||||
== len(centers)
|
||||
== len(half_sizes)
|
||||
== len(quaternions)
|
||||
== len(colors)
|
||||
):
|
||||
raise RecordedPerceptionOverlayError(
|
||||
"integrated perception cuboid presentation arrays are inconsistent"
|
||||
)
|
||||
for item, center, half_size, quaternion, color in zip(
|
||||
accepted,
|
||||
centers,
|
||||
half_sizes,
|
||||
quaternions,
|
||||
colors,
|
||||
strict=True,
|
||||
):
|
||||
track_id = item.get("track_id")
|
||||
if not isinstance(track_id, int) or isinstance(track_id, bool):
|
||||
raise RecordedPerceptionOverlayError(
|
||||
"integrated perception cuboid track identity is invalid"
|
||||
)
|
||||
self._latest[track_id] = _PresentedCuboid(
|
||||
observed_ns=timestamp_ns,
|
||||
track_id=track_id,
|
||||
label=str(item.get("label", "object")),
|
||||
association_group=str(item.get("association_group", "object")),
|
||||
distance_m=float(item["distance_smoothed_m"]),
|
||||
support_points=int(item["clustered_points"]),
|
||||
center=np.asarray(center).copy(),
|
||||
half_size=np.asarray(half_size).copy(),
|
||||
quaternion=np.asarray(quaternion).copy(),
|
||||
color=np.asarray(color).copy(),
|
||||
)
|
||||
|
||||
expired = [
|
||||
track_id
|
||||
for track_id, cuboid in self._latest.items()
|
||||
if timestamp_ns - cuboid.observed_ns > self._hold_ns
|
||||
]
|
||||
for track_id in expired:
|
||||
del self._latest[track_id]
|
||||
if not self._latest:
|
||||
return None
|
||||
|
||||
presented = sorted(self._latest.values(), key=lambda cuboid: cuboid.track_id)
|
||||
presented_colors: list[np.ndarray] = []
|
||||
labels: list[str] = []
|
||||
for cuboid in presented:
|
||||
age_ns = max(0, timestamp_ns - cuboid.observed_ns)
|
||||
color = cuboid.color.copy()
|
||||
if age_ns > 0 and color.shape == (4,):
|
||||
fade = min(1.0, age_ns / self._hold_ns)
|
||||
color[3] = max(24, round(float(color[3]) * (1.0 - 0.55 * fade)))
|
||||
presented_colors.append(color)
|
||||
age_label = "" if age_ns == 0 else f" · hold {age_ns / 1_000_000:.0f} ms"
|
||||
labels.append(
|
||||
f"{cuboid.association_group} #{cuboid.track_id} {cuboid.label} · "
|
||||
f"{cuboid.distance_m:.1f} m · {cuboid.support_points} pts{age_label}"
|
||||
)
|
||||
return (
|
||||
np.stack([cuboid.center for cuboid in presented]),
|
||||
np.stack([cuboid.half_size for cuboid in presented]),
|
||||
np.stack([cuboid.quaternion for cuboid in presented]),
|
||||
np.stack(presented_colors),
|
||||
labels,
|
||||
)
|
||||
|
||||
|
||||
class _OverlayProvider(Protocol):
|
||||
def render(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
application_id: str,
|
||||
recording_id: str,
|
||||
) -> bytes | None: ...
|
||||
|
||||
|
||||
def validate_integrated_perception_result(
|
||||
job_root: Path,
|
||||
result_root: Path,
|
||||
lidar_packs_root: Path,
|
||||
) -> IntegratedPerceptionResult:
|
||||
job = validate_camera_compute_job(job_root)
|
||||
root = result_root.expanduser().resolve(strict=True)
|
||||
if not root.is_dir() or _SAFE_RESULT_ID.fullmatch(root.name) is None:
|
||||
raise SessionIntegrityError("integrated perception root is invalid")
|
||||
result = _read_object(root / "result.json", root)
|
||||
identity = result.get("identity")
|
||||
identity_sha256 = result.get("identity_sha256")
|
||||
if (
|
||||
result.get("schema_version") != RESULT_SCHEMA
|
||||
or result.get("result_id") != root.name
|
||||
or not isinstance(identity, dict)
|
||||
or identity.get("schema_version") != IDENTITY_SCHEMA
|
||||
or not isinstance(identity_sha256, str)
|
||||
or _SHA256.fullmatch(identity_sha256) is None
|
||||
or root.name != f"e10-integrated-perception-{identity_sha256}"
|
||||
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
||||
or identity.get("job_id") != job.job_id
|
||||
or identity.get("input_sha256") != job.input_sha256
|
||||
or identity.get("session_id") != job.session_id
|
||||
or identity.get("source_id") != job.source_id
|
||||
or result.get("ground_truth") is not False
|
||||
or result.get("publication_scope")
|
||||
not in {
|
||||
"recorded-integrated-realtime-qualification-only",
|
||||
"recorded-integrated-semantic-loss-negative-control-only",
|
||||
}
|
||||
or result.get("acceptance_state") not in {"accepted", "rejected"}
|
||||
):
|
||||
raise SessionIntegrityError("integrated perception identity is inconsistent")
|
||||
selection = identity.get("selection")
|
||||
if not isinstance(selection, dict):
|
||||
raise SessionIntegrityError("integrated perception selection is missing")
|
||||
count = selection.get("frame_count")
|
||||
start = selection.get("source_start_frame_index")
|
||||
end = selection.get("source_end_frame_index")
|
||||
timeline_start = selection.get("timeline_start_seconds")
|
||||
timeline_end = selection.get("timeline_end_seconds")
|
||||
if (
|
||||
not isinstance(count, int)
|
||||
or isinstance(count, bool)
|
||||
or count < 2
|
||||
or not isinstance(start, int)
|
||||
or isinstance(start, bool)
|
||||
or not isinstance(end, int)
|
||||
or isinstance(end, bool)
|
||||
or end - start + 1 != count
|
||||
or start < 0
|
||||
or end >= job.segment_count
|
||||
or not _finite(timeline_start)
|
||||
or not _finite(timeline_end)
|
||||
or float(timeline_end) <= float(timeline_start)
|
||||
or result.get("frames_processed") != count
|
||||
):
|
||||
raise SessionIntegrityError("integrated perception selection is invalid")
|
||||
pack_id = identity.get("lidar_pack_id")
|
||||
if not isinstance(pack_id, str) or _SAFE_PACK_ID.fullmatch(pack_id) is None:
|
||||
raise SessionIntegrityError("integrated perception LiDAR pack binding is invalid")
|
||||
pack_root = lidar_packs_root.expanduser().resolve(strict=True) / pack_id
|
||||
_validate_pack(pack_root, job, identity, count, start, end)
|
||||
artifacts = _validate_artifacts(root, result.get("artifacts"))
|
||||
semantic_path = artifacts["e10-semantic-frames"]
|
||||
fusion_path = artifacts["e10-fusion-frames"]
|
||||
world_path = artifacts["e10-world-state"]
|
||||
arrays_path = artifacts["e10-transient-perception"]
|
||||
report_path = artifacts["e10-run-report"]
|
||||
semantic_rows = _read_rows(semantic_path, root, SEMANTIC_SCHEMA)
|
||||
fusion_rows = _read_rows(fusion_path, root, FUSION_SCHEMA)
|
||||
world_rows = _read_rows(world_path, root, WORLD_SCHEMA)
|
||||
if len(fusion_rows) != count or len(world_rows) != count or not semantic_rows:
|
||||
raise SessionIntegrityError("integrated perception frame counts are incomplete")
|
||||
for index, (fusion, world) in enumerate(zip(fusion_rows, world_rows, strict=True)):
|
||||
source_index = start + index
|
||||
if (
|
||||
fusion.get("frame_index") != index
|
||||
or fusion.get("source_frame_index") != source_index
|
||||
or world.get("frame_index") != index
|
||||
or world.get("source_frame_index") != source_index
|
||||
or not isinstance(fusion.get("objects"), list)
|
||||
or not isinstance(world.get("objects"), list)
|
||||
or world.get("object_count") != len(world["objects"])
|
||||
or not isinstance(world.get("delivery"), dict)
|
||||
):
|
||||
raise SessionIntegrityError("integrated perception frame identity changed")
|
||||
semantic_indices: list[int] = []
|
||||
for row in semantic_rows:
|
||||
frame_index = row.get("frame_index")
|
||||
if not isinstance(frame_index, int) or isinstance(frame_index, bool):
|
||||
raise SessionIntegrityError("integrated perception semantic timeline is invalid")
|
||||
semantic_indices.append(frame_index)
|
||||
if (
|
||||
semantic_indices != sorted(set(semantic_indices))
|
||||
or semantic_indices[0] != 0
|
||||
or semantic_indices[-1] >= count
|
||||
):
|
||||
raise SessionIntegrityError("integrated perception semantic timeline is invalid")
|
||||
_validate_arrays(arrays_path, count, semantic_rows, fusion_rows)
|
||||
report = _read_object(report_path, root)
|
||||
acceptance = report.get("acceptance")
|
||||
if (
|
||||
report.get("schema_version") != REPORT_SCHEMA
|
||||
or report.get("result_id") != root.name
|
||||
or report.get("identity") != identity
|
||||
or report.get("ground_truth") is not False
|
||||
or not isinstance(acceptance, dict)
|
||||
or acceptance.get("accepted") is not (result.get("acceptance_state") == "accepted")
|
||||
or acceptance.get("navigation_or_safety_accepted") is not False
|
||||
):
|
||||
raise SessionIntegrityError("integrated perception report is inconsistent")
|
||||
created = result.get("created_at_utc")
|
||||
if not isinstance(created, str) or not 1 <= len(created) <= 64:
|
||||
raise SessionIntegrityError("integrated perception creation time is invalid")
|
||||
return IntegratedPerceptionResult(
|
||||
result_id=root.name,
|
||||
result_root=root,
|
||||
job=job,
|
||||
pack_root=pack_root,
|
||||
created_at_utc=created,
|
||||
accepted=result.get("acceptance_state") == "accepted",
|
||||
publication_scope=str(result["publication_scope"]),
|
||||
source_start_frame_index=start,
|
||||
frame_count=count,
|
||||
timeline_start_seconds=float(timeline_start),
|
||||
timeline_end_seconds=float(timeline_end),
|
||||
semantic_path=semantic_path,
|
||||
fusion_path=fusion_path,
|
||||
world_path=world_path,
|
||||
arrays_path=arrays_path,
|
||||
report_path=report_path,
|
||||
)
|
||||
|
||||
|
||||
class IntegratedPerceptionOverlayStore:
|
||||
"""Prefer the newest accepted E10 result for a recorded session."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
jobs_root: Path,
|
||||
results_root: Path,
|
||||
lidar_packs_root: Path,
|
||||
cache_root: Path,
|
||||
ffmpeg_path: Path,
|
||||
) -> None:
|
||||
self.jobs_root = jobs_root.expanduser().absolute()
|
||||
self.results_root = results_root.expanduser().absolute()
|
||||
self.lidar_packs_root = lidar_packs_root.expanduser().absolute()
|
||||
self.cache_root = cache_root.expanduser().absolute()
|
||||
self.ffmpeg_path = ffmpeg_path.expanduser().absolute()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def render(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
application_id: str,
|
||||
recording_id: str,
|
||||
) -> bytes | None:
|
||||
if application_id != "nodedc_mission_core_recorded":
|
||||
raise ValueError("integrated perception application id is invalid")
|
||||
if (
|
||||
_SAFE_RECORDING_ID.fullmatch(session_id) is None
|
||||
or _SAFE_RECORDING_ID.fullmatch(recording_id) is None
|
||||
):
|
||||
raise ValueError("integrated perception recording id is invalid")
|
||||
with self._lock:
|
||||
result = self._latest(session_id)
|
||||
if result is None:
|
||||
return None
|
||||
cache = _private_child(
|
||||
_private_child(_private_directory(self.cache_root), session_id),
|
||||
result.result_id,
|
||||
)
|
||||
output = cache / f"{recording_id}.rrd"
|
||||
sidecar = output.with_suffix(".rrd.cache.json")
|
||||
cached = _read_cache(output, sidecar, result, recording_id)
|
||||
if cached is not None:
|
||||
return cached
|
||||
payload = _render(
|
||||
result,
|
||||
application_id=application_id,
|
||||
recording_id=recording_id,
|
||||
ffmpeg_path=self.ffmpeg_path,
|
||||
temporary_root=cache,
|
||||
)
|
||||
temporary = output.with_name(f".{output.name}.{os.getpid()}.tmp")
|
||||
try:
|
||||
with temporary.open("xb") as stream:
|
||||
stream.write(payload)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.chmod(temporary, 0o600)
|
||||
os.replace(temporary, output)
|
||||
write_json_atomic(
|
||||
sidecar,
|
||||
{
|
||||
"schema_version": "missioncore.e10-overlay-cache/v1",
|
||||
"renderer_version": OVERLAY_RENDERER_VERSION,
|
||||
"result_id": result.result_id,
|
||||
"recording_id": recording_id,
|
||||
"byte_length": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
},
|
||||
)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
return payload
|
||||
|
||||
def _latest(self, session_id: str) -> IntegratedPerceptionResult | None:
|
||||
try:
|
||||
jobs = sorted(self.jobs_root.iterdir())
|
||||
candidates = sorted(self.results_root.iterdir())
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
if len(jobs) > MAX_SCAN or len(candidates) > MAX_SCAN:
|
||||
raise RecordedPerceptionOverlayError("integrated perception catalog is outside bounds")
|
||||
matches = []
|
||||
for job_root in jobs:
|
||||
if job_root.is_symlink():
|
||||
continue
|
||||
try:
|
||||
job = validate_camera_compute_job(job_root)
|
||||
except (OSError, SessionIntegrityError):
|
||||
continue
|
||||
if job.session_id != session_id:
|
||||
continue
|
||||
for candidate in candidates:
|
||||
if candidate.is_symlink() or _SAFE_RESULT_ID.fullmatch(candidate.name) is None:
|
||||
continue
|
||||
try:
|
||||
value = validate_integrated_perception_result(
|
||||
job_root,
|
||||
candidate,
|
||||
self.lidar_packs_root,
|
||||
)
|
||||
except (OSError, SessionIntegrityError):
|
||||
continue
|
||||
if (
|
||||
value.accepted
|
||||
and value.publication_scope == "recorded-integrated-realtime-qualification-only"
|
||||
):
|
||||
matches.append(value)
|
||||
if not matches:
|
||||
return None
|
||||
return max(matches, key=lambda value: (value.created_at_utc, value.result_id))
|
||||
|
||||
|
||||
def _render(
|
||||
result: IntegratedPerceptionResult,
|
||||
*,
|
||||
application_id: str,
|
||||
recording_id: str,
|
||||
ffmpeg_path: Path,
|
||||
temporary_root: Path,
|
||||
) -> bytes:
|
||||
fusion_rows = _read_rows(result.fusion_path, result.result_root, FUSION_SCHEMA)
|
||||
with np.load(result.arrays_path, allow_pickle=False) as arrays:
|
||||
frame_times = arrays["frame_times_ns"]
|
||||
semantic_indices = arrays["semantic_frame_indices"]
|
||||
semantic_masks = arrays["semantic_masks"]
|
||||
semantic_by_frame = {
|
||||
int(index): mask for index, mask in zip(semantic_indices, semantic_masks, strict=True)
|
||||
}
|
||||
support_offsets = arrays["support_offsets"]
|
||||
support = arrays["support_points"]
|
||||
support_colors = arrays["support_colors"]
|
||||
box_offsets = arrays["box_offsets"]
|
||||
centers = arrays["box_centers"]
|
||||
half_sizes = arrays["box_half_sizes"]
|
||||
quaternions = arrays["box_quaternions"]
|
||||
box_colors = arrays["box_colors"]
|
||||
cuboid_presentation = _CuboidPresentationState()
|
||||
recording = rr.RecordingStream(application_id, recording_id=recording_id)
|
||||
stream = rr.binary_stream(recording)
|
||||
source_path: Path | None = None
|
||||
proxy_path: Path | None = None
|
||||
try:
|
||||
recording.log("/world", rr.ViewCoordinates.RIGHT_HAND_Z_UP, static=True)
|
||||
recording.log(
|
||||
"/world/perception/contract",
|
||||
rr.TextDocument(
|
||||
"LAB E10 integrated source-paced replay. Generic AI and host-arrival "
|
||||
"synchronization are diagnostic, not navigation or safety accepted."
|
||||
),
|
||||
static=True,
|
||||
)
|
||||
source_path = _camera_source_file(
|
||||
result.job, result.source_start_frame_index + result.frame_count, temporary_root
|
||||
)
|
||||
source_end_frame_index = result.source_start_frame_index + result.frame_count - 1
|
||||
proxy_path = _camera_proxy_file(
|
||||
source_path,
|
||||
result.source_start_frame_index,
|
||||
source_end_frame_index,
|
||||
ffmpeg_path,
|
||||
temporary_root,
|
||||
)
|
||||
video = rr.AssetVideo(path=proxy_path)
|
||||
video_timestamps = video.read_frame_timestamps_nanos()
|
||||
if len(video_timestamps) != len(frame_times):
|
||||
raise RecordedPerceptionOverlayError(
|
||||
"integrated perception video frame count changed"
|
||||
)
|
||||
recording.log(
|
||||
"/perception/camera/image",
|
||||
video,
|
||||
static=True,
|
||||
)
|
||||
for index, timestamp in enumerate(frame_times):
|
||||
recording.set_time(SESSION_TIMELINE, duration=np.timedelta64(int(timestamp), "ns"))
|
||||
recording.log(
|
||||
"/perception/camera/image",
|
||||
rr.VideoFrameReference(nanoseconds=int(video_timestamps[index])),
|
||||
)
|
||||
mask = semantic_by_frame.get(index)
|
||||
if mask is not None:
|
||||
recording.log(
|
||||
"/perception/camera/segmentation",
|
||||
rr.SegmentationImage(mask),
|
||||
)
|
||||
objects = fusion_rows[index]["objects"]
|
||||
if objects:
|
||||
recording.log(
|
||||
"/perception/camera/detections",
|
||||
rr.Boxes2D(
|
||||
array=[item["bbox_xyxy"] for item in objects],
|
||||
array_format=rr.Box2DFormat.XYXY,
|
||||
labels=[
|
||||
f"#{item['track_id']} {item['label']} · {float(item['score']):.0%}"
|
||||
for item in objects
|
||||
],
|
||||
show_labels=True,
|
||||
),
|
||||
)
|
||||
else:
|
||||
recording.log("/perception/camera/detections", rr.Clear(recursive=False))
|
||||
point_start, point_end = (
|
||||
int(support_offsets[index]),
|
||||
int(support_offsets[index + 1]),
|
||||
)
|
||||
if point_end > point_start:
|
||||
recording.log(
|
||||
"/world/perception/support",
|
||||
rr.Points3D(
|
||||
support[point_start:point_end],
|
||||
colors=support_colors[point_start:point_end],
|
||||
radii=rr.Radius.ui_points(3.0),
|
||||
),
|
||||
)
|
||||
else:
|
||||
recording.log("/world/perception/support", rr.Clear(recursive=False))
|
||||
box_start, box_end = int(box_offsets[index]), int(box_offsets[index + 1])
|
||||
presented_cuboids = cuboid_presentation.update(
|
||||
int(timestamp),
|
||||
objects,
|
||||
centers[box_start:box_end],
|
||||
half_sizes[box_start:box_end],
|
||||
quaternions[box_start:box_end],
|
||||
box_colors[box_start:box_end],
|
||||
)
|
||||
if presented_cuboids is not None:
|
||||
(
|
||||
presented_centers,
|
||||
presented_half_sizes,
|
||||
presented_quaternions,
|
||||
presented_colors,
|
||||
presented_labels,
|
||||
) = presented_cuboids
|
||||
recording.log(
|
||||
"/world/perception/boxes3d",
|
||||
rr.Boxes3D(
|
||||
centers=presented_centers,
|
||||
half_sizes=presented_half_sizes,
|
||||
quaternions=presented_quaternions,
|
||||
colors=presented_colors,
|
||||
labels=presented_labels,
|
||||
fill_mode=FillMode.Solid,
|
||||
show_labels=True,
|
||||
),
|
||||
)
|
||||
else:
|
||||
recording.log("/world/perception/boxes3d", rr.Clear(recursive=False))
|
||||
payload = stream.read(flush=True, flush_timeout_sec=300.0)
|
||||
except RecordedPerceptionOverlayError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise RecordedPerceptionOverlayError(
|
||||
"failed to serialize integrated perception"
|
||||
) from exc
|
||||
finally:
|
||||
with suppress(Exception):
|
||||
recording.disconnect()
|
||||
if source_path is not None:
|
||||
source_path.unlink(missing_ok=True)
|
||||
if proxy_path is not None:
|
||||
proxy_path.unlink(missing_ok=True)
|
||||
if payload is None or not payload.startswith(b"RRF2"):
|
||||
raise RecordedPerceptionOverlayError("integrated perception Rerun stream is invalid")
|
||||
return payload
|
||||
|
||||
|
||||
def _camera_source_file(job: CameraComputeJob, segment_count: int, root: Path) -> Path:
|
||||
epoch = job.job_root / "input" / "camera" / job.source_id / f"epoch-{job.codec_epoch}"
|
||||
paths = [epoch / "init.mp4"] + [
|
||||
epoch / "segments" / f"{sequence}.m4s" for sequence in range(1, segment_count + 1)
|
||||
]
|
||||
total = sum(path.stat().st_size for path in paths)
|
||||
if total <= 0 or total > MAX_SOURCE_BYTES:
|
||||
raise RecordedPerceptionOverlayError(
|
||||
"integrated perception camera source is outside bounds"
|
||||
)
|
||||
descriptor, name = tempfile.mkstemp(prefix=".e10-camera-", suffix=".mp4", dir=root)
|
||||
path = Path(name)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as output:
|
||||
for source in paths:
|
||||
with source.open("rb") as stream:
|
||||
shutil.copyfileobj(stream, output, length=1024 * 1024)
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
return path
|
||||
except BaseException:
|
||||
path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def _camera_proxy_file(
|
||||
source: Path,
|
||||
start_frame_index: int,
|
||||
end_frame_index: int,
|
||||
ffmpeg_path: Path,
|
||||
root: Path,
|
||||
) -> Path:
|
||||
descriptor, name = tempfile.mkstemp(prefix=".e10-camera-proxy-", suffix=".mp4", dir=root)
|
||||
os.close(descriptor)
|
||||
path = Path(name)
|
||||
path.unlink(missing_ok=True)
|
||||
frame_filter = (
|
||||
f"select=between(n\\,{start_frame_index}\\,{end_frame_index}),setpts=N/(10*TB)"
|
||||
)
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
[
|
||||
str(ffmpeg_path),
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-i",
|
||||
str(source),
|
||||
"-vf",
|
||||
frame_filter,
|
||||
"-an",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"28",
|
||||
"-g",
|
||||
"20",
|
||||
"-keyint_min",
|
||||
"20",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
str(path),
|
||||
],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
timeout=300,
|
||||
)
|
||||
if completed.returncode != 0 or not path.is_file() or path.stat().st_size <= 0:
|
||||
raise RecordedPerceptionOverlayError(
|
||||
f"integrated perception video proxy failed: {completed.stderr[-1000:]!r}"
|
||||
)
|
||||
os.chmod(path, 0o600)
|
||||
return path
|
||||
except BaseException:
|
||||
path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def _validate_pack(
|
||||
root: Path,
|
||||
job: CameraComputeJob,
|
||||
result_identity: dict[str, Any],
|
||||
count: int,
|
||||
start: int,
|
||||
end: int,
|
||||
) -> None:
|
||||
if not root.is_dir() or _SAFE_PACK_ID.fullmatch(root.name) is None:
|
||||
raise SessionIntegrityError("integrated perception LiDAR pack root is invalid")
|
||||
manifest = _read_object(root / "manifest.json", root)
|
||||
identity = manifest.get("identity")
|
||||
identity_sha256 = manifest.get("identity_sha256")
|
||||
arrays_path = root / "lidar-pack.npz"
|
||||
artifact = manifest.get("artifact")
|
||||
if (
|
||||
manifest.get("schema_version") != PACK_SCHEMA
|
||||
or manifest.get("pack_id") != root.name
|
||||
or not isinstance(identity, dict)
|
||||
or identity.get("schema_version") != PACK_SCHEMA
|
||||
or identity.get("job_id") != job.job_id
|
||||
or identity.get("input_sha256") != job.input_sha256
|
||||
or identity.get("calibration_sha256")
|
||||
!= result_identity["configuration"]["profile"]["source"]["calibration_sha256"]
|
||||
or identity.get("frame_count") != count
|
||||
or identity.get("source_start_frame_index") != start
|
||||
or identity.get("source_end_frame_index") != end
|
||||
or not isinstance(identity_sha256, str)
|
||||
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
||||
or root.name != f"e10-lidar-pack-{identity_sha256}"
|
||||
or not isinstance(artifact, dict)
|
||||
or artifact.get("path") != arrays_path.name
|
||||
or artifact.get("byte_length") != arrays_path.stat().st_size
|
||||
or artifact.get("sha256") != _sha256(arrays_path)
|
||||
):
|
||||
raise SessionIntegrityError("integrated perception LiDAR pack is inconsistent")
|
||||
|
||||
|
||||
def _validate_artifacts(root: Path, raw: object) -> dict[str, Path]:
|
||||
expected = {
|
||||
"e10-semantic-frames": ("semantic-frames.jsonl", SEMANTIC_SCHEMA),
|
||||
"e10-fusion-frames": ("fusion-frames.jsonl", FUSION_SCHEMA),
|
||||
"e10-world-state": ("world-state.jsonl", WORLD_SCHEMA),
|
||||
"e10-transient-perception": ("transient-perception.npz", None),
|
||||
"worker-gpu-telemetry": ("gpu-telemetry.jsonl", None),
|
||||
"e10-run-report": ("run-report.json", REPORT_SCHEMA),
|
||||
}
|
||||
if not isinstance(raw, list) or len(raw) != len(expected):
|
||||
raise SessionIntegrityError("integrated perception artifact set is incomplete")
|
||||
result = {}
|
||||
for value in raw:
|
||||
kind = value.get("kind") if isinstance(value, dict) else None
|
||||
if not isinstance(kind, str) or kind not in expected or kind in result:
|
||||
raise SessionIntegrityError("integrated perception artifact descriptor is invalid")
|
||||
name, schema = expected[kind]
|
||||
path = root / name
|
||||
metadata = _confined_file(path, root)
|
||||
if (
|
||||
value.get("path") != name
|
||||
or value.get("schema_version") != schema
|
||||
or value.get("byte_length") != metadata.st_size
|
||||
or value.get("sha256") != _sha256(path)
|
||||
):
|
||||
raise SessionIntegrityError("integrated perception artifact identity changed")
|
||||
result[kind] = path
|
||||
return result
|
||||
|
||||
|
||||
def _validate_arrays(
|
||||
path: Path,
|
||||
count: int,
|
||||
semantic_rows: list[dict[str, Any]],
|
||||
fusion_rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
with np.load(path, allow_pickle=False) as arrays:
|
||||
required = {
|
||||
"frame_times_ns",
|
||||
"semantic_frame_indices",
|
||||
"semantic_masks",
|
||||
"support_offsets",
|
||||
"support_points",
|
||||
"support_colors",
|
||||
"box_offsets",
|
||||
"box_centers",
|
||||
"box_half_sizes",
|
||||
"box_quaternions",
|
||||
"box_colors",
|
||||
}
|
||||
if set(arrays.files) != required:
|
||||
raise SessionIntegrityError("integrated perception array set changed")
|
||||
times = arrays["frame_times_ns"]
|
||||
semantic_indices = arrays["semantic_frame_indices"]
|
||||
masks = arrays["semantic_masks"]
|
||||
support_offsets = arrays["support_offsets"]
|
||||
support = arrays["support_points"]
|
||||
support_colors = arrays["support_colors"]
|
||||
box_offsets = arrays["box_offsets"]
|
||||
centers = arrays["box_centers"]
|
||||
half_sizes = arrays["box_half_sizes"]
|
||||
quaternions = arrays["box_quaternions"]
|
||||
colors = arrays["box_colors"]
|
||||
expected_boxes = sum(
|
||||
sum(
|
||||
str(item.get("cuboid_status", "")).startswith("accepted-")
|
||||
for item in row["objects"]
|
||||
)
|
||||
for row in fusion_rows
|
||||
)
|
||||
if (
|
||||
times.dtype != np.int64
|
||||
or times.shape != (count,)
|
||||
or np.any(np.diff(times) <= 0)
|
||||
or semantic_indices.dtype != np.int64
|
||||
or semantic_indices.shape != (len(semantic_rows),)
|
||||
or not np.array_equal(semantic_indices, [row["frame_index"] for row in semantic_rows])
|
||||
or masks.dtype != np.uint8
|
||||
or masks.shape != (len(semantic_rows), 600, 800)
|
||||
or support_offsets.shape != (count + 1,)
|
||||
or support.shape[1:] != (3,)
|
||||
or support_colors.shape != support.shape
|
||||
or box_offsets.shape != (count + 1,)
|
||||
or centers.shape != (expected_boxes, 3)
|
||||
or half_sizes.shape != centers.shape
|
||||
or quaternions.shape != (expected_boxes, 4)
|
||||
or colors.shape != (expected_boxes, 4)
|
||||
or int(support_offsets[-1]) != support.shape[0]
|
||||
or int(box_offsets[-1]) != expected_boxes
|
||||
or np.any(np.diff(support_offsets) < 0)
|
||||
or np.any(np.diff(box_offsets) < 0)
|
||||
or not np.isfinite(support).all()
|
||||
or not np.isfinite(centers).all()
|
||||
or not np.isfinite(half_sizes).all()
|
||||
or np.any(half_sizes <= 0)
|
||||
):
|
||||
raise SessionIntegrityError("integrated perception arrays are inconsistent")
|
||||
|
||||
|
||||
def _read_rows(path: Path, root: Path, schema: str) -> list[dict[str, Any]]:
|
||||
_confined_file(path, root)
|
||||
rows = []
|
||||
with path.open(encoding="utf-8") as stream:
|
||||
for line in stream:
|
||||
if len(line.encode()) > MAX_LINE_BYTES:
|
||||
raise SessionIntegrityError("integrated perception row is oversized")
|
||||
value = json.loads(line)
|
||||
if not isinstance(value, dict) or value.get("schema_version") != schema:
|
||||
raise SessionIntegrityError("integrated perception row schema changed")
|
||||
rows.append(value)
|
||||
return rows
|
||||
|
||||
|
||||
def _read_cache(
|
||||
output: Path,
|
||||
sidecar: Path,
|
||||
result: IntegratedPerceptionResult,
|
||||
recording_id: str,
|
||||
) -> bytes | None:
|
||||
try:
|
||||
value = _read_object(sidecar, sidecar.parent)
|
||||
payload = output.read_bytes()
|
||||
except (OSError, SessionIntegrityError):
|
||||
return None
|
||||
if (
|
||||
value.get("schema_version") != "missioncore.e10-overlay-cache/v1"
|
||||
or value.get("renderer_version") != OVERLAY_RENDERER_VERSION
|
||||
or value.get("result_id") != result.result_id
|
||||
or value.get("recording_id") != recording_id
|
||||
or value.get("byte_length") != len(payload)
|
||||
or value.get("sha256") != hashlib.sha256(payload).hexdigest()
|
||||
or not payload.startswith(b"RRF2")
|
||||
):
|
||||
return None
|
||||
return payload
|
||||
|
||||
|
||||
def _read_object(path: Path, root: Path) -> dict[str, Any]:
|
||||
metadata = _confined_file(path, root)
|
||||
if not 0 < metadata.st_size <= MAX_JSON_BYTES:
|
||||
raise SessionIntegrityError("integrated perception JSON is outside bounds")
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise SessionIntegrityError("integrated perception JSON is unavailable") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise SessionIntegrityError("integrated perception JSON is not an object")
|
||||
return value
|
||||
|
||||
|
||||
def _confined_file(path: Path, root: Path) -> os.stat_result:
|
||||
try:
|
||||
resolved_root = root.resolve(strict=True)
|
||||
resolved = path.resolve(strict=True)
|
||||
metadata = path.lstat()
|
||||
except OSError as exc:
|
||||
raise SessionIntegrityError("integrated perception artifact is unavailable") from exc
|
||||
if (
|
||||
stat.S_ISLNK(metadata.st_mode)
|
||||
or not stat.S_ISREG(metadata.st_mode)
|
||||
or not resolved.is_relative_to(resolved_root)
|
||||
):
|
||||
raise SessionIntegrityError("integrated perception artifact is not confined")
|
||||
return metadata
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode()
|
||||
|
||||
|
||||
def _finite(value: object) -> TypeGuard[int | float]:
|
||||
return (
|
||||
isinstance(value, int | float)
|
||||
and not isinstance(value, bool)
|
||||
and bool(np.isfinite(value))
|
||||
)
|
||||
|
||||
|
||||
def _private_directory(path: Path) -> Path:
|
||||
path.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if path.is_symlink() or not path.is_dir():
|
||||
raise RecordedPerceptionOverlayError("integrated perception cache root is invalid")
|
||||
with suppress(OSError):
|
||||
os.chmod(path, 0o700)
|
||||
return path.resolve(strict=True)
|
||||
|
||||
|
||||
def _private_child(root: Path, name: str) -> Path:
|
||||
child = root / name
|
||||
child.mkdir(mode=0o700, exist_ok=True)
|
||||
if child.is_symlink() or not child.is_dir() or child.resolve(strict=True).parent != root:
|
||||
raise RecordedPerceptionOverlayError("integrated perception cache child is invalid")
|
||||
with suppress(OSError):
|
||||
os.chmod(child, 0o700)
|
||||
return child.resolve(strict=True)
|
||||
Reference in New Issue
Block a user