feat(perception): integrate calibrated operator pipeline
Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
This commit is contained in:
@@ -2,16 +2,18 @@ from __future__ import annotations
|
||||
|
||||
import math
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping
|
||||
from contextlib import suppress
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Literal
|
||||
from typing import Any, Literal
|
||||
from uuid import uuid4
|
||||
|
||||
import numpy as np
|
||||
import rerun as rr
|
||||
from rerun import blueprint as rrb
|
||||
from rerun.components import FillMode
|
||||
|
||||
from k1link.compute.live_perception import LivePerceptionResultFrame
|
||||
from k1link.data_plane import (
|
||||
DecodedDataPlaneView,
|
||||
DecodedPointCloudView,
|
||||
@@ -49,6 +51,9 @@ class RerunSceneSettings:
|
||||
show_points: bool = True
|
||||
show_trajectory: bool = True
|
||||
show_grid: bool = True
|
||||
show_detections_2d: bool = False
|
||||
show_segmentation: bool = False
|
||||
show_cuboids_3d: bool = False
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return asdict(self)
|
||||
@@ -188,6 +193,82 @@ class RerunBridge:
|
||||
if context.live and context.received_monotonic_ns is not None:
|
||||
self.metrics.record_latency((published_ns - context.received_monotonic_ns) / 1_000_000)
|
||||
|
||||
def process_perception(self, frame: LivePerceptionResultFrame) -> None:
|
||||
"""Publish one validated worker result on the live scene timeline."""
|
||||
|
||||
self._apply_latest_settings()
|
||||
self._recording.set_time(
|
||||
"stream_time",
|
||||
timestamp=frame.captured_at_epoch_ns / 1_000_000_000,
|
||||
)
|
||||
self._recording.set_time(
|
||||
"capture_time",
|
||||
timestamp=frame.captured_at_epoch_ns / 1_000_000_000,
|
||||
)
|
||||
self._recording.set_time("message_sequence", sequence=frame.source_frame_index)
|
||||
self._recording.log(
|
||||
"/perception/camera/image",
|
||||
rr.EncodedImage(contents=frame.image_jpeg, media_type="image/jpeg"),
|
||||
)
|
||||
if frame.segmentation_mask is None:
|
||||
self._recording.log(
|
||||
"/perception/camera/segmentation",
|
||||
rr.Clear(recursive=False),
|
||||
)
|
||||
else:
|
||||
self._recording.log(
|
||||
"/perception/camera/segmentation",
|
||||
rr.SegmentationImage(frame.segmentation_mask),
|
||||
)
|
||||
if frame.objects:
|
||||
self._recording.log(
|
||||
"/perception/camera/detections",
|
||||
rr.Boxes2D(
|
||||
array=[item["bbox_xyxy"] for item in frame.objects],
|
||||
array_format=rr.Box2DFormat.XYXY,
|
||||
labels=[_perception_label(item) for item in frame.objects],
|
||||
colors=[
|
||||
_perception_color(str(item["label"]), alpha=255)
|
||||
for item in frame.objects
|
||||
],
|
||||
show_labels=True,
|
||||
),
|
||||
)
|
||||
else:
|
||||
self._recording.log(
|
||||
"/perception/camera/detections",
|
||||
rr.Clear(recursive=False),
|
||||
)
|
||||
cuboids = [
|
||||
item for item in frame.objects if item.get("cuboid_center_map") is not None
|
||||
]
|
||||
if cuboids:
|
||||
self._recording.log(
|
||||
"/world/perception/boxes3d",
|
||||
rr.Boxes3D(
|
||||
centers=[item["cuboid_center_map"] for item in cuboids],
|
||||
half_sizes=[item["cuboid_half_size"] for item in cuboids],
|
||||
quaternions=[
|
||||
rr.Quaternion(xyzw=item["cuboid_quaternion_xyzw"])
|
||||
for item in cuboids
|
||||
],
|
||||
colors=[_perception_color(str(item["label"]), alpha=96) for item in cuboids],
|
||||
labels=[_perception_label(item) for item in cuboids],
|
||||
fill_mode=FillMode.Solid,
|
||||
show_labels=True,
|
||||
),
|
||||
)
|
||||
else:
|
||||
self._recording.log(
|
||||
"/world/perception/boxes3d",
|
||||
rr.Clear(recursive=False),
|
||||
)
|
||||
self.metrics.published_perception(
|
||||
captured_at_epoch_ns=frame.captured_at_epoch_ns,
|
||||
published_at_epoch_ns=time.time_ns(),
|
||||
published_monotonic_ns=time.monotonic_ns(),
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
@@ -324,18 +405,51 @@ def _blueprint(settings: RerunSceneSettings) -> rrb.Blueprint:
|
||||
start=rr.TimeRangeBoundary.cursor_relative(seconds=-accumulation),
|
||||
end=rr.TimeRangeBoundary.cursor_relative(),
|
||||
)
|
||||
return rrb.Blueprint(
|
||||
rrb.Spatial3DView(
|
||||
origin="/world",
|
||||
name="Пространственная сцена",
|
||||
background=[7, 8, 10, 255],
|
||||
line_grid=rrb.LineGrid3D(
|
||||
visible=settings.show_grid,
|
||||
color=[86, 91, 99, 110],
|
||||
stroke_width=0.75,
|
||||
),
|
||||
time_ranges=[time_range],
|
||||
perception_active = (
|
||||
settings.show_detections_2d
|
||||
or settings.show_segmentation
|
||||
or settings.show_cuboids_3d
|
||||
)
|
||||
spatial = rrb.Spatial3DView(
|
||||
origin="/world",
|
||||
name="Мир · LiDAR и объекты" if perception_active else "Пространственная сцена",
|
||||
background=[7, 8, 10, 255],
|
||||
line_grid=rrb.LineGrid3D(
|
||||
visible=settings.show_grid,
|
||||
color=[86, 91, 99, 110],
|
||||
stroke_width=0.75,
|
||||
),
|
||||
time_ranges=[] if perception_active else [time_range],
|
||||
)
|
||||
spatial.visualizer_overrides["/world/points"] = rrb.EntityBehavior(
|
||||
visible=settings.show_points
|
||||
)
|
||||
spatial.visualizer_overrides["/world/trajectory"] = rrb.EntityBehavior(
|
||||
visible=settings.show_trajectory
|
||||
)
|
||||
spatial.visualizer_overrides["/world/perception/boxes3d"] = rrb.EntityBehavior(
|
||||
visible=settings.show_cuboids_3d
|
||||
)
|
||||
camera = rrb.Spatial2DView(
|
||||
origin="/perception/camera",
|
||||
name="Оригинальное видео · слои AI",
|
||||
)
|
||||
camera.visualizer_overrides["/perception/camera/image"] = rrb.EntityBehavior(
|
||||
visible=True
|
||||
)
|
||||
camera.visualizer_overrides["/perception/camera/detections"] = rrb.EntityBehavior(
|
||||
visible=settings.show_detections_2d
|
||||
)
|
||||
camera.visualizer_overrides["/perception/camera/segmentation"] = rrb.EntityBehavior(
|
||||
visible=settings.show_segmentation
|
||||
)
|
||||
root = (
|
||||
rrb.Horizontal(camera, spatial, column_shares=[0.46, 0.54])
|
||||
if perception_active
|
||||
else spatial
|
||||
)
|
||||
return rrb.Blueprint(
|
||||
root,
|
||||
_live_time_panel(),
|
||||
auto_layout=False,
|
||||
auto_views=False,
|
||||
@@ -343,6 +457,25 @@ def _blueprint(settings: RerunSceneSettings) -> rrb.Blueprint:
|
||||
)
|
||||
|
||||
|
||||
def _perception_label(item: Mapping[str, Any]) -> str:
|
||||
base = f"#{int(item['track_id'])} {item['label']} · {float(item['score']):.0%}"
|
||||
distance = item.get("distance_m")
|
||||
return base if distance is None else f"{base} · {float(distance):.1f} m"
|
||||
|
||||
|
||||
def _perception_color(label: str, *, alpha: int) -> list[int]:
|
||||
colors = {
|
||||
"person": (255, 99, 132),
|
||||
"car": (64, 180, 255),
|
||||
"truck": (255, 180, 64),
|
||||
"bus": (255, 210, 64),
|
||||
"bicycle": (110, 240, 155),
|
||||
"motorcycle": (170, 115, 255),
|
||||
}
|
||||
red, green, blue = colors.get(label, (247, 248, 244))
|
||||
return [red, green, blue, alpha]
|
||||
|
||||
|
||||
def _live_time_panel() -> rrb.TimePanel:
|
||||
"""Keep the hidden vendor timeline on its native live edge."""
|
||||
return rrb.TimePanel(
|
||||
|
||||
Reference in New Issue
Block a user