feat(perception): integrate calibrated operator pipeline

Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
This commit is contained in:
DCCONSTRUCTIONS
2026-07-23 00:23:28 +03:00
parent ada2a55ee6
commit b53d6d5a45
221 changed files with 55923 additions and 1357 deletions
+146 -13
View File
@@ -2,16 +2,18 @@ from __future__ import annotations
import math
import time
from collections.abc import Callable
from collections.abc import Callable, Mapping
from contextlib import suppress
from dataclasses import asdict, dataclass
from typing import Literal
from typing import Any, Literal
from uuid import uuid4
import numpy as np
import rerun as rr
from rerun import blueprint as rrb
from rerun.components import FillMode
from k1link.compute.live_perception import LivePerceptionResultFrame
from k1link.data_plane import (
DecodedDataPlaneView,
DecodedPointCloudView,
@@ -49,6 +51,9 @@ class RerunSceneSettings:
show_points: bool = True
show_trajectory: bool = True
show_grid: bool = True
show_detections_2d: bool = False
show_segmentation: bool = False
show_cuboids_3d: bool = False
def as_dict(self) -> dict[str, object]:
return asdict(self)
@@ -188,6 +193,82 @@ class RerunBridge:
if context.live and context.received_monotonic_ns is not None:
self.metrics.record_latency((published_ns - context.received_monotonic_ns) / 1_000_000)
def process_perception(self, frame: LivePerceptionResultFrame) -> None:
"""Publish one validated worker result on the live scene timeline."""
self._apply_latest_settings()
self._recording.set_time(
"stream_time",
timestamp=frame.captured_at_epoch_ns / 1_000_000_000,
)
self._recording.set_time(
"capture_time",
timestamp=frame.captured_at_epoch_ns / 1_000_000_000,
)
self._recording.set_time("message_sequence", sequence=frame.source_frame_index)
self._recording.log(
"/perception/camera/image",
rr.EncodedImage(contents=frame.image_jpeg, media_type="image/jpeg"),
)
if frame.segmentation_mask is None:
self._recording.log(
"/perception/camera/segmentation",
rr.Clear(recursive=False),
)
else:
self._recording.log(
"/perception/camera/segmentation",
rr.SegmentationImage(frame.segmentation_mask),
)
if frame.objects:
self._recording.log(
"/perception/camera/detections",
rr.Boxes2D(
array=[item["bbox_xyxy"] for item in frame.objects],
array_format=rr.Box2DFormat.XYXY,
labels=[_perception_label(item) for item in frame.objects],
colors=[
_perception_color(str(item["label"]), alpha=255)
for item in frame.objects
],
show_labels=True,
),
)
else:
self._recording.log(
"/perception/camera/detections",
rr.Clear(recursive=False),
)
cuboids = [
item for item in frame.objects if item.get("cuboid_center_map") is not None
]
if cuboids:
self._recording.log(
"/world/perception/boxes3d",
rr.Boxes3D(
centers=[item["cuboid_center_map"] for item in cuboids],
half_sizes=[item["cuboid_half_size"] for item in cuboids],
quaternions=[
rr.Quaternion(xyzw=item["cuboid_quaternion_xyzw"])
for item in cuboids
],
colors=[_perception_color(str(item["label"]), alpha=96) for item in cuboids],
labels=[_perception_label(item) for item in cuboids],
fill_mode=FillMode.Solid,
show_labels=True,
),
)
else:
self._recording.log(
"/world/perception/boxes3d",
rr.Clear(recursive=False),
)
self.metrics.published_perception(
captured_at_epoch_ns=frame.captured_at_epoch_ns,
published_at_epoch_ns=time.time_ns(),
published_monotonic_ns=time.monotonic_ns(),
)
def close(self) -> None:
if self._closed:
return
@@ -324,18 +405,51 @@ def _blueprint(settings: RerunSceneSettings) -> rrb.Blueprint:
start=rr.TimeRangeBoundary.cursor_relative(seconds=-accumulation),
end=rr.TimeRangeBoundary.cursor_relative(),
)
return rrb.Blueprint(
rrb.Spatial3DView(
origin="/world",
name="Пространственная сцена",
background=[7, 8, 10, 255],
line_grid=rrb.LineGrid3D(
visible=settings.show_grid,
color=[86, 91, 99, 110],
stroke_width=0.75,
),
time_ranges=[time_range],
perception_active = (
settings.show_detections_2d
or settings.show_segmentation
or settings.show_cuboids_3d
)
spatial = rrb.Spatial3DView(
origin="/world",
name="Мир · LiDAR и объекты" if perception_active else "Пространственная сцена",
background=[7, 8, 10, 255],
line_grid=rrb.LineGrid3D(
visible=settings.show_grid,
color=[86, 91, 99, 110],
stroke_width=0.75,
),
time_ranges=[] if perception_active else [time_range],
)
spatial.visualizer_overrides["/world/points"] = rrb.EntityBehavior(
visible=settings.show_points
)
spatial.visualizer_overrides["/world/trajectory"] = rrb.EntityBehavior(
visible=settings.show_trajectory
)
spatial.visualizer_overrides["/world/perception/boxes3d"] = rrb.EntityBehavior(
visible=settings.show_cuboids_3d
)
camera = rrb.Spatial2DView(
origin="/perception/camera",
name="Оригинальное видео · слои AI",
)
camera.visualizer_overrides["/perception/camera/image"] = rrb.EntityBehavior(
visible=True
)
camera.visualizer_overrides["/perception/camera/detections"] = rrb.EntityBehavior(
visible=settings.show_detections_2d
)
camera.visualizer_overrides["/perception/camera/segmentation"] = rrb.EntityBehavior(
visible=settings.show_segmentation
)
root = (
rrb.Horizontal(camera, spatial, column_shares=[0.46, 0.54])
if perception_active
else spatial
)
return rrb.Blueprint(
root,
_live_time_panel(),
auto_layout=False,
auto_views=False,
@@ -343,6 +457,25 @@ def _blueprint(settings: RerunSceneSettings) -> rrb.Blueprint:
)
def _perception_label(item: Mapping[str, Any]) -> str:
base = f"#{int(item['track_id'])} {item['label']} · {float(item['score']):.0%}"
distance = item.get("distance_m")
return base if distance is None else f"{base} · {float(distance):.1f} m"
def _perception_color(label: str, *, alpha: int) -> list[int]:
colors = {
"person": (255, 99, 132),
"car": (64, 180, 255),
"truck": (255, 180, 64),
"bus": (255, 210, 64),
"bicycle": (110, 240, 155),
"motorcycle": (170, 115, 255),
}
red, green, blue = colors.get(label, (247, 248, 244))
return [red, green, blue, alpha]
def _live_time_panel() -> rrb.TimePanel:
"""Keep the hidden vendor timeline on its native live edge."""
return rrb.TimePanel(