SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): integrate calibrated operator pipeline

Browse Source 
Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
This commit is contained in:
DCCONSTRUCTIONS
2026-07-23 00:23:28 +03:00
parent ada2a55ee6
commit b53d6d5a45
221 changed files with 55923 additions and 1357 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/viewer/metrics.py
+49
View File
@@ -30,6 +30,12 @@ class MetricsSnapshot(TypedDict):
device_pgo_progress: int | None
modeling_reports: int
modeling_decode_errors: int
perception_frames: int
perception_dropped: int
perception_fps: float
perception_end_to_end_ms: float | None
perception_end_to_end_p95_ms: float | None
perception_stale_ms: float | None
class BridgeMetrics:
@@ -56,6 +62,11 @@ class BridgeMetrics:
self._device_pgo_progress: int | None = None
self._modeling_reports = 0
self._modeling_decode_errors = 0
self._perception_frames = 0
self._perception_dropped = 0
self._perception_times: deque[int] = deque()
self._perception_latencies_ms: deque[float] = deque(maxlen=512)
self._perception_last_publish_monotonic_ns: int | None = None
def received(self, payload_bytes: int) -> None:
with self._lock:
@@ -93,6 +104,26 @@ class BridgeMetrics:
with self._lock:
self._preview_dropped += 1
def perception_dropped(self) -> None:
with self._lock:
self._perception_dropped += 1
def published_perception(
self,
*,
captured_at_epoch_ns: int,
published_at_epoch_ns: int,
published_monotonic_ns: int,
) -> None:
latency_ms = (published_at_epoch_ns - captured_at_epoch_ns) / 1_000_000
with self._lock:
self._perception_frames += 1
self._perception_times.append(published_monotonic_ns)
_trim_rate_window(self._perception_times, published_monotonic_ns)
self._perception_last_publish_monotonic_ns = published_monotonic_ns
if math.isfinite(latency_ms) and latency_ms >= 0:
self._perception_latencies_ms.append(latency_ms)
def acquisition_telemetry(
self,
*,
@@ -125,7 +156,9 @@ class BridgeMetrics:
with self._lock:
_trim_rate_window(self._pcl_times, now_ns)
_trim_rate_window(self._pose_times, now_ns)
_trim_rate_window(self._perception_times, now_ns)
latencies = list(self._latencies_ms)
perception_latencies = list(self._perception_latencies_ms)
last_latency = latencies[-1] if latencies else None
p50 = statistics.median(latencies) if latencies else None
p95 = _percentile(latencies, 0.95) if latencies else None
@@ -151,6 +184,22 @@ class BridgeMetrics:
"device_pgo_progress": self._device_pgo_progress,
"modeling_reports": self._modeling_reports,
"modeling_decode_errors": self._modeling_decode_errors,
"perception_frames": self._perception_frames,
"perception_dropped": self._perception_dropped,
"perception_fps": _window_rate(self._perception_times),
"perception_end_to_end_ms": _rounded(
perception_latencies[-1] if perception_latencies else None
),
"perception_end_to_end_p95_ms": _rounded(
_percentile(perception_latencies, 0.95)
if perception_latencies
else None
),
"perception_stale_ms": _rounded(
None
if self._perception_last_publish_monotonic_ns is None
else (now_ns - self._perception_last_publish_monotonic_ns) / 1_000_000
),
}
src/k1link/viewer/recorded.py
+179 -14
View File
@@ -3,7 +3,7 @@
from __future__ import annotations
from contextlib import suppress
from typing import Literal
from typing import Any, Literal
from uuid import UUID
import rerun as rr
@@ -15,11 +15,29 @@ APPLICATION_ID = "nodedc_mission_core_recorded"
SESSION_TIMELINE = "session_time"
RECORDED_SPATIAL_VIEW_ID = UUID("5f5f11d5-3b0a-4a81-887b-2be767cba1c0")
RECORDED_SPATIAL_RESET_VIEW_ID = UUID("1480abcc-a0ae-4287-ab69-4b606d15d947")
RECORDED_ROOT_CONTAINER_ID = UUID("b02f2aca-8471-4dcb-b786-53df5a320fc8")
RECORDED_SPATIAL_RESET_ROOT_CONTAINER_ID = UUID("27d95ad7-1e72-46ca-b854-d02cd99b7610")
RECORDED_PERCEPTION_ROOT_CONTAINER_ID = UUID("70ea9fd5-bbbb-4b23-8ae8-8098af92b997")
RECORDED_PERCEPTION_RESET_ROOT_CONTAINER_ID = UUID("d2196c6e-99da-4402-857c-4daea0dd3ae0")
RECORDED_PERCEPTION_3D_ROOT_CONTAINER_ID = UUID("81051015-9808-413b-90a4-1cfaacacbc4f")
RECORDED_PERCEPTION_3D_RESET_ROOT_CONTAINER_ID = UUID(
"3de10822-0274-4a58-8d45-f35d86625303"
)
RECORDED_METRICS_ROOT_CONTAINER_ID = UUID("374710a2-1d77-4349-bea4-8719e7aa1f09")
RECORDED_METRICS_RESET_ROOT_CONTAINER_ID = UUID("1e8ac565-bbd6-4554-9213-dad564e534e3")
RECORDED_POINTS_VISUALIZER_ID = UUID("ca037ec0-8761-4417-86ee-846fa2875303")
RECORDED_PERCEPTION_POINTS_VISUALIZER_ID = UUID(
"ab8db306-6981-49c5-b417-94fe2f01dbf0"
)
RECORDED_CAMERA_VIEW_ID = UUID("5c1db75b-07cd-479a-903d-f4f2ed554513")
RECORDED_CAMERA_RESET_VIEW_ID = UUID("d0618e0c-c889-4222-a643-60a4a882935c")
RECORDED_PERCEPTION_3D_VIEW_ID = UUID("0496bd2e-2b4d-4a4f-87b8-3ce4f9f7e114")
RECORDED_PERCEPTION_3D_RESET_VIEW_ID = UUID("31f63ab8-ffdd-4751-ae40-e8e7b4462c8f")
RECORDED_METRICS_VIEW_ID = UUID("f973fc11-0867-4732-ad3c-97008621fab7")
RecordedView = Literal["spatial", "perception", "metrics"]
RECORDED_UNIFIED_ROOT_CONTAINER_ID = UUID("e9934ef8-453f-432e-9136-2b0908190253")
RECORDED_UNIFIED_RESET_ROOT_CONTAINER_ID = UUID("2781407d-9f4e-4405-86e8-bbe6065e83df")
RecordedView = Literal["spatial", "perception", "perception3d", "metrics"]
class RecordedBlueprintError(RuntimeError):
@@ -31,6 +49,11 @@ def recorded_blueprint(
*,
include_initial_playback_state: bool = True,
active_view: RecordedView = "spatial",
view_reset_generation: Literal[0, 1] = 0,
unified_perception: bool = False,
show_detections_2d: bool = False,
show_segmentation: bool = False,
show_cuboids_3d: bool = False,
) -> rrb.Blueprint:
accumulation = max(0.0, settings.accumulation_seconds)
time_ranges: list[rr.VisibleTimeRange] | None = None
@@ -42,6 +65,7 @@ def recorded_blueprint(
end=rr.TimeRangeBoundary.cursor_relative(),
)
]
latest_time_ranges = rrb.VisibleTimeRanges([])
point_visualizer = rr.Points3D.from_fields(
radii=rr.Radius.ui_points(settings.point_size),
colors=(
@@ -51,9 +75,18 @@ def recorded_blueprint(
),
).visualizer()
point_visualizer.id = RECORDED_POINTS_VISUALIZER_ID
perception_point_visualizer = rr.Points3D.from_fields(
radii=rr.Radius.ui_points(settings.point_size),
colors=(
[_parse_hex_color(settings.custom_color)]
if settings.palette == "custom"
else None
),
).visualizer()
perception_point_visualizer.id = RECORDED_PERCEPTION_POINTS_VISUALIZER_ID
spatial_view = rrb.Spatial3DView(
origin="/world",
name="Пространственная сцена",
name="Мир · LiDAR и объекты" if unified_perception else "Пространственная сцена",
background=[7, 8, 10, 255],
line_grid=rrb.LineGrid3D(
visible=settings.show_grid,
@@ -68,29 +101,151 @@ def recorded_blueprint(
"/world/trajectory": rrb.EntityBehavior(
visible=settings.show_trajectory,
),
# Perception is hidden in the mapping-only presentation. Unified
# perception below adds explicit per-entity latest-at overrides,
# allowing the native cloud to retain this view's accumulation.
"/world/perception": rrb.EntityBehavior(visible=False),
},
time_ranges=time_ranges,
)
spatial_view.id = RECORDED_SPATIAL_VIEW_ID
spatial_view.id = (
RECORDED_SPATIAL_RESET_VIEW_ID
if view_reset_generation
else RECORDED_SPATIAL_VIEW_ID
)
camera_view = rrb.Spatial2DView(
origin="/perception/camera",
name="Камера · распознавание",
name="Оригинальное видео · слои AI",
background=[7, 8, 10, 255],
overrides={
"/perception/camera/image": rrb.EntityBehavior(visible=True),
"/perception/camera/detections": rrb.EntityBehavior(
visible=show_detections_2d,
),
"/perception/camera/segmentation": rrb.EntityBehavior(
visible=show_segmentation,
),
},
)
camera_view.id = (
RECORDED_CAMERA_RESET_VIEW_ID
if view_reset_generation
else RECORDED_CAMERA_VIEW_ID
)
perception_3d_view = rrb.Spatial3DView(
# Cuboids are expressed in the same calibrated world frame as the
# native LiDAR recording. Rooting this view at /world/perception
# excluded /world/points and left the operator looking at boxes in an
# empty scene. Keep the main point cloud and the derived objects in
# one latest-at world view; unlike the mapping tab, this deliberately
# has no accumulated time range, so dynamic cuboids never stack.
origin="/world",
name="Сегментация и объекты · 3D",
background=[7, 8, 10, 255],
line_grid=rrb.LineGrid3D(
visible=settings.show_grid,
color=[86, 91, 99, 110],
stroke_width=0.75,
),
overrides={
"/world/points": [
rrb.EntityBehavior(visible=settings.show_points),
perception_point_visualizer,
],
"/world/trajectory": rrb.EntityBehavior(
visible=settings.show_trajectory,
),
"/world/perception": rrb.EntityBehavior(visible=True),
# The overlay already carries the selected fusion support points;
# its grey diagnostic LiDAR copy would otherwise double-render the
# native cloud from /world/points.
"/world/perception/lidar": rrb.EntityBehavior(visible=False),
},
)
perception_3d_view.id = (
RECORDED_PERCEPTION_3D_RESET_VIEW_ID
if view_reset_generation
else RECORDED_PERCEPTION_3D_VIEW_ID
)
camera_view.id = RECORDED_CAMERA_VIEW_ID
metrics_view = rrb.TimeSeriesView(
origin="/metrics/device",
name="Маршрут и время",
)
metrics_view.id = RECORDED_METRICS_VIEW_ID
active_tab = {"spatial": 0, "perception": 1, "metrics": 2}[active_view]
root_container = rrb.Tabs(
spatial_view,
camera_view,
metrics_view,
active_tab=active_tab,
)
root_container.id = RECORDED_ROOT_CONTAINER_ID
root_container: Any
if unified_perception:
# Operator perception is one synchronized composition, not a set of
# mutually exclusive modes. The camera keeps the original frame as
# its base and overlays 2D detections/segmentation. The paired world
# view shows the same cursor in the native LiDAR frame and reveals
# semantic points and cuboids independently.
spatial_view.visualizer_overrides["/world/perception"] = [
rrb.EntityBehavior(visible=True),
latest_time_ranges,
]
spatial_view.visualizer_overrides[
"/world/perception/lidar"
] = [
rrb.EntityBehavior(visible=False),
latest_time_ranges,
]
spatial_view.visualizer_overrides[
"/world/perception/support"
] = [
rrb.EntityBehavior(visible=False),
latest_time_ranges,
]
spatial_view.visualizer_overrides[
"/world/perception/semantic_points"
] = [
rrb.EntityBehavior(visible=show_segmentation),
latest_time_ranges,
]
spatial_view.visualizer_overrides[
"/world/perception/boxes3d"
] = [
rrb.EntityBehavior(visible=show_cuboids_3d),
latest_time_ranges,
]
root_container = rrb.Horizontal(
camera_view,
spatial_view,
column_shares=[0.46, 0.54],
name="Единая сцена восприятия",
)
root_container.id = (
RECORDED_UNIFIED_RESET_ROOT_CONTAINER_ID
if view_reset_generation
else RECORDED_UNIFIED_ROOT_CONTAINER_ID
)
else:
# Keep operator video and 3D cuboids as direct root views. Rerun's nested
# Tabs preserve their own active child and cannot be switched reliably by
# a live blueprint channel after the operator has visited another child.
active_tab = {"spatial": 0, "perception": 1, "perception3d": 2, "metrics": 3}[
active_view
]
root_container = rrb.Tabs(
spatial_view,
camera_view,
perception_3d_view,
metrics_view,
active_tab=active_tab,
)
# Rerun persists the active child on a Tabs container and does not reliably
# replace it from a later blueprint message. Give every operator mode (and
# its explicit reset generation) a stable root identity so the requested
# child is authoritative instead of inheriting a previously visited tab.
root_container.id = {
("spatial", 0): RECORDED_ROOT_CONTAINER_ID,
("spatial", 1): RECORDED_SPATIAL_RESET_ROOT_CONTAINER_ID,
("perception", 0): RECORDED_PERCEPTION_ROOT_CONTAINER_ID,
("perception", 1): RECORDED_PERCEPTION_RESET_ROOT_CONTAINER_ID,
("perception3d", 0): RECORDED_PERCEPTION_3D_ROOT_CONTAINER_ID,
("perception3d", 1): RECORDED_PERCEPTION_3D_RESET_ROOT_CONTAINER_ID,
("metrics", 0): RECORDED_METRICS_ROOT_CONTAINER_ID,
("metrics", 1): RECORDED_METRICS_RESET_ROOT_CONTAINER_ID,
}[(active_view, view_reset_generation)]
if include_initial_playback_state:
return rrb.Blueprint(
@@ -118,6 +273,11 @@ def recorded_blueprint_rrd(
application_id: str = APPLICATION_ID,
recording_id: str,
active_view: RecordedView = "spatial",
view_reset_generation: Literal[0, 1] = 0,
unified_perception: bool = False,
show_detections_2d: bool = False,
show_segmentation: bool = False,
show_cuboids_3d: bool = False,
) -> bytes:
"""Serialize a bounded active blueprint update without recorded data."""
@@ -133,6 +293,11 @@ def recorded_blueprint_rrd(
settings,
include_initial_playback_state=False,
active_view=active_view,
view_reset_generation=view_reset_generation,
unified_perception=unified_perception,
show_detections_2d=show_detections_2d,
show_segmentation=show_segmentation,
show_cuboids_3d=show_cuboids_3d,
),
make_active=True,
make_default=False,
src/k1link/viewer/rerun_bridge.py
+146 -13
View File
@@ -2,16 +2,18 @@ from __future__ import annotations
import math
import time
from collections.abc import Callable
from collections.abc import Callable, Mapping
from contextlib import suppress
from dataclasses import asdict, dataclass
from typing import Literal
from typing import Any, Literal
from uuid import uuid4
import numpy as np
import rerun as rr
from rerun import blueprint as rrb
from rerun.components import FillMode
from k1link.compute.live_perception import LivePerceptionResultFrame
from k1link.data_plane import (
DecodedDataPlaneView,
DecodedPointCloudView,
@@ -49,6 +51,9 @@ class RerunSceneSettings:
show_points: bool = True
show_trajectory: bool = True
show_grid: bool = True
show_detections_2d: bool = False
show_segmentation: bool = False
show_cuboids_3d: bool = False
def as_dict(self) -> dict[str, object]:
return asdict(self)
@@ -188,6 +193,82 @@ class RerunBridge:
if context.live and context.received_monotonic_ns is not None:
self.metrics.record_latency((published_ns - context.received_monotonic_ns) / 1_000_000)
def process_perception(self, frame: LivePerceptionResultFrame) -> None:
"""Publish one validated worker result on the live scene timeline."""
self._apply_latest_settings()
self._recording.set_time(
"stream_time",
timestamp=frame.captured_at_epoch_ns / 1_000_000_000,
)
self._recording.set_time(
"capture_time",
timestamp=frame.captured_at_epoch_ns / 1_000_000_000,
)
self._recording.set_time("message_sequence", sequence=frame.source_frame_index)
self._recording.log(
"/perception/camera/image",
rr.EncodedImage(contents=frame.image_jpeg, media_type="image/jpeg"),
)
if frame.segmentation_mask is None:
self._recording.log(
"/perception/camera/segmentation",
rr.Clear(recursive=False),
)
else:
self._recording.log(
"/perception/camera/segmentation",
rr.SegmentationImage(frame.segmentation_mask),
)
if frame.objects:
self._recording.log(
"/perception/camera/detections",
rr.Boxes2D(
array=[item["bbox_xyxy"] for item in frame.objects],
array_format=rr.Box2DFormat.XYXY,
labels=[_perception_label(item) for item in frame.objects],
colors=[
_perception_color(str(item["label"]), alpha=255)
for item in frame.objects
],
show_labels=True,
),
)
else:
self._recording.log(
"/perception/camera/detections",
rr.Clear(recursive=False),
)
cuboids = [
item for item in frame.objects if item.get("cuboid_center_map") is not None
]
if cuboids:
self._recording.log(
"/world/perception/boxes3d",
rr.Boxes3D(
centers=[item["cuboid_center_map"] for item in cuboids],
half_sizes=[item["cuboid_half_size"] for item in cuboids],
quaternions=[
rr.Quaternion(xyzw=item["cuboid_quaternion_xyzw"])
for item in cuboids
],
colors=[_perception_color(str(item["label"]), alpha=96) for item in cuboids],
labels=[_perception_label(item) for item in cuboids],
fill_mode=FillMode.Solid,
show_labels=True,
),
)
else:
self._recording.log(
"/world/perception/boxes3d",
rr.Clear(recursive=False),
)
self.metrics.published_perception(
captured_at_epoch_ns=frame.captured_at_epoch_ns,
published_at_epoch_ns=time.time_ns(),
published_monotonic_ns=time.monotonic_ns(),
)
def close(self) -> None:
if self._closed:
return
@@ -324,18 +405,51 @@ def _blueprint(settings: RerunSceneSettings) -> rrb.Blueprint:
start=rr.TimeRangeBoundary.cursor_relative(seconds=-accumulation),
end=rr.TimeRangeBoundary.cursor_relative(),
)
return rrb.Blueprint(
rrb.Spatial3DView(
origin="/world",
name="Пространственная сцена",
background=[7, 8, 10, 255],
line_grid=rrb.LineGrid3D(
visible=settings.show_grid,
color=[86, 91, 99, 110],
stroke_width=0.75,
),
time_ranges=[time_range],
perception_active = (
settings.show_detections_2d
or settings.show_segmentation
or settings.show_cuboids_3d
)
spatial = rrb.Spatial3DView(
origin="/world",
name="Мир · LiDAR и объекты" if perception_active else "Пространственная сцена",
background=[7, 8, 10, 255],
line_grid=rrb.LineGrid3D(
visible=settings.show_grid,
color=[86, 91, 99, 110],
stroke_width=0.75,
),
time_ranges=[] if perception_active else [time_range],
)
spatial.visualizer_overrides["/world/points"] = rrb.EntityBehavior(
visible=settings.show_points
)
spatial.visualizer_overrides["/world/trajectory"] = rrb.EntityBehavior(
visible=settings.show_trajectory
)
spatial.visualizer_overrides["/world/perception/boxes3d"] = rrb.EntityBehavior(
visible=settings.show_cuboids_3d
)
camera = rrb.Spatial2DView(
origin="/perception/camera",
name="Оригинальное видео · слои AI",
)
camera.visualizer_overrides["/perception/camera/image"] = rrb.EntityBehavior(
visible=True
)
camera.visualizer_overrides["/perception/camera/detections"] = rrb.EntityBehavior(
visible=settings.show_detections_2d
)
camera.visualizer_overrides["/perception/camera/segmentation"] = rrb.EntityBehavior(
visible=settings.show_segmentation
)
root = (
rrb.Horizontal(camera, spatial, column_shares=[0.46, 0.54])
if perception_active
else spatial
)
return rrb.Blueprint(
root,
_live_time_panel(),
auto_layout=False,
auto_views=False,
@@ -343,6 +457,25 @@ def _blueprint(settings: RerunSceneSettings) -> rrb.Blueprint:
)
def _perception_label(item: Mapping[str, Any]) -> str:
base = f"#{int(item['track_id'])} {item['label']} · {float(item['score']):.0%}"
distance = item.get("distance_m")
return base if distance is None else f"{base} · {float(distance):.1f} m"
def _perception_color(label: str, *, alpha: int) -> list[int]:
colors = {
"person": (255, 99, 132),
"car": (64, 180, 255),
"truck": (255, 180, 64),
"bus": (255, 210, 64),
"bicycle": (110, 240, 155),
"motorcycle": (170, 115, 255),
}
red, green, blue = colors.get(label, (247, 248, 244))
return [red, green, blue, alpha]
def _live_time_panel() -> rrb.TimePanel:
"""Keep the hidden vendor timeline on its native live edge."""
return rrb.TimePanel(