778 lines
29 KiB
Python
778 lines
29 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
import k1link.device_plugins.xgrids_k1.rrd_export as export_module
|
|
import k1link.viewer.recorded as viewer_recorded_module
|
|
from k1link.device_plugins.xgrids_k1.mqtt.capture import (
|
|
CAPTURE_CLOCK_FILENAME,
|
|
FRAME_HEADER,
|
|
RAW_MAGIC,
|
|
)
|
|
from k1link.device_plugins.xgrids_k1.rrd_export import (
|
|
RECORDED_CAMERA_VIEW_ID,
|
|
RECORDED_METRICS_VIEW_ID,
|
|
RECORDED_PERCEPTION_3D_VIEW_ID,
|
|
RECORDED_POINTS_VISUALIZER_ID,
|
|
RECORDED_ROOT_CONTAINER_ID,
|
|
RECORDED_SPATIAL_VIEW_ID,
|
|
RECORDED_VIEW_POINT_DECIMATION_THRESHOLD,
|
|
RECORDED_VIEW_POINT_FRAME_STRIDE,
|
|
RECORDED_VIEW_POINT_STRIDE,
|
|
SESSION_TIMELINE,
|
|
RrdExportCancelled,
|
|
RrdExportError,
|
|
_recorded_blueprint,
|
|
_recorded_view_points,
|
|
_should_publish_recorded_point_frame,
|
|
export_k1mqtt_to_rrd,
|
|
recorded_blueprint_rrd,
|
|
)
|
|
from k1link.viewer.recorded import (
|
|
recorded_blueprint as viewer_recorded_blueprint,
|
|
)
|
|
from k1link.viewer.recorded import (
|
|
recorded_blueprint_rrd as viewer_recorded_blueprint_rrd,
|
|
)
|
|
from k1link.viewer.rerun_bridge import RerunSceneSettings
|
|
|
|
|
|
def _point_payload(x: float) -> bytes:
|
|
return struct.pack("<III", 16, 0, 0) + struct.pack(
|
|
"<fffBBBB",
|
|
x,
|
|
-2.0,
|
|
3.0,
|
|
10,
|
|
20,
|
|
30,
|
|
40,
|
|
)
|
|
|
|
|
|
def test_recorded_operator_projection_thins_only_dense_point_batches() -> None:
|
|
assert RECORDED_VIEW_POINT_FRAME_STRIDE == 5
|
|
count = RECORDED_VIEW_POINT_DECIMATION_THRESHOLD + 3
|
|
positions = export_module.np.arange(
|
|
count * 3,
|
|
dtype=export_module.np.float32,
|
|
).reshape((-1, 3))
|
|
intensities = export_module.np.arange(count, dtype=export_module.np.uint8)
|
|
rgb = export_module.np.arange(count * 3, dtype=export_module.np.uint8).reshape((-1, 3))
|
|
|
|
projected_positions, projected_intensities, projected_rgb = _recorded_view_points(
|
|
positions,
|
|
intensities,
|
|
rgb,
|
|
)
|
|
|
|
assert projected_rgb is not None
|
|
assert export_module.np.array_equal(
|
|
projected_positions,
|
|
positions[::RECORDED_VIEW_POINT_STRIDE],
|
|
)
|
|
assert export_module.np.array_equal(
|
|
projected_intensities,
|
|
intensities[::RECORDED_VIEW_POINT_STRIDE],
|
|
)
|
|
assert export_module.np.array_equal(projected_rgb, rgb[::RECORDED_VIEW_POINT_STRIDE])
|
|
|
|
small_positions = positions[:RECORDED_VIEW_POINT_DECIMATION_THRESHOLD]
|
|
small_intensities = intensities[:RECORDED_VIEW_POINT_DECIMATION_THRESHOLD]
|
|
same_positions, same_intensities, no_rgb = _recorded_view_points(
|
|
small_positions,
|
|
small_intensities,
|
|
None,
|
|
)
|
|
assert same_positions is small_positions
|
|
assert same_intensities is small_intensities
|
|
assert no_rgb is None
|
|
|
|
|
|
def test_recorded_operator_projection_uses_stable_two_hz_point_cadence() -> None:
|
|
published = [
|
|
frame_number
|
|
for frame_number in range(1, 13)
|
|
if _should_publish_recorded_point_frame(frame_number)
|
|
]
|
|
|
|
assert published == [1, 6, 11]
|
|
with pytest.raises(ValueError, match="positive"):
|
|
_should_publish_recorded_point_frame(0)
|
|
|
|
|
|
def _pose_payload(x: float) -> bytes:
|
|
return struct.pack("<ffffffff", x, 2.0, 3.0, 99.0, 1.0, 0.0, 0.0, 0.0)
|
|
|
|
|
|
def _varint(value: int) -> bytes:
|
|
encoded = bytearray()
|
|
while value > 0x7F:
|
|
encoded.append((value & 0x7F) | 0x80)
|
|
value >>= 7
|
|
encoded.append(value)
|
|
return bytes(encoded)
|
|
|
|
|
|
def _proto_key(number: int, wire_type: int) -> bytes:
|
|
return _varint((number << 3) | wire_type)
|
|
|
|
|
|
def _proto_uint(number: int, value: int) -> bytes:
|
|
return _proto_key(number, 0) + _varint(value)
|
|
|
|
|
|
def _proto_bytes(number: int, value: bytes) -> bytes:
|
|
return _proto_key(number, 2) + _varint(len(value)) + value
|
|
|
|
|
|
def _proto_float32(number: int, value: float) -> bytes:
|
|
return _proto_key(number, 5) + struct.pack("<f", value)
|
|
|
|
|
|
def _modeling_payload(*, distance: float, speed: float, scan_ticks: int) -> bytes:
|
|
status = _proto_float32(1, distance) + _proto_float32(2, speed) + _proto_uint(3, scan_ticks)
|
|
return _proto_uint(2, 0) + _proto_bytes(3, status)
|
|
|
|
|
|
def _write_capture(
|
|
directory: Path,
|
|
frames: list[tuple[str, bytes, int]],
|
|
*,
|
|
capture_clock_offsets_ns: tuple[int, int] | None = None,
|
|
) -> Path:
|
|
capture = directory / "mqtt.raw.k1mqtt"
|
|
raw = bytearray(RAW_MAGIC)
|
|
metadata: list[dict[str, object]] = []
|
|
epoch_origin_ns = 1_784_124_315_000_000_000
|
|
monotonic_origin_ns = 9_000_000_000
|
|
for sequence, (topic, payload, session_time_ns) in enumerate(frames, start=1):
|
|
topic_raw = topic.encode("utf-8")
|
|
raw.extend(FRAME_HEADER.pack(len(topic_raw), len(payload)))
|
|
raw.extend(topic_raw)
|
|
raw.extend(payload)
|
|
metadata.append(
|
|
{
|
|
"record_type": "message",
|
|
"sequence": sequence,
|
|
"received_at_epoch_ns": epoch_origin_ns + session_time_ns,
|
|
"received_monotonic_ns": monotonic_origin_ns + session_time_ns,
|
|
}
|
|
)
|
|
capture.write_bytes(raw)
|
|
(directory / "mqtt.metadata.jsonl").write_text(
|
|
"".join(json.dumps(item) + "\n" for item in metadata),
|
|
encoding="utf-8",
|
|
)
|
|
if capture_clock_offsets_ns is not None:
|
|
started_offset_ns, completed_offset_ns = capture_clock_offsets_ns
|
|
(directory / "mqtt.timeline.origin.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": 1,
|
|
"started_at_epoch_ns": epoch_origin_ns + started_offset_ns,
|
|
"started_monotonic_ns": monotonic_origin_ns + started_offset_ns,
|
|
},
|
|
separators=(",", ":"),
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
(directory / CAPTURE_CLOCK_FILENAME).write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": 1,
|
|
"started_at_epoch_ns": epoch_origin_ns + started_offset_ns,
|
|
"started_monotonic_ns": monotonic_origin_ns + started_offset_ns,
|
|
"completed_at_epoch_ns": epoch_origin_ns + completed_offset_ns,
|
|
"completed_monotonic_ns": monotonic_origin_ns + completed_offset_ns,
|
|
},
|
|
separators=(",", ":"),
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
return capture
|
|
|
|
|
|
def _sha256(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def _print_rrd_entity(path: Path, entity: str) -> str:
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"rerun_cli",
|
|
"rrd",
|
|
"print",
|
|
"-vvv",
|
|
"--entity",
|
|
entity,
|
|
str(path),
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return completed.stdout
|
|
|
|
|
|
def test_recorded_blueprint_accumulates_point_frames_on_session_timeline() -> None:
|
|
blueprint = _recorded_blueprint(
|
|
RerunSceneSettings(
|
|
point_size=7.5,
|
|
palette="custom",
|
|
custom_color="#112233",
|
|
accumulation_seconds=18.5,
|
|
show_points=False,
|
|
show_trajectory=True,
|
|
show_grid=False,
|
|
)
|
|
)
|
|
spatial_view = blueprint.root_container.contents[0]
|
|
visible_ranges = spatial_view.properties["VisibleTimeRanges"]
|
|
line_grid = spatial_view.properties["LineGrid3D"]
|
|
|
|
assert visible_ranges.ranges.as_arrow_array().to_pylist() == []
|
|
assert line_grid.visible.as_arrow_array().to_pylist() == [False]
|
|
point_behavior, point_visualizer, point_time_ranges = spatial_view.visualizer_overrides[
|
|
"/world/points"
|
|
]
|
|
trajectory_behavior, trajectory_time_ranges = spatial_view.visualizer_overrides[
|
|
"/world/trajectory"
|
|
]
|
|
expected_accumulation = [
|
|
{
|
|
"timeline": SESSION_TIMELINE,
|
|
"range": {
|
|
"start": -18_500_000_000,
|
|
"end": 0,
|
|
},
|
|
}
|
|
]
|
|
assert point_time_ranges.ranges.as_arrow_array().to_pylist() == expected_accumulation
|
|
assert trajectory_time_ranges.ranges.as_arrow_array().to_pylist() == expected_accumulation
|
|
perception_behavior = spatial_view.visualizer_overrides["/world/perception"]
|
|
assert point_behavior.visible.as_arrow_array().to_pylist() == [False]
|
|
assert trajectory_behavior.visible.as_arrow_array().to_pylist() == [True]
|
|
assert perception_behavior.visible.as_arrow_array().to_pylist() == [False]
|
|
perception_3d_view = blueprint.root_container.contents[2]
|
|
assert perception_3d_view.origin == "/world"
|
|
assert "VisibleTimeRanges" not in perception_3d_view.properties
|
|
assert perception_3d_view.visualizer_overrides[
|
|
"/world/perception"
|
|
].visible.as_arrow_array().to_pylist() == [True]
|
|
assert perception_3d_view.visualizer_overrides[
|
|
"/world/perception/lidar"
|
|
].visible.as_arrow_array().to_pylist() == [False]
|
|
point_overrides = {
|
|
str(batch.component_descriptor()): batch.as_arrow_array().to_pylist()
|
|
for batch in point_visualizer.overrides
|
|
}
|
|
assert point_overrides == {
|
|
"Points3D:colors": [0x112233FF],
|
|
# Negative values are Rerun's encoding for screen-space UI points.
|
|
"Points3D:radii": [-7.5],
|
|
}
|
|
|
|
|
|
def test_zero_accumulation_uses_latest_frame_instead_of_empty_time_range() -> None:
|
|
blueprint = _recorded_blueprint(RerunSceneSettings(accumulation_seconds=0.0, show_grid=True))
|
|
spatial_view = blueprint.root_container.contents[0]
|
|
|
|
assert spatial_view.properties["VisibleTimeRanges"].ranges.as_arrow_array().to_pylist() == []
|
|
point_behavior, point_visualizer = spatial_view.visualizer_overrides["/world/points"]
|
|
(trajectory_behavior,) = spatial_view.visualizer_overrides["/world/trajectory"]
|
|
assert point_behavior.visible.as_arrow_array().to_pylist() == [True]
|
|
assert trajectory_behavior.visible.as_arrow_array().to_pylist() == [True]
|
|
point_components = {str(batch.component_descriptor()) for batch in point_visualizer.overrides}
|
|
assert point_components == {"Points3D:radii"}
|
|
|
|
|
|
def test_cuboids_are_overlaid_on_the_stable_spatial_view_without_unifying_video() -> None:
|
|
baseline = viewer_recorded_blueprint(
|
|
RerunSceneSettings(accumulation_seconds=12.0),
|
|
include_initial_playback_state=False,
|
|
unified_perception=False,
|
|
show_cuboids_3d=False,
|
|
)
|
|
blueprint = viewer_recorded_blueprint(
|
|
RerunSceneSettings(accumulation_seconds=12.0),
|
|
include_initial_playback_state=False,
|
|
unified_perception=False,
|
|
show_cuboids_3d=True,
|
|
)
|
|
spatial_view = blueprint.root_container.contents[0]
|
|
|
|
assert spatial_view.id == baseline.root_container.contents[0].id
|
|
assert spatial_view.visualizer_overrides[
|
|
"/world/perception"
|
|
].visible.as_arrow_array().to_pylist() == [True]
|
|
assert spatial_view.visualizer_overrides[
|
|
"/world/perception/lidar"
|
|
].visible.as_arrow_array().to_pylist() == [False]
|
|
assert spatial_view.visualizer_overrides[
|
|
"/world/perception/support"
|
|
].visible.as_arrow_array().to_pylist() == [False]
|
|
assert spatial_view.visualizer_overrides[
|
|
"/world/perception/semantic_points"
|
|
].visible.as_arrow_array().to_pylist() == [False]
|
|
assert spatial_view.visualizer_overrides[
|
|
"/world/perception/boxes3d"
|
|
].visible.as_arrow_array().to_pylist() == [True]
|
|
|
|
|
|
def test_dynamic_blueprint_reuses_scene_ids_without_playback_mutation() -> None:
|
|
first = _recorded_blueprint(
|
|
RerunSceneSettings(show_points=False, show_trajectory=True),
|
|
include_initial_playback_state=False,
|
|
)
|
|
second = _recorded_blueprint(
|
|
RerunSceneSettings(show_points=True, show_trajectory=False),
|
|
include_initial_playback_state=False,
|
|
)
|
|
|
|
assert not hasattr(first, "time_panel")
|
|
assert not hasattr(second, "time_panel")
|
|
assert first.root_container.id == second.root_container.id == RECORDED_ROOT_CONTAINER_ID
|
|
first_view = first.root_container.contents[0]
|
|
second_view = second.root_container.contents[0]
|
|
assert first_view.id == second_view.id == RECORDED_SPATIAL_VIEW_ID
|
|
assert first.root_container.contents[1].id == RECORDED_CAMERA_VIEW_ID
|
|
assert second.root_container.contents[1].id == RECORDED_CAMERA_VIEW_ID
|
|
assert first.root_container.contents[2].id == RECORDED_PERCEPTION_3D_VIEW_ID
|
|
assert second.root_container.contents[2].id == RECORDED_PERCEPTION_3D_VIEW_ID
|
|
assert first.root_container.contents[3].id == RECORDED_METRICS_VIEW_ID
|
|
assert second.root_container.contents[3].id == RECORDED_METRICS_VIEW_ID
|
|
|
|
first_point_behavior, first_point_visualizer, first_point_time_ranges = (
|
|
first_view.visualizer_overrides["/world/points"]
|
|
)
|
|
second_point_behavior, second_point_visualizer, second_point_time_ranges = (
|
|
second_view.visualizer_overrides["/world/points"]
|
|
)
|
|
first_trajectory, first_trajectory_time_ranges = first_view.visualizer_overrides[
|
|
"/world/trajectory"
|
|
]
|
|
second_trajectory, second_trajectory_time_ranges = second_view.visualizer_overrides[
|
|
"/world/trajectory"
|
|
]
|
|
assert first_point_visualizer.id == RECORDED_POINTS_VISUALIZER_ID
|
|
assert second_point_visualizer.id == RECORDED_POINTS_VISUALIZER_ID
|
|
assert first_point_behavior.visible.as_arrow_array().to_pylist() == [False]
|
|
assert second_point_behavior.visible.as_arrow_array().to_pylist() == [True]
|
|
assert first_trajectory.visible.as_arrow_array().to_pylist() == [True]
|
|
assert second_trajectory.visible.as_arrow_array().to_pylist() == [False]
|
|
assert (
|
|
first_point_time_ranges.ranges.as_arrow_array().to_pylist()
|
|
== first_trajectory_time_ranges.ranges.as_arrow_array().to_pylist()
|
|
)
|
|
assert (
|
|
second_point_time_ranges.ranges.as_arrow_array().to_pylist()
|
|
== second_trajectory_time_ranges.ranges.as_arrow_array().to_pylist()
|
|
)
|
|
|
|
payload = recorded_blueprint_rrd(
|
|
RerunSceneSettings(show_points=False, show_trajectory=False),
|
|
recording_id="stable-recording",
|
|
)
|
|
assert b"PlayState" not in payload
|
|
assert b"play_state" not in payload
|
|
assert str(RECORDED_ROOT_CONTAINER_ID).encode() in payload
|
|
assert str(RECORDED_SPATIAL_VIEW_ID).encode() in payload
|
|
assert str(RECORDED_POINTS_VISUALIZER_ID).encode() in payload
|
|
|
|
|
|
def test_viewer_blueprint_reset_is_bounded_and_recreates_render_views() -> None:
|
|
initial = viewer_recorded_blueprint(
|
|
RerunSceneSettings(accumulation_seconds=12.0),
|
|
include_initial_playback_state=False,
|
|
active_view="perception",
|
|
view_reset_generation=0,
|
|
)
|
|
reset = viewer_recorded_blueprint(
|
|
RerunSceneSettings(accumulation_seconds=12.0),
|
|
include_initial_playback_state=False,
|
|
active_view="perception",
|
|
view_reset_generation=1,
|
|
)
|
|
restored = viewer_recorded_blueprint(
|
|
RerunSceneSettings(accumulation_seconds=12.0),
|
|
include_initial_playback_state=False,
|
|
active_view="perception",
|
|
view_reset_generation=0,
|
|
)
|
|
|
|
assert initial.root_container.id == restored.root_container.id
|
|
assert reset.root_container.id != initial.root_container.id
|
|
assert initial.root_container.contents[0].id != reset.root_container.contents[0].id
|
|
assert restored.root_container.contents[0].id == initial.root_container.contents[0].id
|
|
assert initial.root_container.active_tab == 1
|
|
assert reset.root_container.active_tab == 1
|
|
assert initial.root_container.contents[1].id != reset.root_container.contents[1].id
|
|
assert initial.root_container.contents[2].id != reset.root_container.contents[2].id
|
|
assert restored.root_container.contents[1].id == initial.root_container.contents[1].id
|
|
assert restored.root_container.contents[2].id == initial.root_container.contents[2].id
|
|
cuboids = viewer_recorded_blueprint(
|
|
RerunSceneSettings(accumulation_seconds=12.0),
|
|
include_initial_playback_state=False,
|
|
active_view="perception3d",
|
|
)
|
|
assert cuboids.root_container.active_tab == 2
|
|
assert cuboids.root_container.id != initial.root_container.id
|
|
cuboid_view = cuboids.root_container.contents[2]
|
|
assert cuboid_view.origin == "/world"
|
|
assert "VisibleTimeRanges" not in cuboid_view.properties
|
|
cuboid_points, cuboid_point_visualizer = cuboid_view.visualizer_overrides["/world/points"]
|
|
cuboid_perception = cuboid_view.visualizer_overrides["/world/perception"]
|
|
cuboid_diagnostic_lidar = cuboid_view.visualizer_overrides["/world/perception/lidar"]
|
|
assert cuboid_points.visible.as_arrow_array().to_pylist() == [True]
|
|
assert (
|
|
cuboid_point_visualizer.id
|
|
!= initial.root_container.contents[0].visualizer_overrides["/world/points"][1].id
|
|
)
|
|
assert cuboid_perception.visible.as_arrow_array().to_pylist() == [True]
|
|
assert cuboid_diagnostic_lidar.visible.as_arrow_array().to_pylist() == [False]
|
|
perception_behavior = initial.root_container.contents[0].visualizer_overrides[
|
|
"/world/perception"
|
|
]
|
|
assert perception_behavior.visible.as_arrow_array().to_pylist() == [False]
|
|
|
|
|
|
def test_recorded_blueprint_layer_updates_preserve_store_until_explicit_reset(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
activations: list[tuple[object, bool, bool]] = []
|
|
send_blueprint = viewer_recorded_module.bindings.send_blueprint
|
|
|
|
def capture_activation(
|
|
storage: object,
|
|
make_active: bool,
|
|
make_default: bool,
|
|
recording: object,
|
|
) -> None:
|
|
activations.append((storage, make_active, make_default))
|
|
send_blueprint(storage, make_active, make_default, recording)
|
|
|
|
monkeypatch.setattr(
|
|
viewer_recorded_module.bindings,
|
|
"send_blueprint",
|
|
capture_activation,
|
|
)
|
|
session_id = "b" * 32
|
|
initial = viewer_recorded_blueprint_rrd(
|
|
RerunSceneSettings(accumulation_seconds=12.0),
|
|
recording_id="stable-camera",
|
|
blueprint_session_id=session_id,
|
|
unified_perception=True,
|
|
)
|
|
layers_enabled = viewer_recorded_blueprint_rrd(
|
|
RerunSceneSettings(accumulation_seconds=12.0),
|
|
recording_id="stable-camera",
|
|
blueprint_session_id=session_id,
|
|
unified_perception=True,
|
|
show_detections_2d=True,
|
|
show_segmentation=True,
|
|
show_cuboids_3d=True,
|
|
)
|
|
reset = viewer_recorded_blueprint_rrd(
|
|
RerunSceneSettings(accumulation_seconds=12.0),
|
|
recording_id="stable-camera",
|
|
blueprint_session_id=session_id,
|
|
view_reset_generation=1,
|
|
unified_perception=True,
|
|
show_detections_2d=True,
|
|
show_segmentation=True,
|
|
show_cuboids_3d=True,
|
|
)
|
|
|
|
store_pattern = rb"rec_[0-9a-f]{32}"
|
|
initial_store_ids = set(re.findall(store_pattern, initial))
|
|
layer_store_ids = set(re.findall(store_pattern, layers_enabled))
|
|
reset_store_ids = set(re.findall(store_pattern, reset))
|
|
assert len(initial_store_ids) == 1
|
|
assert layer_store_ids == initial_store_ids
|
|
assert len(reset_store_ids) == 1
|
|
assert reset_store_ids.isdisjoint(initial_store_ids)
|
|
assert [activation[1:] for activation in activations] == [
|
|
(True, False),
|
|
(True, False),
|
|
(True, False),
|
|
]
|
|
assert activations[0][0] is activations[1][0]
|
|
assert activations[2][0] is not activations[0][0]
|
|
assert len(initial) < 350_000
|
|
assert len(layers_enabled) < 350_000
|
|
assert len(reset) < 350_000
|
|
|
|
|
|
def test_viewer_blueprint_unifies_original_video_and_independent_ai_layers() -> None:
|
|
blueprint = viewer_recorded_blueprint(
|
|
RerunSceneSettings(accumulation_seconds=12.0),
|
|
include_initial_playback_state=False,
|
|
unified_perception=True,
|
|
show_detections_2d=True,
|
|
show_segmentation=True,
|
|
show_cuboids_3d=False,
|
|
)
|
|
|
|
assert type(blueprint.root_container).__name__ == "Horizontal"
|
|
camera_view, spatial_view = blueprint.root_container.contents
|
|
assert camera_view.origin == "/perception/camera"
|
|
assert spatial_view.origin == "/world"
|
|
assert camera_view.visualizer_overrides[
|
|
"/perception/camera/image"
|
|
].visible.as_arrow_array().to_pylist() == [True]
|
|
assert camera_view.visualizer_overrides[
|
|
"/perception/camera/detections"
|
|
].visible.as_arrow_array().to_pylist() == [True]
|
|
assert camera_view.visualizer_overrides[
|
|
"/perception/camera/segmentation"
|
|
].visible.as_arrow_array().to_pylist() == [True]
|
|
(semantic_behavior,) = spatial_view.visualizer_overrides["/world/perception/semantic_points"]
|
|
(cuboid_behavior,) = spatial_view.visualizer_overrides["/world/perception/boxes3d"]
|
|
assert semantic_behavior.visible.as_arrow_array().to_pylist() == [True]
|
|
assert cuboid_behavior.visible.as_arrow_array().to_pylist() == [False]
|
|
visible_ranges = spatial_view.properties["VisibleTimeRanges"]
|
|
assert visible_ranges.ranges.as_arrow_array().to_pylist() == []
|
|
_point_behavior, _point_visualizer, point_time_ranges = spatial_view.visualizer_overrides[
|
|
"/world/points"
|
|
]
|
|
assert point_time_ranges.ranges.as_arrow_array().to_pylist() == [
|
|
{
|
|
"timeline": SESSION_TIMELINE,
|
|
"range": {"start": -12_000_000_000, "end": 0},
|
|
}
|
|
]
|
|
|
|
|
|
def test_export_preserves_every_decodable_frame_and_source_timeline(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[
|
|
("lixel/application/report/heartbeat", b"opaque", 0),
|
|
("RealtimePointcloud", _point_payload(1.0), 100_000_000),
|
|
("RealtimePath", _pose_payload(1.0), 600_000_000),
|
|
("RealtimePointcloud", _point_payload(2.0), 900_000_000),
|
|
("RealtimePath", _pose_payload(1.2), 1_200_000_000),
|
|
("lixel/application/report/heartbeat", b"opaque-tail", 9_500_000_000),
|
|
],
|
|
capture_clock_offsets_ns=(-500_000_000, 10_000_000_000),
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
|
|
summary = export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert summary["source_messages"] == 6
|
|
assert summary["decoded_messages"] == 4
|
|
assert summary["point_frames"] == 2
|
|
assert summary["pose_frames"] == 2
|
|
assert summary["ignored_messages"] == 2
|
|
assert summary["points"] == 2
|
|
assert summary["trajectory_poses"] == 2
|
|
assert summary["trajectory_updates"] == 2
|
|
assert summary["session_origin_monotonic_ns"] == 8_500_000_000
|
|
assert summary["timeline"] == "session_time"
|
|
assert summary["timeline_start_ns"] == 0
|
|
assert summary["timeline_end_ns"] == 10_500_000_000
|
|
assert summary["timeline_span_ns"] == 10_500_000_000
|
|
assert summary["first_decoded_time_ns"] == 600_000_000
|
|
assert summary["last_decoded_time_ns"] == 1_700_000_000
|
|
assert summary["source_sha256"] == _sha256(capture)
|
|
assert summary["rrd_bytes"] == output.stat().st_size
|
|
assert summary["rrd_sha256"] == _sha256(output)
|
|
assert output.stat().st_size > 0
|
|
assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
|
|
|
|
origin_table = _print_rrd_entity(output, "/__mission_core/session_origin")
|
|
assert "/__mission_core/session_origin" in origin_table
|
|
assert "session_time" in origin_table
|
|
assert "P0D" in origin_table
|
|
assert "[true]" in origin_table
|
|
end_table = _print_rrd_entity(output, "/__mission_core/session_end")
|
|
assert "/__mission_core/session_end" in end_table
|
|
assert "PT10.5S" in end_table
|
|
|
|
|
|
def test_export_records_device_route_time_series_on_shared_timeline(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[
|
|
("RealtimePointcloud", _point_payload(1.0), 0),
|
|
(
|
|
"lixel/application/report/modeling",
|
|
_modeling_payload(distance=2.25, speed=0.75, scan_ticks=20),
|
|
500_000_000,
|
|
),
|
|
(
|
|
"lixel/application/report/modeling",
|
|
_modeling_payload(distance=3.5, speed=1.25, scan_ticks=22),
|
|
1_000_000_000,
|
|
),
|
|
],
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
|
|
summary = export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert summary["modeling_reports"] == 2
|
|
assert summary["decoded_messages"] == 1
|
|
assert summary["ignored_messages"] == 0
|
|
assert summary["timeline_end_ns"] == 1_000_000_000
|
|
distance_table = _print_rrd_entity(output, "/metrics/device/route_distance_meters")
|
|
elapsed_table = _print_rrd_entity(output, "/metrics/device/scan_elapsed_seconds")
|
|
assert "2.25" in distance_table and "3.5" in distance_table
|
|
assert "10.0" in elapsed_table and "11.0" in elapsed_table
|
|
|
|
|
|
def test_export_uses_explicit_catalog_bound_sealed_clock(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[("RealtimePointcloud", _point_payload(1.0), 500_000_000)],
|
|
)
|
|
origin = {
|
|
"schema_version": 1,
|
|
"started_at_epoch_ns": 1_784_124_314_000_000_000,
|
|
"started_monotonic_ns": 8_000_000_000,
|
|
}
|
|
origin_path = tmp_path / "mqtt.timeline.origin.json"
|
|
origin_path.write_text(json.dumps(origin, separators=(",", ":")) + "\n")
|
|
provisional = {
|
|
**origin,
|
|
"completed_at_epoch_ns": 1_784_124_316_000_000_000,
|
|
"completed_monotonic_ns": 10_000_000_000,
|
|
}
|
|
(tmp_path / "mqtt.timeline.json").write_text(
|
|
json.dumps(provisional, separators=(",", ":")) + "\n"
|
|
)
|
|
sealed = {
|
|
**origin,
|
|
"completed_at_epoch_ns": 1_784_124_321_000_000_000,
|
|
"completed_monotonic_ns": 15_000_000_000,
|
|
}
|
|
sealed_payload = (json.dumps(sealed, separators=(",", ":")) + "\n").encode()
|
|
sealed_digest = hashlib.sha256(sealed_payload).hexdigest()
|
|
sealed_path = tmp_path / f"mqtt.timeline.session-{sealed_digest}.json"
|
|
sealed_path.write_bytes(sealed_payload)
|
|
output = tmp_path / "session.rrd"
|
|
|
|
summary = export_k1mqtt_to_rrd(
|
|
capture,
|
|
output,
|
|
capture_clock_path=sealed_path,
|
|
capture_clock_origin_path=origin_path,
|
|
)
|
|
|
|
assert summary["session_origin_monotonic_ns"] == 8_000_000_000
|
|
assert summary["timeline_end_ns"] == 7_000_000_000
|
|
|
|
|
|
def test_export_rejects_malformed_known_modeling_report(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[
|
|
("RealtimePointcloud", _point_payload(1.0), 0),
|
|
("lixel/application/report/modeling", b"malformed", 1_000_000),
|
|
],
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
output.write_bytes(b"trusted-existing-recording")
|
|
|
|
with pytest.raises(RrdExportError, match="modeling report"):
|
|
export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert output.read_bytes() == b"trusted-existing-recording"
|
|
|
|
|
|
def test_atomic_rename_failure_preserves_existing_recording(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[("RealtimePointcloud", _point_payload(1.0), 0)],
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
trusted = b"existing-good-recording"
|
|
output.write_bytes(trusted)
|
|
|
|
def fail_replace(_source: Path, _destination: Path) -> None:
|
|
raise OSError("synthetic rename failure")
|
|
|
|
monkeypatch.setattr(export_module.os, "replace", fail_replace)
|
|
|
|
with pytest.raises(RrdExportError, match="synthetic rename failure"):
|
|
export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert output.read_bytes() == trusted
|
|
assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
|
|
|
|
|
|
def test_missing_monotonic_metadata_never_replaces_existing_recording(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[("RealtimePointcloud", _point_payload(1.0), 0)],
|
|
)
|
|
(tmp_path / "mqtt.metadata.jsonl").unlink()
|
|
output = tmp_path / "session.rrd"
|
|
output.write_bytes(b"existing-good-recording")
|
|
|
|
with pytest.raises(RrdExportError, match="received_monotonic_ns"):
|
|
export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert output.read_bytes() == b"existing-good-recording"
|
|
assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
|
|
|
|
|
|
def test_export_preserves_valid_prefix_before_crash_metadata_tail(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[
|
|
("RealtimePointcloud", _point_payload(1.0), 100_000_000),
|
|
("RealtimePointcloud", _point_payload(2.0), 200_000_000),
|
|
],
|
|
)
|
|
metadata_path = tmp_path / "mqtt.metadata.jsonl"
|
|
first_line = metadata_path.read_text(encoding="utf-8").splitlines(keepends=True)[0]
|
|
metadata_path.write_text(
|
|
first_line + '{"record_type":"message"',
|
|
encoding="utf-8",
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
|
|
summary = export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert summary["source_messages"] == 1
|
|
assert summary["decoded_messages"] == 1
|
|
assert summary["point_frames"] == 1
|
|
assert summary["timeline_end_ns"] == 0
|
|
assert output.stat().st_size > 0
|
|
|
|
|
|
def test_export_honors_cooperative_cancellation_without_publishing(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[("RealtimePointcloud", _point_payload(1.0), 0)],
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
output.write_bytes(b"trusted-existing-recording")
|
|
cancelled = threading.Event()
|
|
cancelled.set()
|
|
|
|
with pytest.raises(RrdExportCancelled):
|
|
export_k1mqtt_to_rrd(capture, output, cancel_event=cancelled)
|
|
|
|
assert output.read_bytes() == b"trusted-existing-recording"
|
|
assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
|