from __future__ import annotations import hashlib import json import struct import subprocess import sys import threading from pathlib import Path import pytest import k1link.viewer.rrd_export as export_module from k1link.mqtt.capture import FRAME_HEADER, RAW_MAGIC from k1link.viewer.rerun_bridge import RerunSceneSettings from k1link.viewer.rrd_export import ( RECORDED_POINTS_VISUALIZER_ID, RECORDED_ROOT_CONTAINER_ID, RECORDED_SPATIAL_VIEW_ID, SESSION_TIMELINE, RrdExportCancelled, RrdExportError, _recorded_blueprint, export_k1mqtt_to_rrd, recorded_blueprint_rrd, ) def _point_payload(x: float) -> bytes: return struct.pack(" bytes: return struct.pack(" Path: capture = directory / "mqtt.raw.k1mqtt" raw = bytearray(RAW_MAGIC) metadata: list[dict[str, object]] = [] epoch_origin_ns = 1_784_124_315_000_000_000 monotonic_origin_ns = 9_000_000_000 for sequence, (topic, payload, session_time_ns) in enumerate(frames, start=1): topic_raw = topic.encode("utf-8") raw.extend(FRAME_HEADER.pack(len(topic_raw), len(payload))) raw.extend(topic_raw) raw.extend(payload) metadata.append( { "record_type": "message", "sequence": sequence, "received_at_epoch_ns": epoch_origin_ns + session_time_ns, "received_monotonic_ns": monotonic_origin_ns + session_time_ns, } ) capture.write_bytes(raw) (directory / "mqtt.metadata.jsonl").write_text( "".join(json.dumps(item) + "\n" for item in metadata), encoding="utf-8", ) return capture def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def _print_rrd_entity(path: Path, entity: str) -> str: completed = subprocess.run( [ sys.executable, "-m", "rerun_cli", "rrd", "print", "-vvv", "--entity", entity, str(path), ], check=True, capture_output=True, text=True, ) return completed.stdout def test_recorded_blueprint_accumulates_point_frames_on_session_timeline() -> None: blueprint = _recorded_blueprint( RerunSceneSettings( point_size=7.5, palette="custom", custom_color="#112233", accumulation_seconds=18.5, show_points=False, show_trajectory=True, show_grid=False, ) ) spatial_view = blueprint.root_container.contents[0] visible_ranges = spatial_view.properties["VisibleTimeRanges"] line_grid = spatial_view.properties["LineGrid3D"] assert visible_ranges.ranges.as_arrow_array().to_pylist() == [ { "timeline": SESSION_TIMELINE, "range": { "start": -18_500_000_000, "end": 0, }, } ] assert line_grid.visible.as_arrow_array().to_pylist() == [False] point_behavior, point_visualizer = spatial_view.visualizer_overrides["/world/points"] trajectory_behavior = spatial_view.visualizer_overrides["/world/trajectory"] assert point_behavior.visible.as_arrow_array().to_pylist() == [False] assert trajectory_behavior.visible.as_arrow_array().to_pylist() == [True] point_overrides = { str(batch.component_descriptor()): batch.as_arrow_array().to_pylist() for batch in point_visualizer.overrides } assert point_overrides == { "Points3D:colors": [0x112233FF], # Negative values are Rerun's encoding for screen-space UI points. "Points3D:radii": [-7.5], } def test_zero_accumulation_uses_latest_frame_instead_of_empty_time_range() -> None: blueprint = _recorded_blueprint( RerunSceneSettings(accumulation_seconds=0.0, show_grid=True) ) spatial_view = blueprint.root_container.contents[0] assert "VisibleTimeRanges" not in spatial_view.properties point_behavior, point_visualizer = spatial_view.visualizer_overrides["/world/points"] trajectory_behavior = spatial_view.visualizer_overrides["/world/trajectory"] assert point_behavior.visible.as_arrow_array().to_pylist() == [True] assert trajectory_behavior.visible.as_arrow_array().to_pylist() == [True] point_components = { str(batch.component_descriptor()) for batch in point_visualizer.overrides } assert point_components == {"Points3D:radii"} def test_dynamic_blueprint_reuses_scene_ids_without_playback_mutation() -> None: first = _recorded_blueprint( RerunSceneSettings(show_points=False, show_trajectory=True), include_initial_playback_state=False, ) second = _recorded_blueprint( RerunSceneSettings(show_points=True, show_trajectory=False), include_initial_playback_state=False, ) assert not hasattr(first, "time_panel") assert not hasattr(second, "time_panel") assert first.root_container.id == second.root_container.id == RECORDED_ROOT_CONTAINER_ID first_view = first.root_container.contents[0] second_view = second.root_container.contents[0] assert first_view.id == second_view.id == RECORDED_SPATIAL_VIEW_ID first_point_behavior, first_point_visualizer = first_view.visualizer_overrides[ "/world/points" ] second_point_behavior, second_point_visualizer = second_view.visualizer_overrides[ "/world/points" ] first_trajectory = first_view.visualizer_overrides["/world/trajectory"] second_trajectory = second_view.visualizer_overrides["/world/trajectory"] assert first_point_visualizer.id == RECORDED_POINTS_VISUALIZER_ID assert second_point_visualizer.id == RECORDED_POINTS_VISUALIZER_ID assert first_point_behavior.visible.as_arrow_array().to_pylist() == [False] assert second_point_behavior.visible.as_arrow_array().to_pylist() == [True] assert first_trajectory.visible.as_arrow_array().to_pylist() == [True] assert second_trajectory.visible.as_arrow_array().to_pylist() == [False] payload = recorded_blueprint_rrd( RerunSceneSettings(show_points=False, show_trajectory=False), recording_id="stable-recording", ) assert b"PlayState" not in payload assert b"play_state" not in payload assert str(RECORDED_ROOT_CONTAINER_ID).encode() in payload assert str(RECORDED_SPATIAL_VIEW_ID).encode() in payload assert str(RECORDED_POINTS_VISUALIZER_ID).encode() in payload def test_export_preserves_every_decodable_frame_and_source_timeline(tmp_path: Path) -> None: capture = _write_capture( tmp_path, [ ("lixel/application/report/heartbeat", b"opaque", 0), ("RealtimePointcloud", _point_payload(1.0), 100_000_000), ("RealtimePath", _pose_payload(1.0), 600_000_000), ("RealtimePointcloud", _point_payload(2.0), 900_000_000), ("RealtimePath", _pose_payload(1.2), 1_200_000_000), ("lixel/application/report/heartbeat", b"opaque-tail", 9_500_000_000), ], ) output = tmp_path / "session.rrd" summary = export_k1mqtt_to_rrd(capture, output) assert summary["source_messages"] == 6 assert summary["decoded_messages"] == 4 assert summary["point_frames"] == 2 assert summary["pose_frames"] == 2 assert summary["ignored_messages"] == 2 assert summary["points"] == 2 assert summary["trajectory_poses"] == 2 assert summary["trajectory_updates"] == 2 assert summary["session_origin_monotonic_ns"] == 9_000_000_000 assert summary["timeline"] == "session_time" assert summary["timeline_start_ns"] == 0 assert summary["timeline_end_ns"] == 1_200_000_000 assert summary["timeline_span_ns"] == 1_200_000_000 assert summary["first_decoded_time_ns"] == 100_000_000 assert summary["last_decoded_time_ns"] == 1_200_000_000 assert summary["source_sha256"] == _sha256(capture) assert summary["rrd_bytes"] == output.stat().st_size assert summary["rrd_sha256"] == _sha256(output) assert output.stat().st_size > 0 assert list(tmp_path.glob(".session.rrd.*.tmp")) == [] origin_table = _print_rrd_entity(output, "/__mission_core/session_origin") assert "/__mission_core/session_origin" in origin_table assert "session_time" in origin_table assert "P0D" in origin_table assert "[true]" in origin_table def test_atomic_rename_failure_preserves_existing_recording( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: capture = _write_capture( tmp_path, [("RealtimePointcloud", _point_payload(1.0), 0)], ) output = tmp_path / "session.rrd" trusted = b"existing-good-recording" output.write_bytes(trusted) def fail_replace(_source: Path, _destination: Path) -> None: raise OSError("synthetic rename failure") monkeypatch.setattr(export_module.os, "replace", fail_replace) with pytest.raises(RrdExportError, match="synthetic rename failure"): export_k1mqtt_to_rrd(capture, output) assert output.read_bytes() == trusted assert list(tmp_path.glob(".session.rrd.*.tmp")) == [] def test_missing_monotonic_metadata_never_replaces_existing_recording(tmp_path: Path) -> None: capture = _write_capture( tmp_path, [("RealtimePointcloud", _point_payload(1.0), 0)], ) (tmp_path / "mqtt.metadata.jsonl").unlink() output = tmp_path / "session.rrd" output.write_bytes(b"existing-good-recording") with pytest.raises(RrdExportError, match="received_monotonic_ns"): export_k1mqtt_to_rrd(capture, output) assert output.read_bytes() == b"existing-good-recording" assert list(tmp_path.glob(".session.rrd.*.tmp")) == [] def test_export_preserves_valid_prefix_before_crash_metadata_tail(tmp_path: Path) -> None: capture = _write_capture( tmp_path, [ ("RealtimePointcloud", _point_payload(1.0), 100_000_000), ("RealtimePointcloud", _point_payload(2.0), 200_000_000), ], ) metadata_path = tmp_path / "mqtt.metadata.jsonl" first_line = metadata_path.read_text(encoding="utf-8").splitlines(keepends=True)[0] metadata_path.write_text( first_line + '{"record_type":"message"', encoding="utf-8", ) output = tmp_path / "session.rrd" summary = export_k1mqtt_to_rrd(capture, output) assert summary["source_messages"] == 1 assert summary["decoded_messages"] == 1 assert summary["point_frames"] == 1 assert summary["timeline_end_ns"] == 0 assert output.stat().st_size > 0 def test_export_honors_cooperative_cancellation_without_publishing(tmp_path: Path) -> None: capture = _write_capture( tmp_path, [("RealtimePointcloud", _point_payload(1.0), 0)], ) output = tmp_path / "session.rrd" output.write_bytes(b"trusted-existing-recording") cancelled = threading.Event() cancelled.set() with pytest.raises(RrdExportCancelled): export_k1mqtt_to_rrd(capture, output, cancel_event=cancelled) assert output.read_bytes() == b"trusted-existing-recording" assert list(tmp_path.glob(".session.rrd.*.tmp")) == []