feat(sessions): add durable observation archive and replay API

2026-07-17 17:50:54 +03:00
parent aa2df560b7
commit 656f0c524d
23 changed files with 12492 additions and 5 deletions
@@ -0,0 +1,324 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import struct
+import subprocess
+import sys
+import threading
+from pathlib import Path
+
+import pytest
+
+import k1link.viewer.rrd_export as export_module
+from k1link.mqtt.capture import FRAME_HEADER, RAW_MAGIC
+from k1link.viewer.rerun_bridge import RerunSceneSettings
+from k1link.viewer.rrd_export import (
+    RECORDED_POINTS_VISUALIZER_ID,
+    RECORDED_ROOT_CONTAINER_ID,
+    RECORDED_SPATIAL_VIEW_ID,
+    SESSION_TIMELINE,
+    RrdExportCancelled,
+    RrdExportError,
+    _recorded_blueprint,
+    export_k1mqtt_to_rrd,
+    recorded_blueprint_rrd,
+)
+
+
+def _point_payload(x: float) -> bytes:
+    return struct.pack("<III", 16, 0, 0) + struct.pack(
+        "<fffBBBB",
+        x,
+        -2.0,
+        3.0,
+        10,
+        20,
+        30,
+        40,
+    )
+
+
+def _pose_payload(x: float) -> bytes:
+    return struct.pack("<ffffffff", x, 2.0, 3.0, 99.0, 1.0, 0.0, 0.0, 0.0)
+
+
+def _write_capture(
+    directory: Path,
+    frames: list[tuple[str, bytes, int]],
+) -> Path:
+    capture = directory / "mqtt.raw.k1mqtt"
+    raw = bytearray(RAW_MAGIC)
+    metadata: list[dict[str, object]] = []
+    epoch_origin_ns = 1_784_124_315_000_000_000
+    monotonic_origin_ns = 9_000_000_000
+    for sequence, (topic, payload, session_time_ns) in enumerate(frames, start=1):
+        topic_raw = topic.encode("utf-8")
+        raw.extend(FRAME_HEADER.pack(len(topic_raw), len(payload)))
+        raw.extend(topic_raw)
+        raw.extend(payload)
+        metadata.append(
+            {
+                "record_type": "message",
+                "sequence": sequence,
+                "received_at_epoch_ns": epoch_origin_ns + session_time_ns,
+                "received_monotonic_ns": monotonic_origin_ns + session_time_ns,
+            }
+        )
+    capture.write_bytes(raw)
+    (directory / "mqtt.metadata.jsonl").write_text(
+        "".join(json.dumps(item) + "\n" for item in metadata),
+        encoding="utf-8",
+    )
+    return capture
+
+
+def _sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _print_rrd_entity(path: Path, entity: str) -> str:
+    completed = subprocess.run(
+        [
+            sys.executable,
+            "-m",
+            "rerun_cli",
+            "rrd",
+            "print",
+            "-vvv",
+            "--entity",
+            entity,
+            str(path),
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return completed.stdout
+
+
+def test_recorded_blueprint_accumulates_point_frames_on_session_timeline() -> None:
+    blueprint = _recorded_blueprint(
+        RerunSceneSettings(
+            point_size=7.5,
+            palette="custom",
+            custom_color="#112233",
+            accumulation_seconds=18.5,
+            show_points=False,
+            show_trajectory=True,
+            show_grid=False,
+        )
+    )
+    spatial_view = blueprint.root_container.contents[0]
+    visible_ranges = spatial_view.properties["VisibleTimeRanges"]
+    line_grid = spatial_view.properties["LineGrid3D"]
+
+    assert visible_ranges.ranges.as_arrow_array().to_pylist() == [
+        {
+            "timeline": SESSION_TIMELINE,
+            "range": {
+                "start": -18_500_000_000,
+                "end": 0,
+            },
+        }
+    ]
+    assert line_grid.visible.as_arrow_array().to_pylist() == [False]
+    point_behavior, point_visualizer = spatial_view.visualizer_overrides["/world/points"]
+    trajectory_behavior = spatial_view.visualizer_overrides["/world/trajectory"]
+    assert point_behavior.visible.as_arrow_array().to_pylist() == [False]
+    assert trajectory_behavior.visible.as_arrow_array().to_pylist() == [True]
+    point_overrides = {
+        str(batch.component_descriptor()): batch.as_arrow_array().to_pylist()
+        for batch in point_visualizer.overrides
+    }
+    assert point_overrides == {
+        "Points3D:colors": [0x112233FF],
+        # Negative values are Rerun's encoding for screen-space UI points.
+        "Points3D:radii": [-7.5],
+    }
+
+
+def test_zero_accumulation_uses_latest_frame_instead_of_empty_time_range() -> None:
+    blueprint = _recorded_blueprint(
+        RerunSceneSettings(accumulation_seconds=0.0, show_grid=True)
+    )
+    spatial_view = blueprint.root_container.contents[0]
+
+    assert "VisibleTimeRanges" not in spatial_view.properties
+    point_behavior, point_visualizer = spatial_view.visualizer_overrides["/world/points"]
+    trajectory_behavior = spatial_view.visualizer_overrides["/world/trajectory"]
+    assert point_behavior.visible.as_arrow_array().to_pylist() == [True]
+    assert trajectory_behavior.visible.as_arrow_array().to_pylist() == [True]
+    point_components = {
+        str(batch.component_descriptor()) for batch in point_visualizer.overrides
+    }
+    assert point_components == {"Points3D:radii"}
+
+
+def test_dynamic_blueprint_reuses_scene_ids_without_playback_mutation() -> None:
+    first = _recorded_blueprint(
+        RerunSceneSettings(show_points=False, show_trajectory=True),
+        include_initial_playback_state=False,
+    )
+    second = _recorded_blueprint(
+        RerunSceneSettings(show_points=True, show_trajectory=False),
+        include_initial_playback_state=False,
+    )
+
+    assert not hasattr(first, "time_panel")
+    assert not hasattr(second, "time_panel")
+    assert first.root_container.id == second.root_container.id == RECORDED_ROOT_CONTAINER_ID
+    first_view = first.root_container.contents[0]
+    second_view = second.root_container.contents[0]
+    assert first_view.id == second_view.id == RECORDED_SPATIAL_VIEW_ID
+
+    first_point_behavior, first_point_visualizer = first_view.visualizer_overrides[
+        "/world/points"
+    ]
+    second_point_behavior, second_point_visualizer = second_view.visualizer_overrides[
+        "/world/points"
+    ]
+    first_trajectory = first_view.visualizer_overrides["/world/trajectory"]
+    second_trajectory = second_view.visualizer_overrides["/world/trajectory"]
+    assert first_point_visualizer.id == RECORDED_POINTS_VISUALIZER_ID
+    assert second_point_visualizer.id == RECORDED_POINTS_VISUALIZER_ID
+    assert first_point_behavior.visible.as_arrow_array().to_pylist() == [False]
+    assert second_point_behavior.visible.as_arrow_array().to_pylist() == [True]
+    assert first_trajectory.visible.as_arrow_array().to_pylist() == [True]
+    assert second_trajectory.visible.as_arrow_array().to_pylist() == [False]
+
+    payload = recorded_blueprint_rrd(
+        RerunSceneSettings(show_points=False, show_trajectory=False),
+        recording_id="stable-recording",
+    )
+    assert b"PlayState" not in payload
+    assert b"play_state" not in payload
+    assert str(RECORDED_ROOT_CONTAINER_ID).encode() in payload
+    assert str(RECORDED_SPATIAL_VIEW_ID).encode() in payload
+    assert str(RECORDED_POINTS_VISUALIZER_ID).encode() in payload
+
+
+def test_export_preserves_every_decodable_frame_and_source_timeline(tmp_path: Path) -> None:
+    capture = _write_capture(
+        tmp_path,
+        [
+            ("lixel/application/report/heartbeat", b"opaque", 0),
+            ("RealtimePointcloud", _point_payload(1.0), 100_000_000),
+            ("RealtimePath", _pose_payload(1.0), 600_000_000),
+            ("RealtimePointcloud", _point_payload(2.0), 900_000_000),
+            ("RealtimePath", _pose_payload(1.2), 1_200_000_000),
+            ("lixel/application/report/heartbeat", b"opaque-tail", 9_500_000_000),
+        ],
+    )
+    output = tmp_path / "session.rrd"
+
+    summary = export_k1mqtt_to_rrd(capture, output)
+
+    assert summary["source_messages"] == 6
+    assert summary["decoded_messages"] == 4
+    assert summary["point_frames"] == 2
+    assert summary["pose_frames"] == 2
+    assert summary["ignored_messages"] == 2
+    assert summary["points"] == 2
+    assert summary["trajectory_poses"] == 2
+    assert summary["trajectory_updates"] == 2
+    assert summary["session_origin_monotonic_ns"] == 9_000_000_000
+    assert summary["timeline"] == "session_time"
+    assert summary["timeline_start_ns"] == 0
+    assert summary["timeline_end_ns"] == 1_200_000_000
+    assert summary["timeline_span_ns"] == 1_200_000_000
+    assert summary["first_decoded_time_ns"] == 100_000_000
+    assert summary["last_decoded_time_ns"] == 1_200_000_000
+    assert summary["source_sha256"] == _sha256(capture)
+    assert summary["rrd_bytes"] == output.stat().st_size
+    assert summary["rrd_sha256"] == _sha256(output)
+    assert output.stat().st_size > 0
+    assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
+
+    origin_table = _print_rrd_entity(output, "/__mission_core/session_origin")
+    assert "/__mission_core/session_origin" in origin_table
+    assert "session_time" in origin_table
+    assert "P0D" in origin_table
+    assert "[true]" in origin_table
+
+
+def test_atomic_rename_failure_preserves_existing_recording(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    capture = _write_capture(
+        tmp_path,
+        [("RealtimePointcloud", _point_payload(1.0), 0)],
+    )
+    output = tmp_path / "session.rrd"
+    trusted = b"existing-good-recording"
+    output.write_bytes(trusted)
+
+    def fail_replace(_source: Path, _destination: Path) -> None:
+        raise OSError("synthetic rename failure")
+
+    monkeypatch.setattr(export_module.os, "replace", fail_replace)
+
+    with pytest.raises(RrdExportError, match="synthetic rename failure"):
+        export_k1mqtt_to_rrd(capture, output)
+
+    assert output.read_bytes() == trusted
+    assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
+
+
+def test_missing_monotonic_metadata_never_replaces_existing_recording(tmp_path: Path) -> None:
+    capture = _write_capture(
+        tmp_path,
+        [("RealtimePointcloud", _point_payload(1.0), 0)],
+    )
+    (tmp_path / "mqtt.metadata.jsonl").unlink()
+    output = tmp_path / "session.rrd"
+    output.write_bytes(b"existing-good-recording")
+
+    with pytest.raises(RrdExportError, match="received_monotonic_ns"):
+        export_k1mqtt_to_rrd(capture, output)
+
+    assert output.read_bytes() == b"existing-good-recording"
+    assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
+
+
+def test_export_preserves_valid_prefix_before_crash_metadata_tail(tmp_path: Path) -> None:
+    capture = _write_capture(
+        tmp_path,
+        [
+            ("RealtimePointcloud", _point_payload(1.0), 100_000_000),
+            ("RealtimePointcloud", _point_payload(2.0), 200_000_000),
+        ],
+    )
+    metadata_path = tmp_path / "mqtt.metadata.jsonl"
+    first_line = metadata_path.read_text(encoding="utf-8").splitlines(keepends=True)[0]
+    metadata_path.write_text(
+        first_line + '{"record_type":"message"',
+        encoding="utf-8",
+    )
+    output = tmp_path / "session.rrd"
+
+    summary = export_k1mqtt_to_rrd(capture, output)
+
+    assert summary["source_messages"] == 1
+    assert summary["decoded_messages"] == 1
+    assert summary["point_frames"] == 1
+    assert summary["timeline_end_ns"] == 0
+    assert output.stat().st_size > 0
+
+
+def test_export_honors_cooperative_cancellation_without_publishing(tmp_path: Path) -> None:
+    capture = _write_capture(
+        tmp_path,
+        [("RealtimePointcloud", _point_payload(1.0), 0)],
+    )
+    output = tmp_path / "session.rrd"
+    output.write_bytes(b"trusted-existing-recording")
+    cancelled = threading.Event()
+    cancelled.set()
+
+    with pytest.raises(RrdExportCancelled):
+        export_k1mqtt_to_rrd(capture, output, cancel_event=cancelled)
+
+    assert output.read_bytes() == b"trusted-existing-recording"
+    assert list(tmp_path.glob(".session.rrd.*.tmp")) == []