325 lines
11 KiB
Python
325 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
import k1link.device_plugins.xgrids_k1.rrd_export as export_module
|
|
from k1link.device_plugins.xgrids_k1.mqtt.capture import FRAME_HEADER, RAW_MAGIC
|
|
from k1link.device_plugins.xgrids_k1.rrd_export import (
|
|
RECORDED_POINTS_VISUALIZER_ID,
|
|
RECORDED_ROOT_CONTAINER_ID,
|
|
RECORDED_SPATIAL_VIEW_ID,
|
|
SESSION_TIMELINE,
|
|
RrdExportCancelled,
|
|
RrdExportError,
|
|
_recorded_blueprint,
|
|
export_k1mqtt_to_rrd,
|
|
recorded_blueprint_rrd,
|
|
)
|
|
from k1link.viewer.rerun_bridge import RerunSceneSettings
|
|
|
|
|
|
def _point_payload(x: float) -> bytes:
|
|
return struct.pack("<III", 16, 0, 0) + struct.pack(
|
|
"<fffBBBB",
|
|
x,
|
|
-2.0,
|
|
3.0,
|
|
10,
|
|
20,
|
|
30,
|
|
40,
|
|
)
|
|
|
|
|
|
def _pose_payload(x: float) -> bytes:
|
|
return struct.pack("<ffffffff", x, 2.0, 3.0, 99.0, 1.0, 0.0, 0.0, 0.0)
|
|
|
|
|
|
def _write_capture(
|
|
directory: Path,
|
|
frames: list[tuple[str, bytes, int]],
|
|
) -> Path:
|
|
capture = directory / "mqtt.raw.k1mqtt"
|
|
raw = bytearray(RAW_MAGIC)
|
|
metadata: list[dict[str, object]] = []
|
|
epoch_origin_ns = 1_784_124_315_000_000_000
|
|
monotonic_origin_ns = 9_000_000_000
|
|
for sequence, (topic, payload, session_time_ns) in enumerate(frames, start=1):
|
|
topic_raw = topic.encode("utf-8")
|
|
raw.extend(FRAME_HEADER.pack(len(topic_raw), len(payload)))
|
|
raw.extend(topic_raw)
|
|
raw.extend(payload)
|
|
metadata.append(
|
|
{
|
|
"record_type": "message",
|
|
"sequence": sequence,
|
|
"received_at_epoch_ns": epoch_origin_ns + session_time_ns,
|
|
"received_monotonic_ns": monotonic_origin_ns + session_time_ns,
|
|
}
|
|
)
|
|
capture.write_bytes(raw)
|
|
(directory / "mqtt.metadata.jsonl").write_text(
|
|
"".join(json.dumps(item) + "\n" for item in metadata),
|
|
encoding="utf-8",
|
|
)
|
|
return capture
|
|
|
|
|
|
def _sha256(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def _print_rrd_entity(path: Path, entity: str) -> str:
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"rerun_cli",
|
|
"rrd",
|
|
"print",
|
|
"-vvv",
|
|
"--entity",
|
|
entity,
|
|
str(path),
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return completed.stdout
|
|
|
|
|
|
def test_recorded_blueprint_accumulates_point_frames_on_session_timeline() -> None:
|
|
blueprint = _recorded_blueprint(
|
|
RerunSceneSettings(
|
|
point_size=7.5,
|
|
palette="custom",
|
|
custom_color="#112233",
|
|
accumulation_seconds=18.5,
|
|
show_points=False,
|
|
show_trajectory=True,
|
|
show_grid=False,
|
|
)
|
|
)
|
|
spatial_view = blueprint.root_container.contents[0]
|
|
visible_ranges = spatial_view.properties["VisibleTimeRanges"]
|
|
line_grid = spatial_view.properties["LineGrid3D"]
|
|
|
|
assert visible_ranges.ranges.as_arrow_array().to_pylist() == [
|
|
{
|
|
"timeline": SESSION_TIMELINE,
|
|
"range": {
|
|
"start": -18_500_000_000,
|
|
"end": 0,
|
|
},
|
|
}
|
|
]
|
|
assert line_grid.visible.as_arrow_array().to_pylist() == [False]
|
|
point_behavior, point_visualizer = spatial_view.visualizer_overrides["/world/points"]
|
|
trajectory_behavior = spatial_view.visualizer_overrides["/world/trajectory"]
|
|
assert point_behavior.visible.as_arrow_array().to_pylist() == [False]
|
|
assert trajectory_behavior.visible.as_arrow_array().to_pylist() == [True]
|
|
point_overrides = {
|
|
str(batch.component_descriptor()): batch.as_arrow_array().to_pylist()
|
|
for batch in point_visualizer.overrides
|
|
}
|
|
assert point_overrides == {
|
|
"Points3D:colors": [0x112233FF],
|
|
# Negative values are Rerun's encoding for screen-space UI points.
|
|
"Points3D:radii": [-7.5],
|
|
}
|
|
|
|
|
|
def test_zero_accumulation_uses_latest_frame_instead_of_empty_time_range() -> None:
|
|
blueprint = _recorded_blueprint(
|
|
RerunSceneSettings(accumulation_seconds=0.0, show_grid=True)
|
|
)
|
|
spatial_view = blueprint.root_container.contents[0]
|
|
|
|
assert "VisibleTimeRanges" not in spatial_view.properties
|
|
point_behavior, point_visualizer = spatial_view.visualizer_overrides["/world/points"]
|
|
trajectory_behavior = spatial_view.visualizer_overrides["/world/trajectory"]
|
|
assert point_behavior.visible.as_arrow_array().to_pylist() == [True]
|
|
assert trajectory_behavior.visible.as_arrow_array().to_pylist() == [True]
|
|
point_components = {
|
|
str(batch.component_descriptor()) for batch in point_visualizer.overrides
|
|
}
|
|
assert point_components == {"Points3D:radii"}
|
|
|
|
|
|
def test_dynamic_blueprint_reuses_scene_ids_without_playback_mutation() -> None:
|
|
first = _recorded_blueprint(
|
|
RerunSceneSettings(show_points=False, show_trajectory=True),
|
|
include_initial_playback_state=False,
|
|
)
|
|
second = _recorded_blueprint(
|
|
RerunSceneSettings(show_points=True, show_trajectory=False),
|
|
include_initial_playback_state=False,
|
|
)
|
|
|
|
assert not hasattr(first, "time_panel")
|
|
assert not hasattr(second, "time_panel")
|
|
assert first.root_container.id == second.root_container.id == RECORDED_ROOT_CONTAINER_ID
|
|
first_view = first.root_container.contents[0]
|
|
second_view = second.root_container.contents[0]
|
|
assert first_view.id == second_view.id == RECORDED_SPATIAL_VIEW_ID
|
|
|
|
first_point_behavior, first_point_visualizer = first_view.visualizer_overrides[
|
|
"/world/points"
|
|
]
|
|
second_point_behavior, second_point_visualizer = second_view.visualizer_overrides[
|
|
"/world/points"
|
|
]
|
|
first_trajectory = first_view.visualizer_overrides["/world/trajectory"]
|
|
second_trajectory = second_view.visualizer_overrides["/world/trajectory"]
|
|
assert first_point_visualizer.id == RECORDED_POINTS_VISUALIZER_ID
|
|
assert second_point_visualizer.id == RECORDED_POINTS_VISUALIZER_ID
|
|
assert first_point_behavior.visible.as_arrow_array().to_pylist() == [False]
|
|
assert second_point_behavior.visible.as_arrow_array().to_pylist() == [True]
|
|
assert first_trajectory.visible.as_arrow_array().to_pylist() == [True]
|
|
assert second_trajectory.visible.as_arrow_array().to_pylist() == [False]
|
|
|
|
payload = recorded_blueprint_rrd(
|
|
RerunSceneSettings(show_points=False, show_trajectory=False),
|
|
recording_id="stable-recording",
|
|
)
|
|
assert b"PlayState" not in payload
|
|
assert b"play_state" not in payload
|
|
assert str(RECORDED_ROOT_CONTAINER_ID).encode() in payload
|
|
assert str(RECORDED_SPATIAL_VIEW_ID).encode() in payload
|
|
assert str(RECORDED_POINTS_VISUALIZER_ID).encode() in payload
|
|
|
|
|
|
def test_export_preserves_every_decodable_frame_and_source_timeline(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[
|
|
("lixel/application/report/heartbeat", b"opaque", 0),
|
|
("RealtimePointcloud", _point_payload(1.0), 100_000_000),
|
|
("RealtimePath", _pose_payload(1.0), 600_000_000),
|
|
("RealtimePointcloud", _point_payload(2.0), 900_000_000),
|
|
("RealtimePath", _pose_payload(1.2), 1_200_000_000),
|
|
("lixel/application/report/heartbeat", b"opaque-tail", 9_500_000_000),
|
|
],
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
|
|
summary = export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert summary["source_messages"] == 6
|
|
assert summary["decoded_messages"] == 4
|
|
assert summary["point_frames"] == 2
|
|
assert summary["pose_frames"] == 2
|
|
assert summary["ignored_messages"] == 2
|
|
assert summary["points"] == 2
|
|
assert summary["trajectory_poses"] == 2
|
|
assert summary["trajectory_updates"] == 2
|
|
assert summary["session_origin_monotonic_ns"] == 9_000_000_000
|
|
assert summary["timeline"] == "session_time"
|
|
assert summary["timeline_start_ns"] == 0
|
|
assert summary["timeline_end_ns"] == 1_200_000_000
|
|
assert summary["timeline_span_ns"] == 1_200_000_000
|
|
assert summary["first_decoded_time_ns"] == 100_000_000
|
|
assert summary["last_decoded_time_ns"] == 1_200_000_000
|
|
assert summary["source_sha256"] == _sha256(capture)
|
|
assert summary["rrd_bytes"] == output.stat().st_size
|
|
assert summary["rrd_sha256"] == _sha256(output)
|
|
assert output.stat().st_size > 0
|
|
assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
|
|
|
|
origin_table = _print_rrd_entity(output, "/__mission_core/session_origin")
|
|
assert "/__mission_core/session_origin" in origin_table
|
|
assert "session_time" in origin_table
|
|
assert "P0D" in origin_table
|
|
assert "[true]" in origin_table
|
|
|
|
|
|
def test_atomic_rename_failure_preserves_existing_recording(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[("RealtimePointcloud", _point_payload(1.0), 0)],
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
trusted = b"existing-good-recording"
|
|
output.write_bytes(trusted)
|
|
|
|
def fail_replace(_source: Path, _destination: Path) -> None:
|
|
raise OSError("synthetic rename failure")
|
|
|
|
monkeypatch.setattr(export_module.os, "replace", fail_replace)
|
|
|
|
with pytest.raises(RrdExportError, match="synthetic rename failure"):
|
|
export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert output.read_bytes() == trusted
|
|
assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
|
|
|
|
|
|
def test_missing_monotonic_metadata_never_replaces_existing_recording(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[("RealtimePointcloud", _point_payload(1.0), 0)],
|
|
)
|
|
(tmp_path / "mqtt.metadata.jsonl").unlink()
|
|
output = tmp_path / "session.rrd"
|
|
output.write_bytes(b"existing-good-recording")
|
|
|
|
with pytest.raises(RrdExportError, match="received_monotonic_ns"):
|
|
export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert output.read_bytes() == b"existing-good-recording"
|
|
assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
|
|
|
|
|
|
def test_export_preserves_valid_prefix_before_crash_metadata_tail(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[
|
|
("RealtimePointcloud", _point_payload(1.0), 100_000_000),
|
|
("RealtimePointcloud", _point_payload(2.0), 200_000_000),
|
|
],
|
|
)
|
|
metadata_path = tmp_path / "mqtt.metadata.jsonl"
|
|
first_line = metadata_path.read_text(encoding="utf-8").splitlines(keepends=True)[0]
|
|
metadata_path.write_text(
|
|
first_line + '{"record_type":"message"',
|
|
encoding="utf-8",
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
|
|
summary = export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert summary["source_messages"] == 1
|
|
assert summary["decoded_messages"] == 1
|
|
assert summary["point_frames"] == 1
|
|
assert summary["timeline_end_ns"] == 0
|
|
assert output.stat().st_size > 0
|
|
|
|
|
|
def test_export_honors_cooperative_cancellation_without_publishing(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[("RealtimePointcloud", _point_payload(1.0), 0)],
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
output.write_bytes(b"trusted-existing-recording")
|
|
cancelled = threading.Event()
|
|
cancelled.set()
|
|
|
|
with pytest.raises(RrdExportCancelled):
|
|
export_k1mqtt_to_rrd(capture, output, cancel_event=cancelled)
|
|
|
|
assert output.read_bytes() == b"trusted-existing-recording"
|
|
assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
|