483 lines
17 KiB
Python
483 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
import k1link.device_plugins.xgrids_k1.rrd_export as export_module
|
|
from k1link.device_plugins.xgrids_k1.mqtt.capture import (
|
|
CAPTURE_CLOCK_FILENAME,
|
|
FRAME_HEADER,
|
|
RAW_MAGIC,
|
|
)
|
|
from k1link.device_plugins.xgrids_k1.rrd_export import (
|
|
RECORDED_CAMERA_VIEW_ID,
|
|
RECORDED_METRICS_VIEW_ID,
|
|
RECORDED_POINTS_VISUALIZER_ID,
|
|
RECORDED_ROOT_CONTAINER_ID,
|
|
RECORDED_SPATIAL_VIEW_ID,
|
|
SESSION_TIMELINE,
|
|
RrdExportCancelled,
|
|
RrdExportError,
|
|
_recorded_blueprint,
|
|
export_k1mqtt_to_rrd,
|
|
recorded_blueprint_rrd,
|
|
)
|
|
from k1link.viewer.rerun_bridge import RerunSceneSettings
|
|
|
|
|
|
def _point_payload(x: float) -> bytes:
|
|
return struct.pack("<III", 16, 0, 0) + struct.pack(
|
|
"<fffBBBB",
|
|
x,
|
|
-2.0,
|
|
3.0,
|
|
10,
|
|
20,
|
|
30,
|
|
40,
|
|
)
|
|
|
|
|
|
def _pose_payload(x: float) -> bytes:
|
|
return struct.pack("<ffffffff", x, 2.0, 3.0, 99.0, 1.0, 0.0, 0.0, 0.0)
|
|
|
|
|
|
def _varint(value: int) -> bytes:
|
|
encoded = bytearray()
|
|
while value > 0x7F:
|
|
encoded.append((value & 0x7F) | 0x80)
|
|
value >>= 7
|
|
encoded.append(value)
|
|
return bytes(encoded)
|
|
|
|
|
|
def _proto_key(number: int, wire_type: int) -> bytes:
|
|
return _varint((number << 3) | wire_type)
|
|
|
|
|
|
def _proto_uint(number: int, value: int) -> bytes:
|
|
return _proto_key(number, 0) + _varint(value)
|
|
|
|
|
|
def _proto_bytes(number: int, value: bytes) -> bytes:
|
|
return _proto_key(number, 2) + _varint(len(value)) + value
|
|
|
|
|
|
def _proto_float32(number: int, value: float) -> bytes:
|
|
return _proto_key(number, 5) + struct.pack("<f", value)
|
|
|
|
|
|
def _modeling_payload(*, distance: float, speed: float, scan_ticks: int) -> bytes:
|
|
status = _proto_float32(1, distance) + _proto_float32(2, speed) + _proto_uint(3, scan_ticks)
|
|
return _proto_uint(2, 0) + _proto_bytes(3, status)
|
|
|
|
|
|
def _write_capture(
|
|
directory: Path,
|
|
frames: list[tuple[str, bytes, int]],
|
|
*,
|
|
capture_clock_offsets_ns: tuple[int, int] | None = None,
|
|
) -> Path:
|
|
capture = directory / "mqtt.raw.k1mqtt"
|
|
raw = bytearray(RAW_MAGIC)
|
|
metadata: list[dict[str, object]] = []
|
|
epoch_origin_ns = 1_784_124_315_000_000_000
|
|
monotonic_origin_ns = 9_000_000_000
|
|
for sequence, (topic, payload, session_time_ns) in enumerate(frames, start=1):
|
|
topic_raw = topic.encode("utf-8")
|
|
raw.extend(FRAME_HEADER.pack(len(topic_raw), len(payload)))
|
|
raw.extend(topic_raw)
|
|
raw.extend(payload)
|
|
metadata.append(
|
|
{
|
|
"record_type": "message",
|
|
"sequence": sequence,
|
|
"received_at_epoch_ns": epoch_origin_ns + session_time_ns,
|
|
"received_monotonic_ns": monotonic_origin_ns + session_time_ns,
|
|
}
|
|
)
|
|
capture.write_bytes(raw)
|
|
(directory / "mqtt.metadata.jsonl").write_text(
|
|
"".join(json.dumps(item) + "\n" for item in metadata),
|
|
encoding="utf-8",
|
|
)
|
|
if capture_clock_offsets_ns is not None:
|
|
started_offset_ns, completed_offset_ns = capture_clock_offsets_ns
|
|
(directory / "mqtt.timeline.origin.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": 1,
|
|
"started_at_epoch_ns": epoch_origin_ns + started_offset_ns,
|
|
"started_monotonic_ns": monotonic_origin_ns + started_offset_ns,
|
|
},
|
|
separators=(",", ":"),
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
(directory / CAPTURE_CLOCK_FILENAME).write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": 1,
|
|
"started_at_epoch_ns": epoch_origin_ns + started_offset_ns,
|
|
"started_monotonic_ns": monotonic_origin_ns + started_offset_ns,
|
|
"completed_at_epoch_ns": epoch_origin_ns + completed_offset_ns,
|
|
"completed_monotonic_ns": monotonic_origin_ns + completed_offset_ns,
|
|
},
|
|
separators=(",", ":"),
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
return capture
|
|
|
|
|
|
def _sha256(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def _print_rrd_entity(path: Path, entity: str) -> str:
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"rerun_cli",
|
|
"rrd",
|
|
"print",
|
|
"-vvv",
|
|
"--entity",
|
|
entity,
|
|
str(path),
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return completed.stdout
|
|
|
|
|
|
def test_recorded_blueprint_accumulates_point_frames_on_session_timeline() -> None:
|
|
blueprint = _recorded_blueprint(
|
|
RerunSceneSettings(
|
|
point_size=7.5,
|
|
palette="custom",
|
|
custom_color="#112233",
|
|
accumulation_seconds=18.5,
|
|
show_points=False,
|
|
show_trajectory=True,
|
|
show_grid=False,
|
|
)
|
|
)
|
|
spatial_view = blueprint.root_container.contents[0]
|
|
visible_ranges = spatial_view.properties["VisibleTimeRanges"]
|
|
line_grid = spatial_view.properties["LineGrid3D"]
|
|
|
|
assert visible_ranges.ranges.as_arrow_array().to_pylist() == [
|
|
{
|
|
"timeline": SESSION_TIMELINE,
|
|
"range": {
|
|
"start": -18_500_000_000,
|
|
"end": 0,
|
|
},
|
|
}
|
|
]
|
|
assert line_grid.visible.as_arrow_array().to_pylist() == [False]
|
|
point_behavior, point_visualizer = spatial_view.visualizer_overrides["/world/points"]
|
|
trajectory_behavior = spatial_view.visualizer_overrides["/world/trajectory"]
|
|
assert point_behavior.visible.as_arrow_array().to_pylist() == [False]
|
|
assert trajectory_behavior.visible.as_arrow_array().to_pylist() == [True]
|
|
point_overrides = {
|
|
str(batch.component_descriptor()): batch.as_arrow_array().to_pylist()
|
|
for batch in point_visualizer.overrides
|
|
}
|
|
assert point_overrides == {
|
|
"Points3D:colors": [0x112233FF],
|
|
# Negative values are Rerun's encoding for screen-space UI points.
|
|
"Points3D:radii": [-7.5],
|
|
}
|
|
|
|
|
|
def test_zero_accumulation_uses_latest_frame_instead_of_empty_time_range() -> None:
|
|
blueprint = _recorded_blueprint(RerunSceneSettings(accumulation_seconds=0.0, show_grid=True))
|
|
spatial_view = blueprint.root_container.contents[0]
|
|
|
|
assert "VisibleTimeRanges" not in spatial_view.properties
|
|
point_behavior, point_visualizer = spatial_view.visualizer_overrides["/world/points"]
|
|
trajectory_behavior = spatial_view.visualizer_overrides["/world/trajectory"]
|
|
assert point_behavior.visible.as_arrow_array().to_pylist() == [True]
|
|
assert trajectory_behavior.visible.as_arrow_array().to_pylist() == [True]
|
|
point_components = {str(batch.component_descriptor()) for batch in point_visualizer.overrides}
|
|
assert point_components == {"Points3D:radii"}
|
|
|
|
|
|
def test_dynamic_blueprint_reuses_scene_ids_without_playback_mutation() -> None:
|
|
first = _recorded_blueprint(
|
|
RerunSceneSettings(show_points=False, show_trajectory=True),
|
|
include_initial_playback_state=False,
|
|
)
|
|
second = _recorded_blueprint(
|
|
RerunSceneSettings(show_points=True, show_trajectory=False),
|
|
include_initial_playback_state=False,
|
|
)
|
|
|
|
assert not hasattr(first, "time_panel")
|
|
assert not hasattr(second, "time_panel")
|
|
assert first.root_container.id == second.root_container.id == RECORDED_ROOT_CONTAINER_ID
|
|
first_view = first.root_container.contents[0]
|
|
second_view = second.root_container.contents[0]
|
|
assert first_view.id == second_view.id == RECORDED_SPATIAL_VIEW_ID
|
|
assert first.root_container.contents[1].id == RECORDED_CAMERA_VIEW_ID
|
|
assert second.root_container.contents[1].id == RECORDED_CAMERA_VIEW_ID
|
|
assert first.root_container.contents[2].id == RECORDED_METRICS_VIEW_ID
|
|
assert second.root_container.contents[2].id == RECORDED_METRICS_VIEW_ID
|
|
|
|
first_point_behavior, first_point_visualizer = first_view.visualizer_overrides["/world/points"]
|
|
second_point_behavior, second_point_visualizer = second_view.visualizer_overrides[
|
|
"/world/points"
|
|
]
|
|
first_trajectory = first_view.visualizer_overrides["/world/trajectory"]
|
|
second_trajectory = second_view.visualizer_overrides["/world/trajectory"]
|
|
assert first_point_visualizer.id == RECORDED_POINTS_VISUALIZER_ID
|
|
assert second_point_visualizer.id == RECORDED_POINTS_VISUALIZER_ID
|
|
assert first_point_behavior.visible.as_arrow_array().to_pylist() == [False]
|
|
assert second_point_behavior.visible.as_arrow_array().to_pylist() == [True]
|
|
assert first_trajectory.visible.as_arrow_array().to_pylist() == [True]
|
|
assert second_trajectory.visible.as_arrow_array().to_pylist() == [False]
|
|
|
|
payload = recorded_blueprint_rrd(
|
|
RerunSceneSettings(show_points=False, show_trajectory=False),
|
|
recording_id="stable-recording",
|
|
)
|
|
assert b"PlayState" not in payload
|
|
assert b"play_state" not in payload
|
|
assert str(RECORDED_ROOT_CONTAINER_ID).encode() in payload
|
|
assert str(RECORDED_SPATIAL_VIEW_ID).encode() in payload
|
|
assert str(RECORDED_POINTS_VISUALIZER_ID).encode() in payload
|
|
|
|
|
|
def test_export_preserves_every_decodable_frame_and_source_timeline(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[
|
|
("lixel/application/report/heartbeat", b"opaque", 0),
|
|
("RealtimePointcloud", _point_payload(1.0), 100_000_000),
|
|
("RealtimePath", _pose_payload(1.0), 600_000_000),
|
|
("RealtimePointcloud", _point_payload(2.0), 900_000_000),
|
|
("RealtimePath", _pose_payload(1.2), 1_200_000_000),
|
|
("lixel/application/report/heartbeat", b"opaque-tail", 9_500_000_000),
|
|
],
|
|
capture_clock_offsets_ns=(-500_000_000, 10_000_000_000),
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
|
|
summary = export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert summary["source_messages"] == 6
|
|
assert summary["decoded_messages"] == 4
|
|
assert summary["point_frames"] == 2
|
|
assert summary["pose_frames"] == 2
|
|
assert summary["ignored_messages"] == 2
|
|
assert summary["points"] == 2
|
|
assert summary["trajectory_poses"] == 2
|
|
assert summary["trajectory_updates"] == 2
|
|
assert summary["session_origin_monotonic_ns"] == 8_500_000_000
|
|
assert summary["timeline"] == "session_time"
|
|
assert summary["timeline_start_ns"] == 0
|
|
assert summary["timeline_end_ns"] == 10_500_000_000
|
|
assert summary["timeline_span_ns"] == 10_500_000_000
|
|
assert summary["first_decoded_time_ns"] == 600_000_000
|
|
assert summary["last_decoded_time_ns"] == 1_700_000_000
|
|
assert summary["source_sha256"] == _sha256(capture)
|
|
assert summary["rrd_bytes"] == output.stat().st_size
|
|
assert summary["rrd_sha256"] == _sha256(output)
|
|
assert output.stat().st_size > 0
|
|
assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
|
|
|
|
origin_table = _print_rrd_entity(output, "/__mission_core/session_origin")
|
|
assert "/__mission_core/session_origin" in origin_table
|
|
assert "session_time" in origin_table
|
|
assert "P0D" in origin_table
|
|
assert "[true]" in origin_table
|
|
end_table = _print_rrd_entity(output, "/__mission_core/session_end")
|
|
assert "/__mission_core/session_end" in end_table
|
|
assert "PT10.5S" in end_table
|
|
|
|
|
|
def test_export_records_device_route_time_series_on_shared_timeline(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[
|
|
("RealtimePointcloud", _point_payload(1.0), 0),
|
|
(
|
|
"lixel/application/report/modeling",
|
|
_modeling_payload(distance=2.25, speed=0.75, scan_ticks=20),
|
|
500_000_000,
|
|
),
|
|
(
|
|
"lixel/application/report/modeling",
|
|
_modeling_payload(distance=3.5, speed=1.25, scan_ticks=22),
|
|
1_000_000_000,
|
|
),
|
|
],
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
|
|
summary = export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert summary["modeling_reports"] == 2
|
|
assert summary["decoded_messages"] == 1
|
|
assert summary["ignored_messages"] == 0
|
|
assert summary["timeline_end_ns"] == 1_000_000_000
|
|
distance_table = _print_rrd_entity(output, "/metrics/device/route_distance_meters")
|
|
elapsed_table = _print_rrd_entity(output, "/metrics/device/scan_elapsed_seconds")
|
|
assert "2.25" in distance_table and "3.5" in distance_table
|
|
assert "10.0" in elapsed_table and "11.0" in elapsed_table
|
|
|
|
|
|
def test_export_uses_explicit_catalog_bound_sealed_clock(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[("RealtimePointcloud", _point_payload(1.0), 500_000_000)],
|
|
)
|
|
origin = {
|
|
"schema_version": 1,
|
|
"started_at_epoch_ns": 1_784_124_314_000_000_000,
|
|
"started_monotonic_ns": 8_000_000_000,
|
|
}
|
|
origin_path = tmp_path / "mqtt.timeline.origin.json"
|
|
origin_path.write_text(json.dumps(origin, separators=(",", ":")) + "\n")
|
|
provisional = {
|
|
**origin,
|
|
"completed_at_epoch_ns": 1_784_124_316_000_000_000,
|
|
"completed_monotonic_ns": 10_000_000_000,
|
|
}
|
|
(tmp_path / "mqtt.timeline.json").write_text(
|
|
json.dumps(provisional, separators=(",", ":")) + "\n"
|
|
)
|
|
sealed = {
|
|
**origin,
|
|
"completed_at_epoch_ns": 1_784_124_321_000_000_000,
|
|
"completed_monotonic_ns": 15_000_000_000,
|
|
}
|
|
sealed_payload = (json.dumps(sealed, separators=(",", ":")) + "\n").encode()
|
|
sealed_digest = hashlib.sha256(sealed_payload).hexdigest()
|
|
sealed_path = tmp_path / f"mqtt.timeline.session-{sealed_digest}.json"
|
|
sealed_path.write_bytes(sealed_payload)
|
|
output = tmp_path / "session.rrd"
|
|
|
|
summary = export_k1mqtt_to_rrd(
|
|
capture,
|
|
output,
|
|
capture_clock_path=sealed_path,
|
|
capture_clock_origin_path=origin_path,
|
|
)
|
|
|
|
assert summary["session_origin_monotonic_ns"] == 8_000_000_000
|
|
assert summary["timeline_end_ns"] == 7_000_000_000
|
|
|
|
|
|
def test_export_rejects_malformed_known_modeling_report(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[
|
|
("RealtimePointcloud", _point_payload(1.0), 0),
|
|
("lixel/application/report/modeling", b"malformed", 1_000_000),
|
|
],
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
output.write_bytes(b"trusted-existing-recording")
|
|
|
|
with pytest.raises(RrdExportError, match="modeling report"):
|
|
export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert output.read_bytes() == b"trusted-existing-recording"
|
|
|
|
|
|
def test_atomic_rename_failure_preserves_existing_recording(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[("RealtimePointcloud", _point_payload(1.0), 0)],
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
trusted = b"existing-good-recording"
|
|
output.write_bytes(trusted)
|
|
|
|
def fail_replace(_source: Path, _destination: Path) -> None:
|
|
raise OSError("synthetic rename failure")
|
|
|
|
monkeypatch.setattr(export_module.os, "replace", fail_replace)
|
|
|
|
with pytest.raises(RrdExportError, match="synthetic rename failure"):
|
|
export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert output.read_bytes() == trusted
|
|
assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
|
|
|
|
|
|
def test_missing_monotonic_metadata_never_replaces_existing_recording(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[("RealtimePointcloud", _point_payload(1.0), 0)],
|
|
)
|
|
(tmp_path / "mqtt.metadata.jsonl").unlink()
|
|
output = tmp_path / "session.rrd"
|
|
output.write_bytes(b"existing-good-recording")
|
|
|
|
with pytest.raises(RrdExportError, match="received_monotonic_ns"):
|
|
export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert output.read_bytes() == b"existing-good-recording"
|
|
assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
|
|
|
|
|
|
def test_export_preserves_valid_prefix_before_crash_metadata_tail(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[
|
|
("RealtimePointcloud", _point_payload(1.0), 100_000_000),
|
|
("RealtimePointcloud", _point_payload(2.0), 200_000_000),
|
|
],
|
|
)
|
|
metadata_path = tmp_path / "mqtt.metadata.jsonl"
|
|
first_line = metadata_path.read_text(encoding="utf-8").splitlines(keepends=True)[0]
|
|
metadata_path.write_text(
|
|
first_line + '{"record_type":"message"',
|
|
encoding="utf-8",
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
|
|
summary = export_k1mqtt_to_rrd(capture, output)
|
|
|
|
assert summary["source_messages"] == 1
|
|
assert summary["decoded_messages"] == 1
|
|
assert summary["point_frames"] == 1
|
|
assert summary["timeline_end_ns"] == 0
|
|
assert output.stat().st_size > 0
|
|
|
|
|
|
def test_export_honors_cooperative_cancellation_without_publishing(tmp_path: Path) -> None:
|
|
capture = _write_capture(
|
|
tmp_path,
|
|
[("RealtimePointcloud", _point_payload(1.0), 0)],
|
|
)
|
|
output = tmp_path / "session.rrd"
|
|
output.write_bytes(b"trusted-existing-recording")
|
|
cancelled = threading.Event()
|
|
cancelled.set()
|
|
|
|
with pytest.raises(RrdExportCancelled):
|
|
export_k1mqtt_to_rrd(capture, output, cancel_event=cancelled)
|
|
|
|
assert output.read_bytes() == b"trusted-existing-recording"
|
|
assert list(tmp_path.glob(".session.rrd.*.tmp")) == []
|