NODEDC_MISSION_CORE/tests/test_session_store.py

574 lines
22 KiB
Python

from __future__ import annotations
import hashlib
import json
import shutil
import sqlite3
from pathlib import Path
import pytest
from k1link.mqtt.capture import FRAME_HEADER, RAW_MAGIC, iter_capture_frames
from k1link.sessions import (
LayoutConflictError,
SessionIntegrityError,
SessionNotFoundError,
SessionStore,
resolve_missioncore_evidence_dir,
)
def make_legacy_session(
sessions_root: Path,
session_id: str,
*,
created_at: str = "2026-07-16T20:56:32.699Z",
) -> Path:
session = sessions_root / session_id
capture = session / "captures" / "mqtt_live"
capture.mkdir(parents=True)
frames = [
("lixel/application/report/lio_pcl", b"point-frame"),
("lixel/application/report/lio_pose", b"pose-frame"),
]
raw = bytearray(RAW_MAGIC)
metadata: list[dict[str, object]] = []
for sequence, (topic, payload) in enumerate(frames, start=1):
topic_bytes = topic.encode()
frame_offset = len(raw)
raw.extend(FRAME_HEADER.pack(len(topic_bytes), len(payload)))
raw.extend(topic_bytes)
raw.extend(payload)
metadata.append(
{
"schema_version": 1,
"record_type": "message",
"sequence": sequence,
"received_at_utc": f"2026-07-16T20:56:{31 + sequence:02d}.699Z",
"received_at_epoch_ns": 1_784_235_391_699_000_000 + sequence * 1_000_000_000,
"received_monotonic_ns": 9_000_000_000 + sequence * 1_000_000_000,
"topic": topic,
"payload_bytes": len(payload),
"raw_frame_offset": frame_offset,
"raw_payload_offset": frame_offset + FRAME_HEADER.size + len(topic_bytes),
"raw_frame_bytes": FRAME_HEADER.size + len(topic_bytes) + len(payload),
}
)
raw_path = capture / "mqtt.raw.k1mqtt"
raw_path.write_bytes(raw)
raw_hash = hashlib.sha256(raw).hexdigest()
metadata_path = capture / "mqtt.metadata.jsonl"
metadata_path.write_text(
"".join(json.dumps(record, separators=(",", ":")) + "\n" for record in metadata),
encoding="utf-8",
)
metadata_hash = hashlib.sha256(metadata_path.read_bytes()).hexdigest()
(capture / "mqtt.summary.json").write_text(
json.dumps(
{
"created_at_utc": created_at,
"completed_at_utc": "2026-07-16T21:20:43.018Z",
"capture_elapsed_seconds": 1440.1,
"stop_reason": "external_stop",
"error": None,
"message_count": 2,
"raw_bytes": len(raw),
"payload_bytes": sum(len(payload) for _, payload in frames),
"topic_counts": {
"lixel/application/report/lio_pcl": 1,
"lixel/application/report/lio_pose": 1,
},
"artifact_hashes": {
"raw_sha256": raw_hash,
"metadata_jsonl_sha256": metadata_hash,
},
# This is deliberately sensitive and must not enter API DTOs.
"target_ipv4": "192.168.99.77",
}
),
encoding="utf-8",
)
(session / "manifest.redacted.json").write_text(
json.dumps({"started_at_utc": created_at, "target": "redacted"}),
encoding="utf-8",
)
return session
def test_evidence_root_is_private_and_configurable(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
repository = tmp_path / "repo"
assert resolve_missioncore_evidence_dir(repository) == (
repository / ".runtime" / "mission-core" / "evidence" / "sessions"
).resolve()
configured = tmp_path / "external-evidence"
monkeypatch.setenv("MISSIONCORE_EVIDENCE_DIR", str(configured))
assert resolve_missioncore_evidence_dir(repository) == configured.resolve()
def test_catalog_reconciles_sessions_removed_from_one_evidence_root(tmp_path: Path) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
store = SessionStore(repository, data_dir=tmp_path / "data")
assert store.import_legacy_viewer_live(sessions) == (session.name,)
shutil.rmtree(session)
assert store.import_legacy_viewer_live(sessions) == ()
assert store.list_recent().items == ()
def make_recorded_camera_source(
session: Path,
source_id: str = "sensor.camera.left",
*,
complete: bool = True,
) -> Path:
epoch = session / "media" / source_id / "epoch-1"
segments = epoch / "segments"
segments.mkdir(parents=True)
(epoch / "init.mp4").write_bytes(b"ftyp-mission-core")
(segments / "1.m4s").write_bytes(b"moof-camera-frame")
(epoch / "index.jsonl").write_text(
json.dumps({"sequence": 1, "started_at_seconds": 0.0}) + "\n",
encoding="utf-8",
)
if complete:
(epoch / "summary.json").write_text(
json.dumps(
{
"schema_version": "missioncore.camera-recording/v1",
"source_id": source_id,
"segment_count": 1,
}
),
encoding="utf-8",
)
return epoch
def replace_summary_with_recovery_metadata(
session: Path,
*,
corrupt_trailing_line: bool = False,
) -> None:
capture = session / "captures" / "mqtt_live"
raw_path = capture / "mqtt.raw.k1mqtt"
timestamps = (
("2026-07-16T20:56:32.699Z", 1_784_235_392_699_000_000, 10_000_000_000),
("2026-07-16T20:56:35.199Z", 1_784_235_395_199_000_000, 12_500_000_000),
)
records = []
for frame, (timestamp, epoch_ns, monotonic_ns) in zip(
iter_capture_frames(raw_path),
timestamps,
strict=True,
):
records.append(
{
"schema_version": 1,
"record_type": "message",
"sequence": frame.sequence,
"received_at_utc": timestamp,
"received_at_epoch_ns": epoch_ns,
"received_monotonic_ns": monotonic_ns,
"topic": frame.topic,
"payload_bytes": frame.raw_frame_bytes
- FRAME_HEADER.size
- len(frame.topic.encode("utf-8")),
"raw_frame_offset": frame.raw_frame_offset,
"raw_payload_offset": frame.raw_payload_offset,
"raw_frame_bytes": frame.raw_frame_bytes,
}
)
metadata = b"".join(
(json.dumps(record, separators=(",", ":")) + "\n").encode("utf-8")
for record in records
)
if corrupt_trailing_line:
metadata += b'{"record_type":"message"'
(capture / "mqtt.metadata.jsonl").write_bytes(metadata)
(capture / "mqtt.summary.json").write_text("{corrupt", encoding="utf-8")
def serialized(value: object) -> str:
return json.dumps(value, ensure_ascii=False, default=str)
def test_store_uses_private_wal_database_and_idempotently_imports_legacy_session(
tmp_path: Path,
) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
make_legacy_session(sessions, "20260716T205632Z_viewer_live")
store = SessionStore(repository, data_dir=tmp_path / "data")
first = store.import_legacy_viewer_live(sessions)
second = store.import_legacy_viewer_live(sessions)
assert first == second == ("20260716T205632Z_viewer_live",)
page = store.list_recent()
assert len(page.items) == 1
summary = page.items[0]
assert summary.status == "ready"
assert summary.modalities == ("point-cloud", "trajectory")
assert summary.replayable is True
assert summary.source_count == 2
assert "192.168.99.77" not in serialized(page.as_dict())
assert str(repository) not in serialized(page.as_dict())
detail = store.get_session(summary.session_id)
assert [source.source_id for source in detail.sources] == [
"sensor.lidar.primary",
"spatial.trajectory",
]
assert all(source.seekable for source in detail.sources)
assert str(repository) not in serialized(detail.as_dict())
command = store.prepare_replay(summary.session_id, speed=2.0, loop=True)
assert command.source_path.name == "mqtt.raw.k1mqtt"
assert command.speed == 2.0
assert command.loop is True
with sqlite3.connect(store.database_path) as connection:
assert connection.execute("PRAGMA journal_mode").fetchone()[0] == "wal"
assert store.database_path.stat().st_mode & 0o777 == 0o600
def test_recent_sessions_are_sorted_and_cursor_paginated(tmp_path: Path) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
make_legacy_session(
sessions,
"20260716T191025Z_viewer_live",
created_at="2026-07-16T19:10:25.352Z",
)
make_legacy_session(
sessions,
"20260716T205632Z_viewer_live",
created_at="2026-07-16T20:56:32.699Z",
)
store = SessionStore(repository, data_dir=tmp_path / "data")
store.import_legacy_viewer_live(sessions)
first = store.list_recent(limit=1)
assert first.items[0].session_id == "20260716T205632Z_viewer_live"
assert first.next_cursor == "20260716T205632Z_viewer_live"
second = store.list_recent(limit=1, cursor=first.next_cursor)
assert second.items[0].session_id == "20260716T191025Z_viewer_live"
assert second.next_cursor is None
def test_legacy_import_adds_video_only_for_validated_recording_tree(tmp_path: Path) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
complete = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
incomplete = make_legacy_session(sessions, "20260716T205632Z_viewer_live_2")
make_recorded_camera_source(complete)
make_recorded_camera_source(incomplete, complete=False)
store = SessionStore(repository, data_dir=tmp_path / "data")
store.import_legacy_viewer_live(sessions)
complete_detail = store.get_session(complete.name)
assert complete_detail.summary.modalities == ("point-cloud", "trajectory", "video")
assert complete_detail.summary.source_count == 3
camera = next(
source for source in complete_detail.sources if source.source_id == "sensor.camera.left"
)
assert camera.modality == "video"
assert camera.semantic_channel_id == "camera.video.recorded"
assert camera.seekable is True
video_artifact = next(
artifact
for artifact in complete_detail.artifacts
if artifact.artifact_id == camera.artifact_id
)
assert video_artifact.kind == "recorded-video"
assert video_artifact.integrity_status == "validated-structure"
assert str(repository) not in serialized(complete_detail.as_dict())
incomplete_detail = store.get_session(incomplete.name)
assert incomplete_detail.summary.modalities == ("point-cloud", "trajectory")
assert all(source.modality != "video" for source in incomplete_detail.sources)
def test_interrupted_capture_is_recovered_from_aligned_raw_and_metadata_prefix(
tmp_path: Path,
) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
replace_summary_with_recovery_metadata(session, corrupt_trailing_line=True)
store = SessionStore(repository, data_dir=tmp_path / "data")
store.import_legacy_viewer_live(sessions)
detail = store.get_session(session.name)
assert detail.summary.status == "interrupted"
assert detail.summary.replayable is True
assert detail.summary.modalities == ("point-cloud", "trajectory")
assert detail.summary.started_at_utc == "2026-07-16T20:56:32.699Z"
assert detail.summary.completed_at_utc == "2026-07-16T20:56:35.199Z"
assert detail.summary.duration_seconds == 2.5
assert detail.artifacts[0].integrity_status == "validated-prefix"
assert store.prepare_replay(session.name).source_path.name == "mqtt.raw.k1mqtt"
def test_catalog_upsert_promotes_recovered_session_after_summary_is_completed(
tmp_path: Path,
) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
summary_path = session / "captures" / "mqtt_live" / "mqtt.summary.json"
metadata_path = session / "captures" / "mqtt_live" / "mqtt.metadata.jsonl"
completed_summary = summary_path.read_text(encoding="utf-8")
completed_metadata = metadata_path.read_text(encoding="utf-8")
replace_summary_with_recovery_metadata(session)
store = SessionStore(repository, data_dir=tmp_path / "data")
store.import_legacy_viewer_live(sessions)
assert store.get_session(session.name).summary.status == "interrupted"
summary_path.write_text(completed_summary, encoding="utf-8")
metadata_path.write_text(completed_metadata, encoding="utf-8")
store.import_legacy_viewer_live(sessions)
promoted = store.get_session(session.name).summary
assert promoted.status == "ready"
assert promoted.duration_seconds == 1440.1
assert promoted.replayable is True
def test_interrupted_capture_does_not_trust_metadata_outside_session(tmp_path: Path) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
replace_summary_with_recovery_metadata(session)
metadata = session / "captures" / "mqtt_live" / "mqtt.metadata.jsonl"
outside = tmp_path / "outside.metadata.jsonl"
outside.write_bytes(metadata.read_bytes())
metadata.unlink()
metadata.symlink_to(outside)
store = SessionStore(repository, data_dir=tmp_path / "data")
store.import_legacy_viewer_live(sessions)
detail = store.get_session(session.name)
assert detail.summary.status == "failed"
assert detail.summary.replayable is False
assert detail.summary.modalities == ()
def test_interrupted_capture_rejects_newline_terminated_metadata_corruption(
tmp_path: Path,
) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
replace_summary_with_recovery_metadata(session)
metadata = session / "captures" / "mqtt_live" / "mqtt.metadata.jsonl"
with metadata.open("ab") as stream:
stream.write(b"{corrupt\n")
store = SessionStore(repository, data_dir=tmp_path / "data")
store.import_legacy_viewer_live(sessions)
detail = store.get_session(session.name)
assert detail.summary.status == "failed"
assert detail.summary.replayable is False
def test_replay_resolution_rejects_artifact_symlink_escape(tmp_path: Path) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
store = SessionStore(repository, data_dir=tmp_path / "data")
store.import_legacy_viewer_live(sessions)
raw = session / "captures" / "mqtt_live" / "mqtt.raw.k1mqtt"
outside = tmp_path / "outside.k1mqtt"
outside.write_bytes(raw.read_bytes())
raw.unlink()
raw.symlink_to(outside)
with pytest.raises(SessionIntegrityError, match="escapes"):
store.prepare_replay(session.name)
def test_completed_capture_is_not_replayable_when_declared_integrity_fails(
tmp_path: Path,
) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
raw = session / "captures" / "mqtt_live" / "mqtt.raw.k1mqtt"
corrupted = bytearray(raw.read_bytes())
corrupted[-1] ^= 0x01
raw.write_bytes(corrupted)
store = SessionStore(repository, data_dir=tmp_path / "data")
store.import_legacy_viewer_live(sessions)
detail = store.get_session(session.name)
assert detail.summary.status == "failed"
assert detail.summary.replayable is False
assert detail.summary.modalities == ()
def test_completed_capture_is_not_replayable_when_declared_count_is_wrong(
tmp_path: Path,
) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
summary_path = session / "captures" / "mqtt_live" / "mqtt.summary.json"
summary = json.loads(summary_path.read_text(encoding="utf-8"))
summary["message_count"] = 3
summary_path.write_text(json.dumps(summary), encoding="utf-8")
store = SessionStore(repository, data_dir=tmp_path / "data")
store.import_legacy_viewer_live(sessions)
assert store.get_session(session.name).summary.replayable is False
def test_current_session_marker_keeps_active_capture_out_of_replay(tmp_path: Path) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
(sessions / ".current_session").write_text(
f"sessions/{session.name}\n",
encoding="utf-8",
)
store = SessionStore(repository, data_dir=tmp_path / "data")
store.import_legacy_viewer_live(sessions)
with pytest.raises(SessionNotFoundError):
store.get_session(session.name)
(sessions / ".current_session").unlink()
store.import_legacy_viewer_live(sessions)
assert store.get_session(session.name).summary.replayable is True
def test_interrupted_capture_replays_only_committed_prefix_before_partial_raw_tail(
tmp_path: Path,
) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
replace_summary_with_recovery_metadata(session)
raw = session / "captures" / "mqtt_live" / "mqtt.raw.k1mqtt"
committed_bytes = raw.stat().st_size
with raw.open("ab") as stream:
stream.write(FRAME_HEADER.pack(12, 100)[:7])
store = SessionStore(repository, data_dir=tmp_path / "data")
store.import_legacy_viewer_live(sessions)
detail = store.get_session(session.name)
command = store.prepare_replay(session.name)
assert detail.summary.status == "interrupted"
assert detail.summary.replayable is True
assert command.replay_byte_length == committed_bytes
assert command.replay_byte_length < command.source_path.stat().st_size
def test_interrupted_capture_rejects_non_frame_raw_tail(tmp_path: Path) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
replace_summary_with_recovery_metadata(session)
raw = session / "captures" / "mqtt_live" / "mqtt.raw.k1mqtt"
with raw.open("ab") as stream:
stream.write(FRAME_HEADER.pack(0, 0))
store = SessionStore(repository, data_dir=tmp_path / "data")
store.import_legacy_viewer_live(sessions)
assert store.get_session(session.name).summary.replayable is False
def test_interrupted_capture_tolerates_bounded_group_commit_raw_tail(
tmp_path: Path,
) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
replace_summary_with_recovery_metadata(session)
raw = session / "captures" / "mqtt_live" / "mqtt.raw.k1mqtt"
committed_bytes = raw.stat().st_size
with raw.open("ab") as stream:
for payload in (b"pending-one", b"pending-two"):
topic = b"RealtimePath"
stream.write(FRAME_HEADER.pack(len(topic), len(payload)))
stream.write(topic)
stream.write(payload)
store = SessionStore(repository, data_dir=tmp_path / "data")
store.import_legacy_viewer_live(sessions)
command = store.prepare_replay(session.name)
assert store.get_session(session.name).summary.status == "interrupted"
assert command.replay_byte_length == committed_bytes
def test_failed_final_summary_falls_back_to_last_committed_prefix(tmp_path: Path) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
capture = session / "captures" / "mqtt_live"
metadata = capture / "mqtt.metadata.jsonl"
first_record = metadata.read_text(encoding="utf-8").splitlines(keepends=True)[0]
metadata.write_text(first_record, encoding="utf-8")
summary_path = capture / "mqtt.summary.json"
summary = json.loads(summary_path.read_text(encoding="utf-8"))
summary["error"] = "synthetic metadata fsync failure"
summary_path.write_text(json.dumps(summary), encoding="utf-8")
store = SessionStore(repository, data_dir=tmp_path / "data")
store.import_legacy_viewer_live(sessions)
detail = store.get_session(session.name)
assert detail.summary.status == "interrupted"
assert detail.summary.replayable is True
assert detail.summary.modalities == ("point-cloud",)
def test_layout_save_is_atomic_and_revision_checked(tmp_path: Path) -> None:
store = SessionStore(tmp_path / "repo", data_dir=tmp_path / "data")
first = store.save_layout(
"observation.spatial",
schema_version=1,
expected_revision=0,
name="Операторская сцена",
layout={"visible_source_ids": ["sensor.lidar.primary"], "windows": []},
)
assert first.revision == 1
assert store.get_layout("observation.spatial") == first
with pytest.raises(LayoutConflictError, match="revision changed"):
store.save_layout(
"observation.spatial",
schema_version=1,
expected_revision=0,
name="Устаревшая запись",
layout={},
)
second = store.save_layout(
"observation.spatial",
schema_version=1,
expected_revision=1,
name="Операторская сцена",
layout={"visible_source_ids": [], "windows": []},
)
assert second.revision == 2
assert store.get_layout("observation.spatial").layout["visible_source_ids"] == []