from __future__ import annotations import hashlib import json import os import threading import time from concurrent.futures import ThreadPoolExecutor from dataclasses import replace from pathlib import Path from types import SimpleNamespace import pytest import k1link.sessions.recording as recording_module from k1link.sessions.models import ReplayCommand from k1link.sessions.recording import ( CACHE_SCHEMA, RERUN_RECORDING_MEDIA_TYPE, RecordingMaterializationError, SessionRecordingMaterializer, ) def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() class FakeExporter: def __init__(self, *, delay_seconds: float = 0.0) -> None: self.calls = 0 self.delay_seconds = delay_seconds self._lock = threading.Lock() def __call__(self, source: Path, destination: Path) -> dict[str, object]: with self._lock: self.calls += 1 sequence = self.calls if self.delay_seconds: time.sleep(self.delay_seconds) destination.write_bytes(b"RRD" + sequence.to_bytes(2, "big") + source.read_bytes()) return { "source_sha256": _sha256(source), "rrd_sha256": _sha256(destination), "rrd_bytes": destination.stat().st_size, "timeline": "session_time", "timeline_start_ns": 0, "timeline_end_ns": 2_500_000_000, } def _command(tmp_path: Path) -> ReplayCommand: tmp_path.mkdir(parents=True, exist_ok=True) source = tmp_path / "mqtt.raw.k1mqtt" source.write_bytes(b"native-source-recording") metadata = tmp_path / "mqtt.metadata.jsonl" metadata.write_text('{"record_type":"message","sequence":1}\n', encoding="utf-8") return ReplayCommand( session_id="20260716T205632Z_viewer_live", source_path=source, allowed_root=tmp_path, session_root=tmp_path, replay_byte_length=source.stat().st_size, metadata_byte_length=metadata.stat().st_size, expected_source_sha256=None, speed=1.0, loop=False, ) def test_materializer_reuses_only_a_digest_validated_private_cache(tmp_path: Path) -> None: command = _command(tmp_path) exporter = FakeExporter() materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) first = materializer.materialize(command) second = materializer.materialize(command) assert first == second assert exporter.calls == 1 assert first.media_type == RERUN_RECORDING_MEDIA_TYPE assert first.timeline == "session_time" assert first.timeline_start_ns == 0 assert first.timeline_end_ns == 2_500_000_000 assert first.path.is_relative_to(materializer.recordings_root) assert first.path.stat().st_mode & 0o777 == 0o600 sidecar = first.path.with_name("scene.rrd.cache.json") assert sidecar.stat().st_mode & 0o777 == 0o600 document = json.loads(sidecar.read_text(encoding="utf-8")) assert document["schema_version"] == CACHE_SCHEMA assert "source_path" not in document assert "recording_path" not in document assert str(tmp_path) not in sidecar.read_text(encoding="utf-8") after_restart = SessionRecordingMaterializer( tmp_path / "private", exporter=exporter, ).materialize(command) assert after_restart == first assert exporter.calls == 1 @pytest.mark.parametrize( "obsolete_schema", [ "missioncore.derived-rerun-recording-cache/v1", "missioncore.derived-rerun-recording-cache/v2", "missioncore.derived-rerun-recording-cache/v4", "missioncore.derived-rerun-recording-cache/v5", ], ) def test_materializer_rebuilds_incompatible_recording_cache_schema( tmp_path: Path, obsolete_schema: str, ) -> None: command = _command(tmp_path) exporter = FakeExporter() private_root = tmp_path / "private" first = SessionRecordingMaterializer(private_root, exporter=exporter).materialize(command) sidecar = first.path.with_name("scene.rrd.cache.json") document = json.loads(sidecar.read_text(encoding="utf-8")) document["schema_version"] = obsolete_schema sidecar.write_text(json.dumps(document), encoding="utf-8") rebuilt = SessionRecordingMaterializer(private_root, exporter=exporter).materialize(command) assert exporter.calls == 2 assert rebuilt.path == first.path assert json.loads(sidecar.read_text(encoding="utf-8"))["schema_version"] == CACHE_SCHEMA def test_failed_rebuild_preserves_previously_published_cache(tmp_path: Path) -> None: command = _command(tmp_path) exporter = FakeExporter() materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) first = materializer.materialize(command) published_bytes = first.path.read_bytes() sidecar = first.path.with_name("scene.rrd.cache.json") published_sidecar = sidecar.read_bytes() command.source_path.write_bytes(b"new-source-that-requires-rebuild") changed = replace(command, replay_byte_length=command.source_path.stat().st_size) def fail_export(_source: Path, destination: Path) -> dict[str, object]: destination.write_bytes(b"incomplete-candidate") raise OSError("simulated export failure") failing = SessionRecordingMaterializer(tmp_path / "private", exporter=fail_export) with pytest.raises(RecordingMaterializationError): failing.materialize(changed) assert first.path.read_bytes() == published_bytes assert sidecar.read_bytes() == published_sidecar assert not tuple(first.path.parent.glob(".scene.*.candidate.rrd")) def test_materializer_rebuilds_when_source_or_recording_changes(tmp_path: Path) -> None: command = _command(tmp_path) exporter = FakeExporter() materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) first = materializer.materialize(command) original_recording_stat = first.path.stat() corrupted = bytearray(first.path.read_bytes()) corrupted[-1] ^= 0x01 first.path.write_bytes(corrupted) os.utime( first.path, ns=(original_recording_stat.st_atime_ns, original_recording_stat.st_mtime_ns), ) repaired = materializer.materialize(command) assert exporter.calls == 2 assert repaired.sha256 == _sha256(repaired.path) assert repaired.sha256 != hashlib.sha256(corrupted).hexdigest() command.source_path.write_bytes(b"new-native-source") command = replace(command, replay_byte_length=command.source_path.stat().st_size) updated = materializer.materialize(command) assert exporter.calls == 3 assert updated.source_sha256 == _sha256(command.source_path) def test_concurrent_materialization_exports_one_recording(tmp_path: Path) -> None: command = _command(tmp_path) exporter = FakeExporter(delay_seconds=0.05) materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) with ThreadPoolExecutor(max_workers=8) as executor: recordings = tuple(executor.map(materializer.materialize, [command] * 8)) assert exporter.calls == 1 assert len({recording.sha256 for recording in recordings}) == 1 assert len({recording.path for recording in recordings}) == 1 def test_materializer_never_follows_cache_symlinks_outside_private_root( tmp_path: Path, ) -> None: command = _command(tmp_path) exporter = FakeExporter() materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) outside = tmp_path / "outside" outside.mkdir() session_cache = materializer.recordings_root / command.session_id session_cache.symlink_to(outside, target_is_directory=True) with pytest.raises(RecordingMaterializationError, match="must not be a symlink"): materializer.materialize(command) assert list(outside.iterdir()) == [] session_cache.unlink() session_cache.mkdir() outside_recording = outside / "scene.rrd" outside_recording.write_bytes(b"do-not-touch") (session_cache / "scene.rrd").symlink_to(outside_recording) recording = materializer.materialize(command) assert recording.path.read_bytes().startswith(b"RRD") assert outside_recording.read_bytes() == b"do-not-touch" def test_materializer_revalidates_prepared_source_without_following_replaced_symlink( tmp_path: Path, ) -> None: command = _command(tmp_path / "session") exporter = FakeExporter() materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) outside = tmp_path / "outside.k1mqtt" outside.write_bytes(command.source_path.read_bytes()) command.source_path.unlink() command.source_path.symlink_to(outside) with pytest.raises(RecordingMaterializationError, match="missing or unsafe"): materializer.materialize(command) assert exporter.calls == 0 assert outside.read_bytes() == b"native-source-recording" def test_materializer_exports_only_validated_prefix_before_crash_tail( tmp_path: Path, ) -> None: command = _command(tmp_path / "session") committed_raw_bytes = command.replay_byte_length committed_metadata_bytes = command.metadata_byte_length with command.source_path.open("ab") as stream: stream.write(b"uncommitted-raw-tail") metadata = command.source_path.with_name("mqtt.metadata.jsonl") with metadata.open("ab") as stream: stream.write(b'{"record_type":"message"') observed: list[tuple[int, int]] = [] class PrefixExporter(FakeExporter): def __call__(self, source: Path, destination: Path) -> dict[str, object]: observed.append( (source.stat().st_size, source.with_name("mqtt.metadata.jsonl").stat().st_size) ) return super().__call__(source, destination) exporter = PrefixExporter() materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) recording = materializer.materialize(command) assert observed == [(committed_raw_bytes, committed_metadata_bytes)] assert recording.source_sha256 == hashlib.sha256(b"native-source-recording").hexdigest() assert command.source_path.read_bytes().endswith(b"uncommitted-raw-tail") def test_global_singleflight_bounds_exports_across_different_sessions(tmp_path: Path) -> None: first = _command(tmp_path / "first") second = replace( _command(tmp_path / "second"), session_id="20260716T205633Z_viewer_live", ) class ConcurrencyExporter(FakeExporter): def __init__(self) -> None: super().__init__() self.active = 0 self.max_active = 0 def __call__(self, source: Path, destination: Path) -> dict[str, object]: with self._lock: self.active += 1 self.max_active = max(self.max_active, self.active) try: time.sleep(0.03) return super().__call__(source, destination) finally: with self._lock: self.active -= 1 exporter = ConcurrencyExporter() materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) with ThreadPoolExecutor(max_workers=2) as executor: tuple(executor.map(materializer.materialize, (first, second))) assert exporter.calls == 2 assert exporter.max_active == 1 def test_cross_process_file_lock_serializes_independent_materializer_instances( tmp_path: Path, ) -> None: first = _command(tmp_path / "first") second = replace( _command(tmp_path / "second"), session_id="20260716T205634Z_viewer_live", ) class ConcurrencyExporter(FakeExporter): def __init__(self) -> None: super().__init__() self.active = 0 self.max_active = 0 def __call__(self, source: Path, destination: Path) -> dict[str, object]: with self._lock: self.active += 1 self.max_active = max(self.max_active, self.active) try: time.sleep(0.03) return super().__call__(source, destination) finally: with self._lock: self.active -= 1 exporter = ConcurrencyExporter() private_root = tmp_path / "private" first_materializer = SessionRecordingMaterializer(private_root, exporter=exporter) second_materializer = SessionRecordingMaterializer(private_root, exporter=exporter) with ThreadPoolExecutor(max_workers=2) as executor: first_future = executor.submit(first_materializer.materialize, first) second_future = executor.submit(second_materializer.materialize, second) assert first_future.result(timeout=2).path.exists() assert second_future.result(timeout=2).path.exists() assert exporter.calls == 2 assert exporter.max_active == 1 def test_export_scavenges_confined_crash_candidates_before_quota_check( tmp_path: Path, ) -> None: command = _command(tmp_path / "session") private_root = tmp_path / "private" materializer = SessionRecordingMaterializer( private_root, exporter=FakeExporter(), cache_max_bytes=32 * 1024, free_space_reserve_bytes=0, ) session_cache = materializer.recordings_root / command.session_id session_cache.mkdir() stale_candidate = session_cache / ".scene.deadbeef.candidate.rrd" stale_candidate.write_bytes(b"x" * (24 * 1024)) stale_export_temp = session_cache / "..scene.deadbeef.candidate.rrd.uuid.tmp" stale_export_temp.write_bytes(b"x" * 1024) stale_stage = session_cache / ".source.deadbeef.tmp" stale_stage.mkdir() (stale_stage / "mqtt.raw.k1mqtt").write_bytes(b"x" * 1024) recording = materializer.materialize(command) assert recording.path.exists() assert not stale_candidate.exists() assert not stale_export_temp.exists() assert not stale_stage.exists() def test_cached_recording_and_response_lease_do_not_wait_for_other_export( tmp_path: Path, ) -> None: first = _command(tmp_path / "first") second = replace( _command(tmp_path / "second"), session_id="20260716T205633Z_viewer_live", ) class BlockingSecondExporter(FakeExporter): def __init__(self) -> None: super().__init__() self.second_started = threading.Event() self.release_second = threading.Event() def __call__(self, source: Path, destination: Path) -> dict[str, object]: with self._lock: next_call = self.calls + 1 if next_call == 2: self.second_started.set() if not self.release_second.wait(timeout=2): raise AssertionError("test did not release the blocked export") return super().__call__(source, destination) exporter = BlockingSecondExporter() materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) first_recording = materializer.materialize(first) with ThreadPoolExecutor(max_workers=3) as executor: exporting = executor.submit(materializer.materialize, second) assert exporter.second_started.wait(timeout=1) cached = executor.submit(materializer.materialize, first).result(timeout=0.2) pinned, release = executor.submit( materializer.materialize_pinned, first, ).result(timeout=0.2) assert cached == first_recording assert pinned == first_recording release() exporter.release_second.set() assert exporting.result(timeout=1).session_id == second.session_id def test_cache_quota_evicts_lru_derived_recording_without_deleting_raw( tmp_path: Path, ) -> None: first = _command(tmp_path / "first") second = replace( _command(tmp_path / "second"), session_id="20260716T205633Z_viewer_live", ) class SizedExporter(FakeExporter): def __call__(self, source: Path, destination: Path) -> dict[str, object]: with self._lock: self.calls += 1 destination.write_bytes(b"R" * (40 * 1024)) return { "source_sha256": _sha256(source), "rrd_sha256": _sha256(destination), "rrd_bytes": destination.stat().st_size, "timeline": "session_time", "timeline_start_ns": 0, "timeline_end_ns": 1, } materializer = SessionRecordingMaterializer( tmp_path / "private", exporter=SizedExporter(), cache_max_bytes=50 * 1024, free_space_reserve_bytes=0, ) first_recording = materializer.materialize(first) first_source_bytes = first.source_path.read_bytes() second_recording = materializer.materialize(second) assert not first_recording.path.exists() assert second_recording.path.exists() assert first.source_path.read_bytes() == first_source_bytes assert second.source_path.exists() def test_pinned_cache_entry_survives_eviction_until_release(tmp_path: Path) -> None: first = _command(tmp_path / "first") second = replace( _command(tmp_path / "second"), session_id="20260716T205633Z_viewer_live", ) class SizedExporter(FakeExporter): def __call__(self, source: Path, destination: Path) -> dict[str, object]: with self._lock: self.calls += 1 destination.write_bytes(b"R" * (40 * 1024)) return { "source_sha256": _sha256(source), "rrd_sha256": _sha256(destination), "rrd_bytes": destination.stat().st_size, "timeline": "session_time", "timeline_start_ns": 0, "timeline_end_ns": 1, } materializer = SessionRecordingMaterializer( tmp_path / "private", exporter=SizedExporter(), cache_max_bytes=50 * 1024, free_space_reserve_bytes=0, ) first_recording, release = materializer.materialize_pinned(first) with pytest.raises(RecordingMaterializationError, match="cache quota"): materializer.materialize(second) assert first_recording.path.exists() release() second_recording = materializer.materialize(second) assert not first_recording.path.exists() assert second_recording.path.exists() def test_cache_free_space_reserve_refuses_export_without_touching_raw( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: command = _command(tmp_path / "session") source_bytes = command.source_path.read_bytes() materializer = SessionRecordingMaterializer( tmp_path / "private", exporter=FakeExporter(), cache_max_bytes=1024 * 1024, free_space_reserve_bytes=1024, ) monkeypatch.setattr( recording_module.shutil, "disk_usage", lambda _path: SimpleNamespace(free=1023), ) with pytest.raises(RecordingMaterializationError, match="free-space reserve"): materializer.materialize(command) assert command.source_path.read_bytes() == source_bytes