feat(sessions): add durable observation archive and replay API

tests/test_session_recording.py

@@ -0,0 +1,522 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import replace
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+import k1link.sessions.recording as recording_module
+from k1link.sessions.models import ReplayCommand
+from k1link.sessions.recording import (
+    CACHE_SCHEMA,
+    RERUN_RECORDING_MEDIA_TYPE,
+    RecordingMaterializationError,
+    SessionRecordingMaterializer,
+)
+
+
+def _sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+class FakeExporter:
+    def __init__(self, *, delay_seconds: float = 0.0) -> None:
+        self.calls = 0
+        self.delay_seconds = delay_seconds
+        self._lock = threading.Lock()
+
+    def __call__(self, source: Path, destination: Path) -> dict[str, object]:
+        with self._lock:
+            self.calls += 1
+            sequence = self.calls
+        if self.delay_seconds:
+            time.sleep(self.delay_seconds)
+        destination.write_bytes(b"RRD" + sequence.to_bytes(2, "big") + source.read_bytes())
+        return {
+            "source_sha256": _sha256(source),
+            "rrd_sha256": _sha256(destination),
+            "rrd_bytes": destination.stat().st_size,
+            "timeline": "session_time",
+            "timeline_start_ns": 0,
+            "timeline_end_ns": 2_500_000_000,
+        }
+
+
+def _command(tmp_path: Path) -> ReplayCommand:
+    tmp_path.mkdir(parents=True, exist_ok=True)
+    source = tmp_path / "mqtt.raw.k1mqtt"
+    source.write_bytes(b"native-source-recording")
+    metadata = tmp_path / "mqtt.metadata.jsonl"
+    metadata.write_text('{"record_type":"message","sequence":1}\n', encoding="utf-8")
+    return ReplayCommand(
+        session_id="20260716T205632Z_viewer_live",
+        source_path=source,
+        allowed_root=tmp_path,
+        session_root=tmp_path,
+        replay_byte_length=source.stat().st_size,
+        metadata_byte_length=metadata.stat().st_size,
+        expected_source_sha256=None,
+        speed=1.0,
+        loop=False,
+    )
+
+
+def test_materializer_reuses_only_a_digest_validated_private_cache(tmp_path: Path) -> None:
+    command = _command(tmp_path)
+    exporter = FakeExporter()
+    materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
+
+    first = materializer.materialize(command)
+    second = materializer.materialize(command)
+
+    assert first == second
+    assert exporter.calls == 1
+    assert first.media_type == RERUN_RECORDING_MEDIA_TYPE
+    assert first.timeline == "session_time"
+    assert first.timeline_start_ns == 0
+    assert first.timeline_end_ns == 2_500_000_000
+    assert first.path.is_relative_to(materializer.recordings_root)
+    assert first.path.stat().st_mode & 0o777 == 0o600
+    sidecar = first.path.with_name("scene.rrd.cache.json")
+    assert sidecar.stat().st_mode & 0o777 == 0o600
+    document = json.loads(sidecar.read_text(encoding="utf-8"))
+    assert document["schema_version"] == CACHE_SCHEMA
+    assert "source_path" not in document
+    assert "recording_path" not in document
+    assert str(tmp_path) not in sidecar.read_text(encoding="utf-8")
+
+    after_restart = SessionRecordingMaterializer(
+        tmp_path / "private",
+        exporter=exporter,
+    ).materialize(command)
+    assert after_restart == first
+    assert exporter.calls == 1
+
+
+@pytest.mark.parametrize(
+    "obsolete_schema",
+    [
+        "missioncore.derived-rerun-recording-cache/v1",
+        "missioncore.derived-rerun-recording-cache/v2",
+        "missioncore.derived-rerun-recording-cache/v4",
+        "missioncore.derived-rerun-recording-cache/v5",
+    ],
+)
+def test_materializer_rebuilds_incompatible_recording_cache_schema(
+    tmp_path: Path,
+    obsolete_schema: str,
+) -> None:
+    command = _command(tmp_path)
+    exporter = FakeExporter()
+    private_root = tmp_path / "private"
+    first = SessionRecordingMaterializer(private_root, exporter=exporter).materialize(command)
+    sidecar = first.path.with_name("scene.rrd.cache.json")
+    document = json.loads(sidecar.read_text(encoding="utf-8"))
+    document["schema_version"] = obsolete_schema
+    sidecar.write_text(json.dumps(document), encoding="utf-8")
+
+    rebuilt = SessionRecordingMaterializer(private_root, exporter=exporter).materialize(command)
+
+    assert exporter.calls == 2
+    assert rebuilt.path == first.path
+    assert json.loads(sidecar.read_text(encoding="utf-8"))["schema_version"] == CACHE_SCHEMA
+
+
+def test_failed_rebuild_preserves_previously_published_cache(tmp_path: Path) -> None:
+    command = _command(tmp_path)
+    exporter = FakeExporter()
+    materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
+    first = materializer.materialize(command)
+    published_bytes = first.path.read_bytes()
+    sidecar = first.path.with_name("scene.rrd.cache.json")
+    published_sidecar = sidecar.read_bytes()
+    command.source_path.write_bytes(b"new-source-that-requires-rebuild")
+    changed = replace(command, replay_byte_length=command.source_path.stat().st_size)
+
+    def fail_export(_source: Path, destination: Path) -> dict[str, object]:
+        destination.write_bytes(b"incomplete-candidate")
+        raise OSError("simulated export failure")
+
+    failing = SessionRecordingMaterializer(tmp_path / "private", exporter=fail_export)
+    with pytest.raises(RecordingMaterializationError):
+        failing.materialize(changed)
+
+    assert first.path.read_bytes() == published_bytes
+    assert sidecar.read_bytes() == published_sidecar
+    assert not tuple(first.path.parent.glob(".scene.*.candidate.rrd"))
+
+
+def test_materializer_rebuilds_when_source_or_recording_changes(tmp_path: Path) -> None:
+    command = _command(tmp_path)
+    exporter = FakeExporter()
+    materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
+
+    first = materializer.materialize(command)
+    original_recording_stat = first.path.stat()
+    corrupted = bytearray(first.path.read_bytes())
+    corrupted[-1] ^= 0x01
+    first.path.write_bytes(corrupted)
+    os.utime(
+        first.path,
+        ns=(original_recording_stat.st_atime_ns, original_recording_stat.st_mtime_ns),
+    )
+
+    repaired = materializer.materialize(command)
+    assert exporter.calls == 2
+    assert repaired.sha256 == _sha256(repaired.path)
+    assert repaired.sha256 != hashlib.sha256(corrupted).hexdigest()
+
+    command.source_path.write_bytes(b"new-native-source")
+    command = replace(command, replay_byte_length=command.source_path.stat().st_size)
+    updated = materializer.materialize(command)
+    assert exporter.calls == 3
+    assert updated.source_sha256 == _sha256(command.source_path)
+
+
+def test_concurrent_materialization_exports_one_recording(tmp_path: Path) -> None:
+    command = _command(tmp_path)
+    exporter = FakeExporter(delay_seconds=0.05)
+    materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
+
+    with ThreadPoolExecutor(max_workers=8) as executor:
+        recordings = tuple(executor.map(materializer.materialize, [command] * 8))
+
+    assert exporter.calls == 1
+    assert len({recording.sha256 for recording in recordings}) == 1
+    assert len({recording.path for recording in recordings}) == 1
+
+
+def test_materializer_never_follows_cache_symlinks_outside_private_root(
+    tmp_path: Path,
+) -> None:
+    command = _command(tmp_path)
+    exporter = FakeExporter()
+    materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
+    outside = tmp_path / "outside"
+    outside.mkdir()
+    session_cache = materializer.recordings_root / command.session_id
+    session_cache.symlink_to(outside, target_is_directory=True)
+
+    with pytest.raises(RecordingMaterializationError, match="must not be a symlink"):
+        materializer.materialize(command)
+
+    assert list(outside.iterdir()) == []
+    session_cache.unlink()
+    session_cache.mkdir()
+    outside_recording = outside / "scene.rrd"
+    outside_recording.write_bytes(b"do-not-touch")
+    (session_cache / "scene.rrd").symlink_to(outside_recording)
+
+    recording = materializer.materialize(command)
+
+    assert recording.path.read_bytes().startswith(b"RRD")
+    assert outside_recording.read_bytes() == b"do-not-touch"
+
+
+def test_materializer_revalidates_prepared_source_without_following_replaced_symlink(
+    tmp_path: Path,
+) -> None:
+    command = _command(tmp_path / "session")
+    exporter = FakeExporter()
+    materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
+    outside = tmp_path / "outside.k1mqtt"
+    outside.write_bytes(command.source_path.read_bytes())
+    command.source_path.unlink()
+    command.source_path.symlink_to(outside)
+
+    with pytest.raises(RecordingMaterializationError, match="missing or unsafe"):
+        materializer.materialize(command)
+
+    assert exporter.calls == 0
+    assert outside.read_bytes() == b"native-source-recording"
+
+
+def test_materializer_exports_only_validated_prefix_before_crash_tail(
+    tmp_path: Path,
+) -> None:
+    command = _command(tmp_path / "session")
+    committed_raw_bytes = command.replay_byte_length
+    committed_metadata_bytes = command.metadata_byte_length
+    with command.source_path.open("ab") as stream:
+        stream.write(b"uncommitted-raw-tail")
+    metadata = command.source_path.with_name("mqtt.metadata.jsonl")
+    with metadata.open("ab") as stream:
+        stream.write(b'{"record_type":"message"')
+    observed: list[tuple[int, int]] = []
+
+    class PrefixExporter(FakeExporter):
+        def __call__(self, source: Path, destination: Path) -> dict[str, object]:
+            observed.append(
+                (source.stat().st_size, source.with_name("mqtt.metadata.jsonl").stat().st_size)
+            )
+            return super().__call__(source, destination)
+
+    exporter = PrefixExporter()
+    materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
+
+    recording = materializer.materialize(command)
+
+    assert observed == [(committed_raw_bytes, committed_metadata_bytes)]
+    assert recording.source_sha256 == hashlib.sha256(b"native-source-recording").hexdigest()
+    assert command.source_path.read_bytes().endswith(b"uncommitted-raw-tail")
+
+
+def test_global_singleflight_bounds_exports_across_different_sessions(tmp_path: Path) -> None:
+    first = _command(tmp_path / "first")
+    second = replace(
+        _command(tmp_path / "second"),
+        session_id="20260716T205633Z_viewer_live",
+    )
+
+    class ConcurrencyExporter(FakeExporter):
+        def __init__(self) -> None:
+            super().__init__()
+            self.active = 0
+            self.max_active = 0
+
+        def __call__(self, source: Path, destination: Path) -> dict[str, object]:
+            with self._lock:
+                self.active += 1
+                self.max_active = max(self.max_active, self.active)
+            try:
+                time.sleep(0.03)
+                return super().__call__(source, destination)
+            finally:
+                with self._lock:
+                    self.active -= 1
+
+    exporter = ConcurrencyExporter()
+    materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
+
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        tuple(executor.map(materializer.materialize, (first, second)))
+
+    assert exporter.calls == 2
+    assert exporter.max_active == 1
+
+
+def test_cross_process_file_lock_serializes_independent_materializer_instances(
+    tmp_path: Path,
+) -> None:
+    first = _command(tmp_path / "first")
+    second = replace(
+        _command(tmp_path / "second"),
+        session_id="20260716T205634Z_viewer_live",
+    )
+
+    class ConcurrencyExporter(FakeExporter):
+        def __init__(self) -> None:
+            super().__init__()
+            self.active = 0
+            self.max_active = 0
+
+        def __call__(self, source: Path, destination: Path) -> dict[str, object]:
+            with self._lock:
+                self.active += 1
+                self.max_active = max(self.max_active, self.active)
+            try:
+                time.sleep(0.03)
+                return super().__call__(source, destination)
+            finally:
+                with self._lock:
+                    self.active -= 1
+
+    exporter = ConcurrencyExporter()
+    private_root = tmp_path / "private"
+    first_materializer = SessionRecordingMaterializer(private_root, exporter=exporter)
+    second_materializer = SessionRecordingMaterializer(private_root, exporter=exporter)
+
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        first_future = executor.submit(first_materializer.materialize, first)
+        second_future = executor.submit(second_materializer.materialize, second)
+        assert first_future.result(timeout=2).path.exists()
+        assert second_future.result(timeout=2).path.exists()
+
+    assert exporter.calls == 2
+    assert exporter.max_active == 1
+
+
+def test_export_scavenges_confined_crash_candidates_before_quota_check(
+    tmp_path: Path,
+) -> None:
+    command = _command(tmp_path / "session")
+    private_root = tmp_path / "private"
+    materializer = SessionRecordingMaterializer(
+        private_root,
+        exporter=FakeExporter(),
+        cache_max_bytes=32 * 1024,
+        free_space_reserve_bytes=0,
+    )
+    session_cache = materializer.recordings_root / command.session_id
+    session_cache.mkdir()
+    stale_candidate = session_cache / ".scene.deadbeef.candidate.rrd"
+    stale_candidate.write_bytes(b"x" * (24 * 1024))
+    stale_export_temp = session_cache / "..scene.deadbeef.candidate.rrd.uuid.tmp"
+    stale_export_temp.write_bytes(b"x" * 1024)
+    stale_stage = session_cache / ".source.deadbeef.tmp"
+    stale_stage.mkdir()
+    (stale_stage / "mqtt.raw.k1mqtt").write_bytes(b"x" * 1024)
+
+    recording = materializer.materialize(command)
+
+    assert recording.path.exists()
+    assert not stale_candidate.exists()
+    assert not stale_export_temp.exists()
+    assert not stale_stage.exists()
+
+
+def test_cached_recording_and_response_lease_do_not_wait_for_other_export(
+    tmp_path: Path,
+) -> None:
+    first = _command(tmp_path / "first")
+    second = replace(
+        _command(tmp_path / "second"),
+        session_id="20260716T205633Z_viewer_live",
+    )
+
+    class BlockingSecondExporter(FakeExporter):
+        def __init__(self) -> None:
+            super().__init__()
+            self.second_started = threading.Event()
+            self.release_second = threading.Event()
+
+        def __call__(self, source: Path, destination: Path) -> dict[str, object]:
+            with self._lock:
+                next_call = self.calls + 1
+            if next_call == 2:
+                self.second_started.set()
+                if not self.release_second.wait(timeout=2):
+                    raise AssertionError("test did not release the blocked export")
+            return super().__call__(source, destination)
+
+    exporter = BlockingSecondExporter()
+    materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
+    first_recording = materializer.materialize(first)
+
+    with ThreadPoolExecutor(max_workers=3) as executor:
+        exporting = executor.submit(materializer.materialize, second)
+        assert exporter.second_started.wait(timeout=1)
+
+        cached = executor.submit(materializer.materialize, first).result(timeout=0.2)
+        pinned, release = executor.submit(
+            materializer.materialize_pinned,
+            first,
+        ).result(timeout=0.2)
+
+        assert cached == first_recording
+        assert pinned == first_recording
+        release()
+        exporter.release_second.set()
+        assert exporting.result(timeout=1).session_id == second.session_id
+
+
+def test_cache_quota_evicts_lru_derived_recording_without_deleting_raw(
+    tmp_path: Path,
+) -> None:
+    first = _command(tmp_path / "first")
+    second = replace(
+        _command(tmp_path / "second"),
+        session_id="20260716T205633Z_viewer_live",
+    )
+
+    class SizedExporter(FakeExporter):
+        def __call__(self, source: Path, destination: Path) -> dict[str, object]:
+            with self._lock:
+                self.calls += 1
+            destination.write_bytes(b"R" * (40 * 1024))
+            return {
+                "source_sha256": _sha256(source),
+                "rrd_sha256": _sha256(destination),
+                "rrd_bytes": destination.stat().st_size,
+                "timeline": "session_time",
+                "timeline_start_ns": 0,
+                "timeline_end_ns": 1,
+            }
+
+    materializer = SessionRecordingMaterializer(
+        tmp_path / "private",
+        exporter=SizedExporter(),
+        cache_max_bytes=50 * 1024,
+        free_space_reserve_bytes=0,
+    )
+    first_recording = materializer.materialize(first)
+    first_source_bytes = first.source_path.read_bytes()
+
+    second_recording = materializer.materialize(second)
+
+    assert not first_recording.path.exists()
+    assert second_recording.path.exists()
+    assert first.source_path.read_bytes() == first_source_bytes
+    assert second.source_path.exists()
+
+
+def test_pinned_cache_entry_survives_eviction_until_release(tmp_path: Path) -> None:
+    first = _command(tmp_path / "first")
+    second = replace(
+        _command(tmp_path / "second"),
+        session_id="20260716T205633Z_viewer_live",
+    )
+
+    class SizedExporter(FakeExporter):
+        def __call__(self, source: Path, destination: Path) -> dict[str, object]:
+            with self._lock:
+                self.calls += 1
+            destination.write_bytes(b"R" * (40 * 1024))
+            return {
+                "source_sha256": _sha256(source),
+                "rrd_sha256": _sha256(destination),
+                "rrd_bytes": destination.stat().st_size,
+                "timeline": "session_time",
+                "timeline_start_ns": 0,
+                "timeline_end_ns": 1,
+            }
+
+    materializer = SessionRecordingMaterializer(
+        tmp_path / "private",
+        exporter=SizedExporter(),
+        cache_max_bytes=50 * 1024,
+        free_space_reserve_bytes=0,
+    )
+    first_recording, release = materializer.materialize_pinned(first)
+
+    with pytest.raises(RecordingMaterializationError, match="cache quota"):
+        materializer.materialize(second)
+
+    assert first_recording.path.exists()
+
+    release()
+    second_recording = materializer.materialize(second)
+
+    assert not first_recording.path.exists()
+    assert second_recording.path.exists()
+
+
+def test_cache_free_space_reserve_refuses_export_without_touching_raw(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    command = _command(tmp_path / "session")
+    source_bytes = command.source_path.read_bytes()
+    materializer = SessionRecordingMaterializer(
+        tmp_path / "private",
+        exporter=FakeExporter(),
+        cache_max_bytes=1024 * 1024,
+        free_space_reserve_bytes=1024,
+    )
+    monkeypatch.setattr(
+        recording_module.shutil,
+        "disk_usage",
+        lambda _path: SimpleNamespace(free=1023),
+    )
+
+    with pytest.raises(RecordingMaterializationError, match="free-space reserve"):
+        materializer.materialize(command)
+
+    assert command.source_path.read_bytes() == source_bytes