560 lines
20 KiB
Python
560 lines
20 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
import k1link.sessions.recording as recording_module
|
|
from k1link.sessions.models import ReplayArtifact, ReplayCommand
|
|
from k1link.sessions.recording import (
|
|
CACHE_SCHEMA,
|
|
RERUN_RECORDING_MEDIA_TYPE,
|
|
RecordingMaterializationError,
|
|
SessionRecordingMaterializer,
|
|
)
|
|
|
|
|
|
def _sha256(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
class FakeExporter:
|
|
def __init__(self, *, delay_seconds: float = 0.0) -> None:
|
|
self.calls = 0
|
|
self.delay_seconds = delay_seconds
|
|
self._lock = threading.Lock()
|
|
|
|
def __call__(self, source: Path, destination: Path) -> dict[str, object]:
|
|
with self._lock:
|
|
self.calls += 1
|
|
sequence = self.calls
|
|
if self.delay_seconds:
|
|
time.sleep(self.delay_seconds)
|
|
destination.write_bytes(b"RRD" + sequence.to_bytes(2, "big") + source.read_bytes())
|
|
return {
|
|
"source_sha256": _sha256(source),
|
|
"rrd_sha256": _sha256(destination),
|
|
"rrd_bytes": destination.stat().st_size,
|
|
"timeline": "session_time",
|
|
"timeline_start_ns": 0,
|
|
"timeline_end_ns": 2_500_000_000,
|
|
}
|
|
|
|
|
|
def _command(tmp_path: Path) -> ReplayCommand:
|
|
tmp_path.mkdir(parents=True, exist_ok=True)
|
|
source = tmp_path / "mqtt.raw.k1mqtt"
|
|
source.write_bytes(b"native-source-recording")
|
|
metadata = tmp_path / "mqtt.metadata.jsonl"
|
|
metadata.write_text('{"record_type":"message","sequence":1}\n', encoding="utf-8")
|
|
return ReplayCommand(
|
|
session_id="20260716T205632Z_viewer_live",
|
|
plugin_id="test.recording-plugin",
|
|
allowed_root=tmp_path,
|
|
session_root=tmp_path,
|
|
primary_artifact_id="primary",
|
|
artifacts=(
|
|
ReplayArtifact(
|
|
artifact_id="primary",
|
|
path=source,
|
|
media_type="application/x-test-recording",
|
|
file_byte_length=source.stat().st_size,
|
|
replay_byte_length=source.stat().st_size,
|
|
expected_sha256=None,
|
|
),
|
|
ReplayArtifact(
|
|
artifact_id="index",
|
|
path=metadata,
|
|
media_type="application/x-ndjson",
|
|
file_byte_length=metadata.stat().st_size,
|
|
replay_byte_length=metadata.stat().st_size,
|
|
expected_sha256=None,
|
|
),
|
|
),
|
|
timeline_origin_epoch_ns=0,
|
|
timeline_origin_monotonic_ns=0,
|
|
speed=1.0,
|
|
loop=False,
|
|
)
|
|
|
|
|
|
def _replace_primary(command: ReplayCommand, **changes: object) -> ReplayCommand:
|
|
replacement = replace(command.primary_artifact, **changes)
|
|
return replace(
|
|
command,
|
|
artifacts=tuple(
|
|
replacement if artifact.artifact_id == command.primary_artifact_id else artifact
|
|
for artifact in command.artifacts
|
|
),
|
|
)
|
|
|
|
|
|
def test_materializer_reuses_only_a_digest_validated_private_cache(tmp_path: Path) -> None:
|
|
command = _command(tmp_path)
|
|
exporter = FakeExporter()
|
|
materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
|
|
|
|
first = materializer.materialize(command)
|
|
second = materializer.materialize(command)
|
|
|
|
assert first == second
|
|
assert exporter.calls == 1
|
|
assert first.media_type == RERUN_RECORDING_MEDIA_TYPE
|
|
assert first.timeline == "session_time"
|
|
assert first.timeline_start_ns == 0
|
|
assert first.timeline_end_ns == 2_500_000_000
|
|
assert first.path.is_relative_to(materializer.recordings_root)
|
|
assert first.path.stat().st_mode & 0o777 == 0o600
|
|
sidecar = first.path.with_name("scene.rrd.cache.json")
|
|
assert sidecar.stat().st_mode & 0o777 == 0o600
|
|
document = json.loads(sidecar.read_text(encoding="utf-8"))
|
|
assert document["schema_version"] == CACHE_SCHEMA
|
|
assert "source_path" not in document
|
|
assert "recording_path" not in document
|
|
assert str(tmp_path) not in sidecar.read_text(encoding="utf-8")
|
|
|
|
after_restart = SessionRecordingMaterializer(
|
|
tmp_path / "private",
|
|
exporter=exporter,
|
|
).materialize(command)
|
|
assert after_restart == first
|
|
assert exporter.calls == 1
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"obsolete_schema",
|
|
[
|
|
"missioncore.derived-rerun-recording-cache/v1",
|
|
"missioncore.derived-rerun-recording-cache/v2",
|
|
"missioncore.derived-rerun-recording-cache/v4",
|
|
"missioncore.derived-rerun-recording-cache/v5",
|
|
],
|
|
)
|
|
def test_materializer_rebuilds_incompatible_recording_cache_schema(
|
|
tmp_path: Path,
|
|
obsolete_schema: str,
|
|
) -> None:
|
|
command = _command(tmp_path)
|
|
exporter = FakeExporter()
|
|
private_root = tmp_path / "private"
|
|
first = SessionRecordingMaterializer(private_root, exporter=exporter).materialize(command)
|
|
sidecar = first.path.with_name("scene.rrd.cache.json")
|
|
document = json.loads(sidecar.read_text(encoding="utf-8"))
|
|
document["schema_version"] = obsolete_schema
|
|
sidecar.write_text(json.dumps(document), encoding="utf-8")
|
|
|
|
rebuilt = SessionRecordingMaterializer(private_root, exporter=exporter).materialize(command)
|
|
|
|
assert exporter.calls == 2
|
|
assert rebuilt.path == first.path
|
|
assert json.loads(sidecar.read_text(encoding="utf-8"))["schema_version"] == CACHE_SCHEMA
|
|
|
|
|
|
def test_failed_rebuild_preserves_previously_published_cache(tmp_path: Path) -> None:
|
|
command = _command(tmp_path)
|
|
exporter = FakeExporter()
|
|
materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
|
|
first = materializer.materialize(command)
|
|
published_bytes = first.path.read_bytes()
|
|
sidecar = first.path.with_name("scene.rrd.cache.json")
|
|
published_sidecar = sidecar.read_bytes()
|
|
command.primary_artifact.path.write_bytes(b"new-source-that-requires-rebuild")
|
|
changed = _replace_primary(
|
|
command,
|
|
file_byte_length=command.primary_artifact.path.stat().st_size,
|
|
replay_byte_length=command.primary_artifact.path.stat().st_size,
|
|
)
|
|
|
|
def fail_export(_source: Path, destination: Path) -> dict[str, object]:
|
|
destination.write_bytes(b"incomplete-candidate")
|
|
raise OSError("simulated export failure")
|
|
|
|
failing = SessionRecordingMaterializer(tmp_path / "private", exporter=fail_export)
|
|
with pytest.raises(RecordingMaterializationError):
|
|
failing.materialize(changed)
|
|
|
|
assert first.path.read_bytes() == published_bytes
|
|
assert sidecar.read_bytes() == published_sidecar
|
|
assert not tuple(first.path.parent.glob(".scene.*.candidate.rrd"))
|
|
|
|
|
|
def test_materializer_rebuilds_when_source_or_recording_changes(tmp_path: Path) -> None:
|
|
command = _command(tmp_path)
|
|
exporter = FakeExporter()
|
|
materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
|
|
|
|
first = materializer.materialize(command)
|
|
original_recording_stat = first.path.stat()
|
|
corrupted = bytearray(first.path.read_bytes())
|
|
corrupted[-1] ^= 0x01
|
|
first.path.write_bytes(corrupted)
|
|
os.utime(
|
|
first.path,
|
|
ns=(original_recording_stat.st_atime_ns, original_recording_stat.st_mtime_ns),
|
|
)
|
|
|
|
repaired = materializer.materialize(command)
|
|
assert exporter.calls == 2
|
|
assert repaired.sha256 == _sha256(repaired.path)
|
|
assert repaired.sha256 != hashlib.sha256(corrupted).hexdigest()
|
|
|
|
command.primary_artifact.path.write_bytes(b"new-native-source")
|
|
command = _replace_primary(
|
|
command,
|
|
file_byte_length=command.primary_artifact.path.stat().st_size,
|
|
replay_byte_length=command.primary_artifact.path.stat().st_size,
|
|
)
|
|
updated = materializer.materialize(command)
|
|
assert exporter.calls == 3
|
|
assert updated.source_sha256 == _sha256(command.primary_artifact.path)
|
|
|
|
|
|
def test_concurrent_materialization_exports_one_recording(tmp_path: Path) -> None:
|
|
command = _command(tmp_path)
|
|
exporter = FakeExporter(delay_seconds=0.05)
|
|
materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
|
|
|
|
with ThreadPoolExecutor(max_workers=8) as executor:
|
|
recordings = tuple(executor.map(materializer.materialize, [command] * 8))
|
|
|
|
assert exporter.calls == 1
|
|
assert len({recording.sha256 for recording in recordings}) == 1
|
|
assert len({recording.path for recording in recordings}) == 1
|
|
|
|
|
|
def test_materializer_never_follows_cache_symlinks_outside_private_root(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
command = _command(tmp_path)
|
|
exporter = FakeExporter()
|
|
materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
|
|
outside = tmp_path / "outside"
|
|
outside.mkdir()
|
|
session_cache = materializer.recordings_root / command.session_id
|
|
session_cache.symlink_to(outside, target_is_directory=True)
|
|
|
|
with pytest.raises(RecordingMaterializationError, match="must not be a symlink"):
|
|
materializer.materialize(command)
|
|
|
|
assert list(outside.iterdir()) == []
|
|
session_cache.unlink()
|
|
session_cache.mkdir()
|
|
outside_recording = outside / "scene.rrd"
|
|
outside_recording.write_bytes(b"do-not-touch")
|
|
(session_cache / "scene.rrd").symlink_to(outside_recording)
|
|
|
|
recording = materializer.materialize(command)
|
|
|
|
assert recording.path.read_bytes().startswith(b"RRD")
|
|
assert outside_recording.read_bytes() == b"do-not-touch"
|
|
|
|
|
|
def test_materializer_revalidates_prepared_source_without_following_replaced_symlink(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
command = _command(tmp_path / "session")
|
|
exporter = FakeExporter()
|
|
materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
|
|
outside = tmp_path / "outside.k1mqtt"
|
|
outside.write_bytes(command.primary_artifact.path.read_bytes())
|
|
command.primary_artifact.path.unlink()
|
|
command.primary_artifact.path.symlink_to(outside)
|
|
|
|
with pytest.raises(RecordingMaterializationError, match="missing or unsafe"):
|
|
materializer.materialize(command)
|
|
|
|
assert exporter.calls == 0
|
|
assert outside.read_bytes() == b"native-source-recording"
|
|
|
|
|
|
def test_materializer_exports_only_validated_prefix_before_crash_tail(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
command = _command(tmp_path / "session")
|
|
committed_raw_bytes = command.primary_artifact.replay_byte_length
|
|
committed_metadata_bytes = command.artifacts[1].replay_byte_length
|
|
with command.primary_artifact.path.open("ab") as stream:
|
|
stream.write(b"uncommitted-raw-tail")
|
|
metadata = command.primary_artifact.path.with_name("mqtt.metadata.jsonl")
|
|
with metadata.open("ab") as stream:
|
|
stream.write(b'{"record_type":"message"')
|
|
observed: list[tuple[int, int]] = []
|
|
|
|
class PrefixExporter(FakeExporter):
|
|
def __call__(self, source: Path, destination: Path) -> dict[str, object]:
|
|
observed.append(
|
|
(source.stat().st_size, source.with_name("mqtt.metadata.jsonl").stat().st_size)
|
|
)
|
|
return super().__call__(source, destination)
|
|
|
|
exporter = PrefixExporter()
|
|
materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
|
|
|
|
recording = materializer.materialize(command)
|
|
|
|
assert observed == [(committed_raw_bytes, committed_metadata_bytes)]
|
|
assert recording.source_sha256 == hashlib.sha256(b"native-source-recording").hexdigest()
|
|
assert command.primary_artifact.path.read_bytes().endswith(b"uncommitted-raw-tail")
|
|
|
|
|
|
def test_global_singleflight_bounds_exports_across_different_sessions(tmp_path: Path) -> None:
|
|
first = _command(tmp_path / "first")
|
|
second = replace(
|
|
_command(tmp_path / "second"),
|
|
session_id="20260716T205633Z_viewer_live",
|
|
)
|
|
|
|
class ConcurrencyExporter(FakeExporter):
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
self.active = 0
|
|
self.max_active = 0
|
|
|
|
def __call__(self, source: Path, destination: Path) -> dict[str, object]:
|
|
with self._lock:
|
|
self.active += 1
|
|
self.max_active = max(self.max_active, self.active)
|
|
try:
|
|
time.sleep(0.03)
|
|
return super().__call__(source, destination)
|
|
finally:
|
|
with self._lock:
|
|
self.active -= 1
|
|
|
|
exporter = ConcurrencyExporter()
|
|
materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
tuple(executor.map(materializer.materialize, (first, second)))
|
|
|
|
assert exporter.calls == 2
|
|
assert exporter.max_active == 1
|
|
|
|
|
|
def test_cross_process_file_lock_serializes_independent_materializer_instances(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
first = _command(tmp_path / "first")
|
|
second = replace(
|
|
_command(tmp_path / "second"),
|
|
session_id="20260716T205634Z_viewer_live",
|
|
)
|
|
|
|
class ConcurrencyExporter(FakeExporter):
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
self.active = 0
|
|
self.max_active = 0
|
|
|
|
def __call__(self, source: Path, destination: Path) -> dict[str, object]:
|
|
with self._lock:
|
|
self.active += 1
|
|
self.max_active = max(self.max_active, self.active)
|
|
try:
|
|
time.sleep(0.03)
|
|
return super().__call__(source, destination)
|
|
finally:
|
|
with self._lock:
|
|
self.active -= 1
|
|
|
|
exporter = ConcurrencyExporter()
|
|
private_root = tmp_path / "private"
|
|
first_materializer = SessionRecordingMaterializer(private_root, exporter=exporter)
|
|
second_materializer = SessionRecordingMaterializer(private_root, exporter=exporter)
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
first_future = executor.submit(first_materializer.materialize, first)
|
|
second_future = executor.submit(second_materializer.materialize, second)
|
|
assert first_future.result(timeout=2).path.exists()
|
|
assert second_future.result(timeout=2).path.exists()
|
|
|
|
assert exporter.calls == 2
|
|
assert exporter.max_active == 1
|
|
|
|
|
|
def test_export_scavenges_confined_crash_candidates_before_quota_check(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
command = _command(tmp_path / "session")
|
|
private_root = tmp_path / "private"
|
|
materializer = SessionRecordingMaterializer(
|
|
private_root,
|
|
exporter=FakeExporter(),
|
|
cache_max_bytes=32 * 1024,
|
|
free_space_reserve_bytes=0,
|
|
)
|
|
session_cache = materializer.recordings_root / command.session_id
|
|
session_cache.mkdir()
|
|
stale_candidate = session_cache / ".scene.deadbeef.candidate.rrd"
|
|
stale_candidate.write_bytes(b"x" * (24 * 1024))
|
|
stale_export_temp = session_cache / "..scene.deadbeef.candidate.rrd.uuid.tmp"
|
|
stale_export_temp.write_bytes(b"x" * 1024)
|
|
stale_stage = session_cache / ".source.deadbeef.tmp"
|
|
stale_stage.mkdir()
|
|
(stale_stage / "mqtt.raw.k1mqtt").write_bytes(b"x" * 1024)
|
|
|
|
recording = materializer.materialize(command)
|
|
|
|
assert recording.path.exists()
|
|
assert not stale_candidate.exists()
|
|
assert not stale_export_temp.exists()
|
|
assert not stale_stage.exists()
|
|
|
|
|
|
def test_cached_recording_and_response_lease_do_not_wait_for_other_export(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
first = _command(tmp_path / "first")
|
|
second = replace(
|
|
_command(tmp_path / "second"),
|
|
session_id="20260716T205633Z_viewer_live",
|
|
)
|
|
|
|
class BlockingSecondExporter(FakeExporter):
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
self.second_started = threading.Event()
|
|
self.release_second = threading.Event()
|
|
|
|
def __call__(self, source: Path, destination: Path) -> dict[str, object]:
|
|
with self._lock:
|
|
next_call = self.calls + 1
|
|
if next_call == 2:
|
|
self.second_started.set()
|
|
if not self.release_second.wait(timeout=2):
|
|
raise AssertionError("test did not release the blocked export")
|
|
return super().__call__(source, destination)
|
|
|
|
exporter = BlockingSecondExporter()
|
|
materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
|
|
first_recording = materializer.materialize(first)
|
|
|
|
with ThreadPoolExecutor(max_workers=3) as executor:
|
|
exporting = executor.submit(materializer.materialize, second)
|
|
assert exporter.second_started.wait(timeout=1)
|
|
|
|
cached = executor.submit(materializer.materialize, first).result(timeout=0.2)
|
|
pinned, release = executor.submit(
|
|
materializer.materialize_pinned,
|
|
first,
|
|
).result(timeout=0.2)
|
|
|
|
assert cached == first_recording
|
|
assert pinned == first_recording
|
|
release()
|
|
exporter.release_second.set()
|
|
assert exporting.result(timeout=1).session_id == second.session_id
|
|
|
|
|
|
def test_cache_quota_evicts_lru_derived_recording_without_deleting_raw(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
first = _command(tmp_path / "first")
|
|
second = replace(
|
|
_command(tmp_path / "second"),
|
|
session_id="20260716T205633Z_viewer_live",
|
|
)
|
|
|
|
class SizedExporter(FakeExporter):
|
|
def __call__(self, source: Path, destination: Path) -> dict[str, object]:
|
|
with self._lock:
|
|
self.calls += 1
|
|
destination.write_bytes(b"R" * (40 * 1024))
|
|
return {
|
|
"source_sha256": _sha256(source),
|
|
"rrd_sha256": _sha256(destination),
|
|
"rrd_bytes": destination.stat().st_size,
|
|
"timeline": "session_time",
|
|
"timeline_start_ns": 0,
|
|
"timeline_end_ns": 1,
|
|
}
|
|
|
|
materializer = SessionRecordingMaterializer(
|
|
tmp_path / "private",
|
|
exporter=SizedExporter(),
|
|
cache_max_bytes=50 * 1024,
|
|
free_space_reserve_bytes=0,
|
|
)
|
|
first_recording = materializer.materialize(first)
|
|
first_source_bytes = first.primary_artifact.path.read_bytes()
|
|
|
|
second_recording = materializer.materialize(second)
|
|
|
|
assert not first_recording.path.exists()
|
|
assert second_recording.path.exists()
|
|
assert first.primary_artifact.path.read_bytes() == first_source_bytes
|
|
assert second.primary_artifact.path.exists()
|
|
|
|
|
|
def test_pinned_cache_entry_survives_eviction_until_release(tmp_path: Path) -> None:
|
|
first = _command(tmp_path / "first")
|
|
second = replace(
|
|
_command(tmp_path / "second"),
|
|
session_id="20260716T205633Z_viewer_live",
|
|
)
|
|
|
|
class SizedExporter(FakeExporter):
|
|
def __call__(self, source: Path, destination: Path) -> dict[str, object]:
|
|
with self._lock:
|
|
self.calls += 1
|
|
destination.write_bytes(b"R" * (40 * 1024))
|
|
return {
|
|
"source_sha256": _sha256(source),
|
|
"rrd_sha256": _sha256(destination),
|
|
"rrd_bytes": destination.stat().st_size,
|
|
"timeline": "session_time",
|
|
"timeline_start_ns": 0,
|
|
"timeline_end_ns": 1,
|
|
}
|
|
|
|
materializer = SessionRecordingMaterializer(
|
|
tmp_path / "private",
|
|
exporter=SizedExporter(),
|
|
cache_max_bytes=50 * 1024,
|
|
free_space_reserve_bytes=0,
|
|
)
|
|
first_recording, release = materializer.materialize_pinned(first)
|
|
|
|
with pytest.raises(RecordingMaterializationError, match="cache quota"):
|
|
materializer.materialize(second)
|
|
|
|
assert first_recording.path.exists()
|
|
|
|
release()
|
|
second_recording = materializer.materialize(second)
|
|
|
|
assert not first_recording.path.exists()
|
|
assert second_recording.path.exists()
|
|
|
|
|
|
def test_cache_free_space_reserve_refuses_export_without_touching_raw(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
command = _command(tmp_path / "session")
|
|
source_bytes = command.primary_artifact.path.read_bytes()
|
|
materializer = SessionRecordingMaterializer(
|
|
tmp_path / "private",
|
|
exporter=FakeExporter(),
|
|
cache_max_bytes=1024 * 1024,
|
|
free_space_reserve_bytes=1024,
|
|
)
|
|
monkeypatch.setattr(
|
|
recording_module.shutil,
|
|
"disk_usage",
|
|
lambda _path: SimpleNamespace(free=1023),
|
|
)
|
|
|
|
with pytest.raises(RecordingMaterializationError, match="free-space reserve"):
|
|
materializer.materialize(command)
|
|
|
|
assert command.primary_artifact.path.read_bytes() == source_bytes
|