diff --git a/.gitignore b/.gitignore index 2261aa9..0674369 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,9 @@ private/ # Real laboratory captures and decoded artifacts are sensitive and large. captures/ sessions/ +# The host runtime package shares the domain name but is source, not evidence. +!src/k1link/sessions/ +!src/k1link/sessions/**/*.py artifacts/raw/ artifacts/decoded/ *.pcap diff --git a/src/k1link/mqtt/capture.py b/src/k1link/mqtt/capture.py index 5530f5e..03af9ea 100644 --- a/src/k1link/mqtt/capture.py +++ b/src/k1link/mqtt/capture.py @@ -33,6 +33,9 @@ MAX_TOPIC_BYTES = 65_535 CONNECT_TIMEOUT_SECONDS = 10.0 KEEPALIVE_SECONDS = 30 LOOP_INTERVAL_SECONDS = 0.25 +GROUP_COMMIT_INTERVAL_SECONDS = 0.5 +GROUP_COMMIT_MAX_BYTES = 4 * 1024 * 1024 +GROUP_COMMIT_MAX_MESSAGES = 32 # Eight-byte file signature followed by repeated >IQ, topic UTF-8 bytes, payload bytes. RAW_MAGIC = b"K1MQTT\x00\x01" @@ -166,6 +169,9 @@ class _CaptureWriter: self.topic_counts: dict[str, int] = {} self._raw: IO[bytes] | None = None self._metadata: IO[str] | None = None + self._pending_metadata: list[str] = [] + self._pending_raw_bytes = 0 + self._last_commit_monotonic = time.monotonic() def open(self) -> None: self.out_dir.mkdir(parents=True, exist_ok=True) @@ -179,6 +185,7 @@ class _CaptureWriter: self._raw = _open_binary_exclusive(self.raw_path) self._raw.write(RAW_MAGIC) self._metadata = _open_text_exclusive(self.metadata_path) + _fsync_directory(self.out_dir) except BaseException: with suppress(OSError): self.close() @@ -212,7 +219,9 @@ class _CaptureWriter: "max_message_bytes": self.max_message_bytes, "reason": "message_too_large", } + self._commit_pending() self._write_metadata(metadata, record) + os.fsync(metadata.fileno()) raise MessageTooLargeError( f"message on {topic!r} is {len(payload)} bytes; " f"limit is {self.max_message_bytes} bytes" @@ -246,7 +255,17 @@ class _CaptureWriter: "raw_payload_offset": offset + len(header) + len(topic_bytes), "raw_frame_bytes": frame_bytes, } - self._write_metadata(metadata, record) + self._pending_metadata.append( + json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n" + ) + self._pending_raw_bytes += frame_bytes + if ( + len(self._pending_metadata) >= GROUP_COMMIT_MAX_MESSAGES + or self._pending_raw_bytes >= GROUP_COMMIT_MAX_BYTES + or time.monotonic() - self._last_commit_monotonic + >= GROUP_COMMIT_INTERVAL_SECONDS + ): + self._commit_pending() return CapturedMqttMessage( sequence=self.message_count, topic=topic, @@ -261,7 +280,12 @@ class _CaptureWriter: def close(self) -> None: first_error: OSError | None = None - # Make raw frames durable before making their JSONL references durable. + try: + self._commit_pending() + except OSError as exc: + first_error = exc + # Make the magic (including the zero-message case) and any last stream + # buffers durable before the handles are released. for stream in (self._raw, self._metadata): if stream is None or stream.closed: continue @@ -280,6 +304,13 @@ class _CaptureWriter: if first_error is not None: raise first_error + def maybe_commit(self, now_monotonic: float | None = None) -> None: + if not self._pending_metadata: + return + now = time.monotonic() if now_monotonic is None else now_monotonic + if now - self._last_commit_monotonic >= GROUP_COMMIT_INTERVAL_SECONDS: + self._commit_pending(now_monotonic=now) + @property def raw_bytes(self) -> int: if self.raw_path.exists(): @@ -301,6 +332,26 @@ class _CaptureWriter: stream.write(json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n") stream.flush() + def _commit_pending(self, *, now_monotonic: float | None = None) -> None: + if not self._pending_metadata: + return + raw = self._require_raw() + metadata = self._require_metadata() + # Group commit invariant: durable raw bytes always precede durable + # metadata references. A crash can therefore lose only the bounded + # in-memory group, never expose metadata pointing past durable raw. + raw.flush() + os.fsync(raw.fileno()) + payload = "".join(self._pending_metadata) + self._pending_metadata.clear() + self._pending_raw_bytes = 0 + metadata.write(payload) + metadata.flush() + os.fsync(metadata.fileno()) + self._last_commit_monotonic = ( + time.monotonic() if now_monotonic is None else now_monotonic + ) + def validate_private_ipv4(value: str) -> str: """Require a literal RFC1918 address so capture cannot target arbitrary hosts.""" @@ -536,6 +587,11 @@ def capture_mqtt( break loop_result = client.loop(timeout=LOOP_INTERVAL_SECONDS) + try: + writer.maybe_commit() + except OSError as exc: + fail("capture_error", f"group commit failed: {type(exc).__name__}: {exc}") + break if loop_result != mqtt.MQTT_ERR_SUCCESS and state.error is None: fail( "connection_lost", @@ -681,3 +737,12 @@ def _write_summary_exclusive(path: Path, summary: CaptureSummary) -> None: stream.write(serialized) stream.flush() os.fsync(stream.fileno()) + _fsync_directory(path.parent) + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) diff --git a/src/k1link/sessions/__init__.py b/src/k1link/sessions/__init__.py new file mode 100644 index 0000000..8ab766a --- /dev/null +++ b/src/k1link/sessions/__init__.py @@ -0,0 +1,65 @@ +"""Host-owned observation session catalog and durable layout persistence.""" + +from .active import ( + ActiveSessionLease, + ActiveSessionLeaseError, + recover_stale_active_session_marker, +) +from .media import ( + RECORDED_MEDIA_MANIFEST_SCHEMA, + RecordedMediaFile, + RecordedMediaInspector, + RecordedMediaManifest, + validate_recorded_media_timeline, +) +from .models import ( + LayoutConflictError, + RecordedMediaArtifact, + ReplayCommand, + SessionIntegrityError, + SessionNotFoundError, + SessionNotReplayableError, +) +from .preparation import ( + RecordingPreparationQueueFull, + RecordingPreparationSnapshot, + SessionRecordingPreparationManager, +) +from .recording import ( + MaterializedRecording, + RecordingMaterializationCancelled, + RecordingMaterializationError, + SessionRecordingMaterializer, +) +from .store import ( + SessionStore, + resolve_missioncore_data_dir, + resolve_missioncore_evidence_dir, +) + +__all__ = [ + "LayoutConflictError", + "ActiveSessionLease", + "ActiveSessionLeaseError", + "MaterializedRecording", + "RecordingMaterializationCancelled", + "RecordedMediaArtifact", + "RECORDED_MEDIA_MANIFEST_SCHEMA", + "RecordedMediaFile", + "RecordedMediaInspector", + "RecordedMediaManifest", + "ReplayCommand", + "RecordingMaterializationError", + "RecordingPreparationQueueFull", + "RecordingPreparationSnapshot", + "SessionIntegrityError", + "SessionNotFoundError", + "SessionNotReplayableError", + "SessionRecordingMaterializer", + "SessionRecordingPreparationManager", + "SessionStore", + "recover_stale_active_session_marker", + "resolve_missioncore_data_dir", + "resolve_missioncore_evidence_dir", + "validate_recorded_media_timeline", +] diff --git a/src/k1link/sessions/active.py b/src/k1link/sessions/active.py new file mode 100644 index 0000000..7f40ab1 --- /dev/null +++ b/src/k1link/sessions/active.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +import importlib +import os +import stat +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +ACTIVE_SESSION_MARKER = ".current_session" + + +class ActiveSessionLeaseError(RuntimeError): + """The evidence root already has an active writer or cannot be leased.""" + + +@dataclass(slots=True) +class ActiveSessionLease: + """Cross-process lease that keeps an in-progress session out of replay. + + The marker is intentionally written before the session directory is + created. Discovery therefore observes either no candidate yet or a + candidate protected by an already locked marker. A process crash releases + the OS lock; startup recovery can then remove the stale marker while + preserving the interrupted evidence directory. + """ + + sessions_root: Path + session_root: Path + _descriptor: int + _marker_identity: tuple[int, int] + _released: bool = False + + @classmethod + def acquire(cls, sessions_root: Path, session_root: Path) -> ActiveSessionLease: + root = sessions_root.expanduser().resolve() + root.mkdir(mode=0o700, parents=True, exist_ok=True) + target = session_root.expanduser().absolute() + if target.parent.resolve() != root or not target.name: + raise ActiveSessionLeaseError("active session must be a direct child of evidence root") + + marker = root / ACTIVE_SESSION_MARKER + flags = os.O_RDWR | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(marker, flags, 0o600) + except FileExistsError as exc: + raise ActiveSessionLeaseError( + "another observation session owns the evidence root" + ) from exc + try: + payload = f"{target.name}\n".encode() + os.write(descriptor, payload) + os.fsync(descriptor) + _lock_descriptor(descriptor, blocking=False) + _fsync_directory(root) + metadata = os.fstat(descriptor) + return cls( + sessions_root=root, + session_root=target, + _descriptor=descriptor, + _marker_identity=(metadata.st_dev, metadata.st_ino), + ) + except BaseException: + try: + marker.unlink(missing_ok=True) + _fsync_directory(root) + finally: + os.close(descriptor) + raise + + def release(self) -> None: + if self._released: + return + marker = self.sessions_root / ACTIVE_SESSION_MARKER + try: + try: + metadata = marker.lstat() + except FileNotFoundError: + metadata = None + if metadata is not None and (metadata.st_dev, metadata.st_ino) == self._marker_identity: + marker.unlink() + _fsync_directory(self.sessions_root) + finally: + _unlock_descriptor(self._descriptor) + os.close(self._descriptor) + self._released = True + + def __enter__(self) -> ActiveSessionLease: + return self + + def __exit__(self, *_: object) -> None: + self.release() + + +def recover_stale_active_session_marker(sessions_root: Path) -> bool: + """Remove only an unlocked marker left by a terminated writer.""" + + root = sessions_root.expanduser().resolve() + marker = root / ACTIVE_SESSION_MARKER + try: + marker_stat = marker.lstat() + except FileNotFoundError: + return False + except OSError: + return False + if not stat.S_ISREG(marker_stat.st_mode) or marker_stat.st_size > 4096: + return False + try: + descriptor = os.open(marker, os.O_RDWR | getattr(os, "O_NOFOLLOW", 0)) + except OSError: + return False + locked = False + try: + try: + _lock_descriptor(descriptor, blocking=False) + locked = True + except OSError: + return False + opened = os.fstat(descriptor) + try: + current = marker.lstat() + except OSError: + return False + if (opened.st_dev, opened.st_ino) != (current.st_dev, current.st_ino): + return False + marker.unlink() + _fsync_directory(root) + return True + finally: + if locked: + _unlock_descriptor(descriptor) + os.close(descriptor) + + +def _lock_descriptor(descriptor: int, *, blocking: bool) -> None: + if os.name == "nt": + msvcrt = cast(Any, importlib.import_module("msvcrt")) + + os.lseek(descriptor, 0, os.SEEK_SET) + mode = msvcrt.LK_LOCK if blocking else msvcrt.LK_NBLCK + msvcrt.locking(descriptor, mode, 1) + return + import fcntl + + operation = fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB) + fcntl.flock(descriptor, operation) + + +def _unlock_descriptor(descriptor: int) -> None: + try: + if os.name == "nt": + msvcrt = cast(Any, importlib.import_module("msvcrt")) + + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_UNLCK, 1) + return + import fcntl + + fcntl.flock(descriptor, fcntl.LOCK_UN) + except OSError: + return + + +def _fsync_directory(path: Path) -> None: + try: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + except OSError: + return + try: + os.fsync(descriptor) + finally: + os.close(descriptor) diff --git a/src/k1link/sessions/legacy.py b/src/k1link/sessions/legacy.py new file mode 100644 index 0000000..416166e --- /dev/null +++ b/src/k1link/sessions/legacy.py @@ -0,0 +1,818 @@ +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import stat +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from functools import lru_cache +from pathlib import Path +from typing import IO, Any + +from k1link.mqtt.capture import ( + FRAME_HEADER, + GROUP_COMMIT_MAX_BYTES, + GROUP_COMMIT_MAX_MESSAGES, + MAX_CONFIGURABLE_MESSAGE_BYTES, + MAX_TOPIC_BYTES, + RAW_MAGIC, +) + +from .models import ( + LegacyMediaSourceCandidate, + LegacySessionCandidate, + SessionModality, + SessionStatus, +) + +MAX_LEGACY_JSON_BYTES = 2 * 1024 * 1024 +LEGACY_SESSION_PATTERN = re.compile(r"^[A-Za-z0-9._-]+_viewer_live(?:_[0-9]+)?$") +SHA256_PATTERN = re.compile(r"^[a-f0-9]{64}$") +MEDIA_SOURCE_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +MEDIA_EPOCH_PATTERN = re.compile(r"^epoch-(?!0+$)[0-9]+$") +MEDIA_SEGMENT_PATTERN = re.compile(r"^[0-9]+\.m4s$") +MAX_MEDIA_INDEX_BYTES = 32 * 1024 * 1024 +MAX_RECOVERY_METADATA_BYTES = 64 * 1024 * 1024 +MAX_RECOVERY_METADATA_LINE_BYTES = 64 * 1024 +MAX_RECOVERY_MESSAGES = 500_000 + + +@dataclass(frozen=True, slots=True) +class _RecoveredCapture: + message_count: int + topic_counts: dict[str, int] + started_at_utc: str + completed_at_utc: str + duration_seconds: float + raw_committed_bytes: int + metadata_committed_bytes: int + + +def discover_legacy_viewer_sessions(root: Path) -> tuple[LegacySessionCandidate, ...]: + """Describe legacy live sessions without copying or decoding their payloads.""" + + allowed_root = root.expanduser().resolve() + if not allowed_root.is_dir(): + return () + active_session_root = _active_session_root(allowed_root) + candidates: list[LegacySessionCandidate] = [] + for entry in sorted(allowed_root.iterdir()): + if not entry.is_dir() or not LEGACY_SESSION_PATTERN.fullmatch(entry.name): + continue + resolved = entry.resolve() + if not resolved.is_relative_to(allowed_root): + continue + # The saved-session catalog deliberately omits the writer-owned run. + # Besides avoiding a misleading interrupted/error row, this prevents + # the two-second reconciler from repeatedly scanning and hashing a + # capture that is still growing. Marker release makes it discoverable + # on the next reconciliation pass. + if active_session_root == resolved: + continue + candidates.append( + _describe_session( + allowed_root, + resolved, + active=False, + ) + ) + return tuple(candidates) + + +def _describe_session( + allowed_root: Path, + session_root: Path, + *, + active: bool, +) -> LegacySessionCandidate: + capture_root = session_root / "captures" / "mqtt_live" + raw_path = capture_root / "mqtt.raw.k1mqtt" + summary = _read_json_object(capture_root / "mqtt.summary.json") + manifest = _read_json_object(session_root / "manifest.redacted.json") + raw_bytes = raw_path.stat().st_size if _confined_file(raw_path, session_root) else 0 + raw_magic_ok = False + if raw_bytes >= len(RAW_MAGIC): + with raw_path.open("rb") as stream: + raw_magic_ok = stream.read(len(RAW_MAGIC)) == RAW_MAGIC + + has_completed_summary = _is_completed_summary(summary) + completed = ( + _validate_completed_capture(capture_root, session_root, raw_path, summary) + if has_completed_summary and raw_magic_ok + else None + ) + recovered = ( + _recover_interrupted_capture(capture_root, session_root, raw_path) + if raw_magic_ok + and completed is None + and (not has_completed_summary or summary.get("error") is not None) + else None + ) + message_count = ( + _non_negative_int(summary.get("message_count")) + if completed is not None + else recovered.message_count if recovered is not None else 0 + ) + summary_topic_counts = summary.get("topic_counts") + topic_counts = ( + _normalized_topic_counts(summary_topic_counts) + if completed is not None + else recovered.topic_counts if recovered is not None else {} + ) + media_sources = _discover_media_sources(session_root) + modalities = list(_modalities(topic_counts)) + if media_sources: + modalities.append("video") + replayable = bool( + not active + and + raw_magic_ok + and raw_bytes > len(RAW_MAGIC) + and message_count > 0 + and any(modality in {"point-cloud", "trajectory"} for modality in modalities) + ) + if completed is not None: + status = ( + "interrupted" + if active + else _status(replayable, summary.get("error"), summary.get("stop_reason")) + ) + started_at = ( + _safe_timestamp(summary.get("created_at_utc")) + or _safe_timestamp(manifest.get("started_at_utc")) + or _timestamp_from_session_name(session_root.name) + ) + completed_at = _safe_timestamp(summary.get("completed_at_utc")) or _safe_timestamp( + manifest.get("completed_at_utc") + ) + duration = _duration(summary.get("capture_elapsed_seconds")) + declared_hash = _declared_raw_hash(summary) + integrity_status = "verified" if declared_hash is not None else "validated-structure" + else: + status = "interrupted" if recovered is not None or active else "failed" + started_at = ( + recovered.started_at_utc + if recovered is not None + else _safe_timestamp(manifest.get("started_at_utc")) + or _timestamp_from_session_name(session_root.name) + ) + completed_at = recovered.completed_at_utc if recovered is not None else None + duration = recovered.duration_seconds if recovered is not None else None + declared_hash = None + integrity_status = "validated-prefix" if recovered is not None else "unverified" + + replay_raw_bytes = ( + completed.raw_committed_bytes + if completed is not None + else recovered.raw_committed_bytes if recovered is not None else 0 + ) + replay_metadata_bytes = ( + completed.metadata_committed_bytes + if completed is not None + else recovered.metadata_committed_bytes if recovered is not None else 0 + ) + + return LegacySessionCandidate( + session_id=session_root.name, + display_name=session_root.name, + status=status, + started_at_utc=started_at, + completed_at_utc=completed_at, + duration_seconds=duration, + modalities=tuple(modalities), + replayable=replayable, + total_bytes=raw_bytes + sum(source.byte_length for source in media_sources), + allowed_root=allowed_root, + session_root=session_root, + raw_path=raw_path.resolve(strict=False), + raw_byte_length=raw_bytes, + replay_raw_byte_length=replay_raw_bytes, + replay_metadata_byte_length=replay_metadata_bytes, + raw_sha256=declared_hash, + raw_integrity_status=integrity_status, + media_sources=media_sources, + ) + + +def _is_completed_summary(summary: dict[str, Any]) -> bool: + return ( + "message_count" in summary + and isinstance(summary.get("message_count"), int) + and not isinstance(summary.get("message_count"), bool) + and isinstance(summary.get("topic_counts"), dict) + and isinstance(summary.get("stop_reason"), str) + ) + + +def _validate_completed_capture( + capture_root: Path, + session_root: Path, + raw_path: Path, + summary: dict[str, Any], +) -> _RecoveredCapture | None: + metadata_path = capture_root / "mqtt.metadata.jsonl" + try: + raw_stat = raw_path.lstat() + metadata_stat = metadata_path.lstat() + except OSError: + return None + if not stat.S_ISREG(raw_stat.st_mode) or not stat.S_ISREG(metadata_stat.st_mode): + return None + fingerprint = json.dumps( + { + "message_count": summary.get("message_count"), + "topic_counts": summary.get("topic_counts"), + "raw_bytes": summary.get("raw_bytes"), + "artifact_hashes": summary.get("artifact_hashes"), + }, + sort_keys=True, + separators=(",", ":"), + ) + return _validate_completed_capture_cached( + str(capture_root), + str(session_root), + str(raw_path), + _stat_identity(raw_stat), + _stat_identity(metadata_stat), + fingerprint, + ) + + +@lru_cache(maxsize=128) +def _validate_completed_capture_cached( + capture_root_text: str, + session_root_text: str, + raw_path_text: str, + _raw_identity: tuple[int, int, int, int, int], + _metadata_identity: tuple[int, int, int, int, int], + summary_fingerprint: str, +) -> _RecoveredCapture | None: + capture_root = Path(capture_root_text) + session_root = Path(session_root_text) + raw_path = Path(raw_path_text) + summary = json.loads(summary_fingerprint) + validated = _scan_capture_prefix( + capture_root, + session_root, + raw_path, + tolerate_incomplete_metadata_tail=False, + tolerate_raw_crash_tail=False, + ) + if validated is None: + return None + if validated.message_count != _non_negative_int(summary.get("message_count")): + return None + if validated.topic_counts != _normalized_topic_counts(summary.get("topic_counts")): + return None + declared_raw_bytes = summary.get("raw_bytes") + if declared_raw_bytes is not None and ( + not isinstance(declared_raw_bytes, int) + or isinstance(declared_raw_bytes, bool) + or declared_raw_bytes != validated.raw_committed_bytes + ): + return None + raw_hash = _declared_raw_hash(summary) + if raw_hash is not None and _sha256_stable(raw_path) != raw_hash: + return None + metadata_hash = _declared_metadata_hash(summary) + metadata_path = capture_root / "mqtt.metadata.jsonl" + if metadata_hash is not None and _sha256_stable(metadata_path) != metadata_hash: + return None + return validated + + +def _recover_interrupted_capture( + capture_root: Path, + session_root: Path, + raw_path: Path, +) -> _RecoveredCapture | None: + return _scan_capture_prefix( + capture_root, + session_root, + raw_path, + tolerate_incomplete_metadata_tail=True, + tolerate_raw_crash_tail=True, + ) + + +def _scan_capture_prefix( + capture_root: Path, + session_root: Path, + raw_path: Path, + *, + tolerate_incomplete_metadata_tail: bool, + tolerate_raw_crash_tail: bool, +) -> _RecoveredCapture | None: + metadata_path = capture_root / "mqtt.metadata.jsonl" + if not _confined_file(raw_path, session_root) or not _confined_file( + metadata_path, session_root + ): + return None + try: + raw_size = raw_path.stat().st_size + with raw_path.open("rb") as raw_stream, metadata_path.open("rb") as metadata_stream: + if raw_stream.read(len(RAW_MAGIC)) != RAW_MAGIC: + return None + consumed_metadata_bytes = 0 + message_count = 0 + topic_counts: dict[str, int] = {} + first_epoch_ns: int | None = None + last_epoch_ns: int | None = None + first_monotonic_ns: int | None = None + last_monotonic_ns: int | None = None + first_timestamp: str | None = None + last_timestamp: str | None = None + metadata_committed_bytes = 0 + while ( + consumed_metadata_bytes < MAX_RECOVERY_METADATA_BYTES + and message_count < MAX_RECOVERY_MESSAGES + ): + remaining = MAX_RECOVERY_METADATA_BYTES - consumed_metadata_bytes + read_limit = min(MAX_RECOVERY_METADATA_LINE_BYTES + 1, remaining + 1) + line = metadata_stream.readline(read_limit) + if not line: + break + consumed_metadata_bytes += len(line) + if ( + len(line) > MAX_RECOVERY_METADATA_LINE_BYTES + or consumed_metadata_bytes > MAX_RECOVERY_METADATA_BYTES + ): + return None + if not line.endswith(b"\n"): + if tolerate_incomplete_metadata_tail: + break + return None + record = _metadata_record(line) + if record is None: + return None + metadata_committed_bytes = consumed_metadata_bytes + if record.get("record_type") != "message": + continue + expected_sequence = message_count + 1 + validated = _validate_recovery_frame( + raw_stream, + raw_size=raw_size, + record=record, + expected_sequence=expected_sequence, + ) + if validated is None: + return None + topic, epoch_ns, monotonic_ns, timestamp = validated + if last_monotonic_ns is not None and monotonic_ns < last_monotonic_ns: + return None + message_count = expected_sequence + topic_counts[topic] = topic_counts.get(topic, 0) + 1 + if first_epoch_ns is None: + first_epoch_ns = epoch_ns + first_monotonic_ns = monotonic_ns + first_timestamp = timestamp + last_epoch_ns = epoch_ns + last_monotonic_ns = monotonic_ns + last_timestamp = timestamp + if metadata_stream.read(1): + return None + raw_committed_bytes = raw_stream.tell() + if raw_committed_bytes != raw_size and not ( + tolerate_raw_crash_tail + and _tolerable_raw_crash_tail(raw_stream, raw_size=raw_size) + ): + return None + except OSError: + return None + if ( + message_count == 0 + or first_epoch_ns is None + or last_epoch_ns is None + or first_timestamp is None + or last_timestamp is None + or first_monotonic_ns is None + or last_monotonic_ns is None + ): + return None + return _RecoveredCapture( + message_count=message_count, + topic_counts=topic_counts, + started_at_utc=first_timestamp, + completed_at_utc=last_timestamp, + duration_seconds=(last_monotonic_ns - first_monotonic_ns) / 1_000_000_000, + raw_committed_bytes=raw_committed_bytes, + metadata_committed_bytes=metadata_committed_bytes, + ) + + +def _metadata_record(line: bytes) -> dict[str, Any] | None: + try: + value = json.loads(line.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _tolerable_raw_crash_tail(raw_stream: IO[bytes], *, raw_size: int) -> bool: + """Accept at most one raw frame which was not committed by metadata. + + The capture writer flushes raw before metadata and does not begin the next + message until the current metadata record is written. Consequently a + process crash can leave only a prefix (or all) of one additional frame. + Anything beyond that boundary is corruption, not a recoverable tail. + """ + + tail_offset = raw_stream.tell() + tail_bytes = raw_size - tail_offset + if tail_bytes <= 0: + return True + if tail_bytes > GROUP_COMMIT_MAX_BYTES + MAX_CONFIGURABLE_MESSAGE_BYTES: + return False + frames = 0 + while raw_stream.tell() < raw_size: + remaining = raw_size - raw_stream.tell() + header = raw_stream.read(min(remaining, FRAME_HEADER.size)) + if len(header) < FRAME_HEADER.size: + return frames < GROUP_COMMIT_MAX_MESSAGES + topic_length, payload_length = FRAME_HEADER.unpack(header) + if not 1 <= topic_length <= MAX_TOPIC_BYTES: + return False + if payload_length > MAX_CONFIGURABLE_MESSAGE_BYTES: + return False + frame_bytes = FRAME_HEADER.size + topic_length + payload_length + frame_remaining = raw_size - (raw_stream.tell() - FRAME_HEADER.size) + available_topic_bytes = min( + topic_length, + max(0, frame_remaining - FRAME_HEADER.size), + ) + topic_prefix = raw_stream.read(available_topic_bytes) + if available_topic_bytes < topic_length: + return frames < GROUP_COMMIT_MAX_MESSAGES + try: + topic = topic_prefix.decode("utf-8") + except UnicodeDecodeError: + return False + if not topic: + return False + payload_available = frame_remaining - FRAME_HEADER.size - topic_length + if payload_available < payload_length: + return frames < GROUP_COMMIT_MAX_MESSAGES + raw_stream.seek(payload_length, 1) + frames += 1 + if frames > GROUP_COMMIT_MAX_MESSAGES or frame_remaining < frame_bytes: + return False + return frames <= GROUP_COMMIT_MAX_MESSAGES + + +def _validate_recovery_frame( + raw_stream: IO[bytes], + *, + raw_size: int, + record: dict[str, Any], + expected_sequence: int, +) -> tuple[str, int, int, str] | None: + sequence = record.get("sequence") + topic = record.get("topic") + payload_bytes = record.get("payload_bytes") + frame_offset = record.get("raw_frame_offset") + payload_offset = record.get("raw_payload_offset") + frame_bytes = record.get("raw_frame_bytes") + epoch_ns = record.get("received_at_epoch_ns") + monotonic_ns = record.get("received_monotonic_ns") + if ( + sequence != expected_sequence + or not isinstance(topic, str) + or not topic + or not isinstance(payload_bytes, int) + or isinstance(payload_bytes, bool) + or not 0 <= payload_bytes <= MAX_CONFIGURABLE_MESSAGE_BYTES + or not isinstance(frame_offset, int) + or isinstance(frame_offset, bool) + or frame_offset != raw_stream.tell() + or not isinstance(payload_offset, int) + or isinstance(payload_offset, bool) + or not isinstance(frame_bytes, int) + or isinstance(frame_bytes, bool) + or not isinstance(epoch_ns, int) + or isinstance(epoch_ns, bool) + or epoch_ns < 0 + or not isinstance(monotonic_ns, int) + or isinstance(monotonic_ns, bool) + or monotonic_ns < 0 + ): + return None + header = raw_stream.read(FRAME_HEADER.size) + if len(header) != FRAME_HEADER.size: + return None + topic_length, raw_payload_bytes = FRAME_HEADER.unpack(header) + if ( + not 1 <= topic_length <= MAX_TOPIC_BYTES + or raw_payload_bytes != payload_bytes + or payload_offset != frame_offset + FRAME_HEADER.size + topic_length + or frame_bytes != FRAME_HEADER.size + topic_length + payload_bytes + or frame_offset + frame_bytes > raw_size + ): + return None + raw_topic = raw_stream.read(topic_length) + try: + decoded_topic = raw_topic.decode("utf-8") + except UnicodeDecodeError: + return None + if decoded_topic != topic: + return None + raw_stream.seek(payload_bytes, 1) + timestamp = _safe_timestamp(record.get("received_at_utc")) + if timestamp is None: + try: + timestamp = datetime.fromtimestamp(epoch_ns / 1_000_000_000, tz=UTC).isoformat() + except (OSError, OverflowError, ValueError): + return None + return topic, epoch_ns, monotonic_ns, timestamp + + +def _read_json_object(path: Path) -> dict[str, Any]: + try: + if not path.is_file() or path.stat().st_size > MAX_LEGACY_JSON_BYTES: + return {} + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return {} + return value if isinstance(value, dict) else {} + + +def _confined_file(path: Path, session_root: Path) -> bool: + try: + resolved = path.resolve(strict=True) + except OSError: + return False + return resolved.is_file() and resolved.is_relative_to(session_root) + + +def _non_negative_int(value: object) -> int: + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else 0 + + +def _normalized_topic_counts(value: object) -> dict[str, int]: + if not isinstance(value, dict): + return {} + result: dict[str, int] = {} + for topic, raw_count in value.items(): + count = _non_negative_int(raw_count) + if isinstance(topic, str) and count > 0: + result[topic] = count + return result + + +def _duration(value: object) -> float | None: + if not isinstance(value, (int, float)) or isinstance(value, bool): + return None + duration = float(value) + return duration if math.isfinite(duration) and duration >= 0 else None + + +def _safe_timestamp(value: object) -> str | None: + if not isinstance(value, str) or len(value) > 64: + return None + candidate = value.strip() + try: + parsed = datetime.fromisoformat(candidate.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + return None + return candidate + + +def _timestamp_from_session_name(session_id: str) -> str | None: + prefix = session_id.split("_", 1)[0] + try: + parsed = datetime.strptime(prefix, "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC) + except ValueError: + return None + return parsed.isoformat().replace("+00:00", "Z") + + +def _declared_raw_hash(summary: dict[str, Any]) -> str | None: + hashes = summary.get("artifact_hashes") + if not isinstance(hashes, dict): + return None + value = hashes.get("raw_sha256") + return value if isinstance(value, str) and SHA256_PATTERN.fullmatch(value) else None + + +def _declared_metadata_hash(summary: dict[str, Any]) -> str | None: + hashes = summary.get("artifact_hashes") + if not isinstance(hashes, dict): + return None + value = hashes.get("metadata_jsonl_sha256") + return value if isinstance(value, str) and SHA256_PATTERN.fullmatch(value) else None + + +def _sha256_stable(path: Path) -> str: + try: + current = path.lstat() + except OSError: + return "" + if not stat.S_ISREG(current.st_mode): + return "" + return _sha256_cached(str(path), _stat_identity(current)) + + +@lru_cache(maxsize=128) +def _sha256_cached( + path_text: str, + expected_identity: tuple[int, int, int, int, int], +) -> str: + path = Path(path_text) + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError: + return "" + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode) or _stat_identity(before) != expected_identity: + return "" + digest = hashlib.sha256() + while chunk := os.read(descriptor, 1024 * 1024): + digest.update(chunk) + after = os.fstat(descriptor) + try: + current = os.lstat(path) + except OSError: + return "" + if _stat_identity(before) != _stat_identity(after) or ( + current.st_dev, + current.st_ino, + ) != (before.st_dev, before.st_ino): + return "" + return digest.hexdigest() + finally: + os.close(descriptor) + + +def _stat_identity(value: os.stat_result) -> tuple[int, int, int, int, int]: + return (value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns, value.st_ctime_ns) + + +def _active_session_root(root: Path) -> Path | None: + marker = root / ".current_session" + try: + marker_stat = marker.lstat() + if not stat.S_ISREG(marker_stat.st_mode) or marker_stat.st_size > 4096: + return None + value = marker.read_text(encoding="utf-8").strip() + except (OSError, UnicodeDecodeError): + return None + if not value: + return None + raw = Path(value).expanduser() + if raw.is_absolute(): + candidate = raw + elif raw.parts and raw.parts[0] == root.name: + candidate = root.parent / raw + else: + candidate = root / raw + try: + resolved = candidate.resolve(strict=True) + except OSError: + return None + return resolved if resolved.is_dir() and resolved.is_relative_to(root) else None + + +def _modalities(topic_counts: Mapping[str, int]) -> tuple[SessionModality, ...]: + topics = { + topic + for topic, count in topic_counts.items() + if isinstance(topic, str) and _non_negative_int(count) > 0 + } + result: list[SessionModality] = [] + if any(topic == "RealtimePointcloud" or topic.endswith("/lio_pcl") for topic in topics): + result.append("point-cloud") + if any(topic == "RealtimePath" or topic.endswith("/lio_pose") for topic in topics): + result.append("trajectory") + return tuple(result) + + +def _status(replayable: bool, error: object, stop_reason: object) -> SessionStatus: + if error is None and stop_reason in { + "duration_elapsed", + "external_stop", + "keyboard_interrupt", + }: + return "ready" if replayable else "failed" + return "interrupted" if replayable else "failed" + + +def _discover_media_sources(session_root: Path) -> tuple[LegacyMediaSourceCandidate, ...]: + media_root = session_root / "media" + try: + resolved_media_root = media_root.resolve(strict=True) + except OSError: + return () + if not resolved_media_root.is_dir() or not resolved_media_root.is_relative_to(session_root): + return () + candidates: list[LegacyMediaSourceCandidate] = [] + for source_root in sorted(resolved_media_root.iterdir()): + if not source_root.is_dir() or not MEDIA_SOURCE_PATTERN.fullmatch(source_root.name): + continue + try: + resolved_source_root = source_root.resolve(strict=True) + except OSError: + continue + if not resolved_source_root.is_relative_to(resolved_media_root): + continue + epochs = [ + epoch + for epoch in sorted(resolved_source_root.iterdir()) + if epoch.is_dir() + and MEDIA_EPOCH_PATTERN.fullmatch(epoch.name) + and _validated_media_epoch(epoch, resolved_source_root.name) + ] + if not epochs: + continue + byte_length = sum(_media_epoch_bytes(epoch) for epoch in epochs) + artifact_suffix = hashlib.sha256(resolved_source_root.name.encode()).hexdigest()[:16] + candidates.append( + LegacyMediaSourceCandidate( + source_id=resolved_source_root.name, + artifact_id=f"recorded-video-{artifact_suffix}", + locator=resolved_source_root, + byte_length=byte_length, + epoch_count=len(epochs), + ) + ) + return tuple(candidates) + + +def _validated_media_epoch(epoch: Path, expected_source_id: str) -> bool: + try: + resolved_epoch = epoch.resolve(strict=True) + init_path = (epoch / "init.mp4").resolve(strict=True) + segments_root = (epoch / "segments").resolve(strict=True) + index_path = (epoch / "index.jsonl").resolve(strict=True) + summary_path = (epoch / "summary.json").resolve(strict=True) + except OSError: + return False + if not all( + path.is_relative_to(resolved_epoch) + for path in (init_path, segments_root, index_path, summary_path) + ): + return False + if ( + not init_path.is_file() + or init_path.stat().st_size <= 0 + or not segments_root.is_dir() + or not index_path.is_file() + or not 0 < index_path.stat().st_size <= MAX_MEDIA_INDEX_BYTES + or not summary_path.is_file() + ): + return False + summary = _read_json_object(summary_path) + if summary.get("schema_version") not in {1, "missioncore.camera-recording/v1"}: + return False + if summary.get("source_id") != expected_source_id: + return False + segment_count = _non_negative_int(summary.get("segment_count")) + if segment_count < 1: + return False + segments = [ + path.resolve() + for path in sorted(segments_root.iterdir()) + if path.is_file() and MEDIA_SEGMENT_PATTERN.fullmatch(path.name) + ] + if len(segments) != segment_count or any( + not path.is_relative_to(segments_root) or path.stat().st_size <= 0 for path in segments + ): + return False + try: + index_records = [ + json.loads(line) + for line in index_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return False + if len(index_records) != segment_count or not all( + isinstance(record, dict) + and _non_negative_int(record.get("sequence")) > 0 + for record in index_records + ): + return False + sequences = [int(record["sequence"]) for record in index_records] + return len(sequences) == len(set(sequences)) + + +def _media_epoch_bytes(epoch: Path) -> int: + try: + resolved_epoch = epoch.resolve(strict=True) + except OSError: + return 0 + total = 0 + for path in resolved_epoch.rglob("*"): + try: + resolved = path.resolve(strict=True) + except OSError: + continue + if resolved.is_file() and resolved.is_relative_to(resolved_epoch): + total += resolved.stat().st_size + return total diff --git a/src/k1link/sessions/media.py b/src/k1link/sessions/media.py new file mode 100644 index 0000000..2a41256 --- /dev/null +++ b/src/k1link/sessions/media.py @@ -0,0 +1,1460 @@ +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import secrets +import stat +import threading +from collections.abc import Iterator +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path +from typing import Any, TypeGuard + +from k1link.viewer.replay import MAX_METADATA_LINE_CHARS + +from .models import RecordedMediaArtifact, ReplayCommand, SessionIntegrityError + +CAMERA_ARCHIVE_SCHEMA = "missioncore.camera-recording/v1" +CAMERA_INDEX_SCHEMA = "missioncore.camera-recording-index/v1" +RECORDED_MEDIA_MANIFEST_SCHEMA = "missioncore.observation-recorded-media/v2" +RECORDED_MEDIA_PREPARATION_SCHEMA = "missioncore.recorded-media-preparation/v1" +MAX_MEDIA_INDEX_BYTES = 32 * 1024 * 1024 +MAX_INIT_BYTES = 8 * 1024 * 1024 +MAX_MEDIA_SEGMENTS = 500_000 +MAX_MEDIA_SEGMENT_BYTES = 64 * 1024 * 1024 +MAX_SAFE_INTEGER = (1 << 53) - 1 +MAX_MP4_BOXES = 100_000 +MAX_MP4_SAMPLES_PER_FRAGMENT = 1_000_000 +MAX_MP4_FRAGMENT_DURATION_SECONDS = 3_600.0 +MEDIA_RECORDING_TIMELINE_TOLERANCE_SECONDS = 0.05 +MAX_MEDIA_EPOCHS = 4_096 +MAX_PREPARED_MEDIA_SIDECAR_BYTES = 256 * 1024 * 1024 + +_EPOCH_PATTERN = re.compile(r"^epoch-([1-9][0-9]*)$") +_SHA256_PATTERN = re.compile(r"^[a-f0-9]{64}$") +_SIDECAR_NAME_PATTERN = re.compile(r"^[a-f0-9]{64}\.json$") +_SIDECAR_TEMP_PATTERN = re.compile(r"^\.tmp-[a-f0-9]{32}$") + + +@dataclass(frozen=True, slots=True) +class RecordedMediaSegment: + sequence: int + path: Path + byte_length: int + sha256: str + + +@dataclass(frozen=True, slots=True) +class RecordedMediaEpoch: + ordinal: int + path: Path + init_path: Path + init_byte_length: int + init_sha256: str + media_type: str + timeline_start_seconds: float + timeline_end_seconds: float + segments: tuple[RecordedMediaSegment, ...] + + +@dataclass(frozen=True, slots=True) +class RecordedMediaManifest: + session_id: str + public_source_id: str + artifact_id: str + synchronization: str + generation_sha256: str + timeline_start_seconds: float + timeline_end_seconds: float + byte_length: int + epochs: tuple[RecordedMediaEpoch, ...] + + +@dataclass(frozen=True, slots=True) +class RecordedMediaFile: + payload: bytes + media_type: str + byte_length: int + sha256: str + filename: str + + +@dataclass(frozen=True, slots=True) +class _CachedManifest: + identity: tuple[tuple[int, int, int, int], ...] + manifest: RecordedMediaManifest + + +@dataclass(frozen=True, slots=True) +class _Mp4VideoTiming: + track_id: int + timescale: int + default_sample_duration: int | None + + +@dataclass(slots=True) +class _Mp4ParseBudget: + boxes_remaining: int = MAX_MP4_BOXES + samples_remaining: int = MAX_MP4_SAMPLES_PER_FRAGMENT + + def consume_box(self) -> None: + self.boxes_remaining -= 1 + if self.boxes_remaining < 0: + raise SessionIntegrityError("recorded media ISO-BMFF box budget was exceeded") + + def consume_samples(self, count: int) -> None: + if count < 0 or count > self.samples_remaining: + raise SessionIntegrityError("recorded media ISO-BMFF sample budget was exceeded") + self.samples_remaining -= count + + +class RecordedMediaInspector: + """Validate canonical camera archives and retain only path-free descriptors. + + Cache keys are opaque catalog identities. A cache or persisted-sidecar hit + rechecks the full native and camera source stat identity without rereading + or parsing media; requested payload files are independently stat/hash + checked before they are served. + """ + + def __init__(self, cache_root: Path | None = None) -> None: + self._lock = threading.Lock() + self._cache: dict[tuple[str, str], _CachedManifest] = {} + self._cache_root = _prepare_sidecar_root(cache_root) + + def inspect( + self, + artifact: RecordedMediaArtifact, + replay: ReplayCommand, + ) -> RecordedMediaManifest: + if artifact.session_id != replay.session_id: + raise SessionIntegrityError("recorded media does not belong to replay session") + epoch_paths = _epoch_paths(artifact.source_path) + key = (artifact.session_id, artifact.artifact_id) + with self._lock: + cached = self._cache.get(key) + if cached is not None: + identity = _prepared_source_identity( + replay.source_path, + epoch_paths, + cached.manifest.epochs, + ) + if cached.identity == identity: + return cached.manifest + prepared = self._load_prepared_sidecar(artifact, replay, epoch_paths) + if prepared is not None: + manifest, identity = prepared + with self._lock: + self._cache[key] = _CachedManifest(identity=identity, manifest=manifest) + return manifest + origin_epoch_ns, origin_monotonic_ns = _raw_timeline_origin(replay.source_path) + manifest = _read_manifest( + artifact, + epoch_paths, + origin_epoch_ns=origin_epoch_ns, + origin_monotonic_ns=origin_monotonic_ns, + ) + identity = _prepared_source_identity( + replay.source_path, + epoch_paths, + manifest.epochs, + ) + self._publish_prepared_sidecar(manifest, identity) + with self._lock: + self._cache[key] = _CachedManifest(identity=identity, manifest=manifest) + return manifest + + def _load_prepared_sidecar( + self, + artifact: RecordedMediaArtifact, + replay: ReplayCommand, + epoch_paths: tuple[Path, ...], + ) -> tuple[ + RecordedMediaManifest, + tuple[tuple[int, int, int, int], ...], + ] | None: + root = self._cache_root + if root is None: + return None + path = root / _sidecar_name(artifact.session_id, artifact.artifact_id) + try: + payload = _read_confined_file(path, root, MAX_PREPARED_MEDIA_SIDECAR_BYTES) + document = _decode_prepared_sidecar(payload) + manifest = _manifest_from_sidecar(document, artifact, epoch_paths) + identity = _prepared_source_identity( + replay.source_path, + epoch_paths, + manifest.epochs, + ) + if _sidecar_identity(document, expected_length=len(identity)) != identity: + return None + return manifest, identity + except (OSError, SessionIntegrityError, UnicodeDecodeError, json.JSONDecodeError): + return None + + def _publish_prepared_sidecar( + self, + manifest: RecordedMediaManifest, + identity: tuple[tuple[int, int, int, int], ...], + ) -> None: + root = self._cache_root + if root is None: + return + payload = _encode_prepared_sidecar(manifest, identity) + _write_sidecar_atomic( + root, + _sidecar_name(manifest.session_id, manifest.artifact_id), + payload, + ) + + def get_prepared( + self, + session_id: str, + artifact_id: str, + ) -> RecordedMediaManifest | None: + """Return an already prepared manifest without touching its files.""" + + with self._lock: + cached = self._cache.get((session_id, artifact_id)) + return None if cached is None else cached.manifest + + def get_init(self, manifest: RecordedMediaManifest, ordinal: int) -> RecordedMediaFile: + epoch = _epoch_by_ordinal(manifest, ordinal) + return _validated_file( + epoch.init_path, + parent=epoch.path, + media_type=epoch.media_type, + expected_bytes=epoch.init_byte_length, + expected_sha256=epoch.init_sha256, + filename=f"recorded-camera-{ordinal}-init.mp4", + ) + + def get_segment( + self, + manifest: RecordedMediaManifest, + ordinal: int, + sequence: int, + ) -> RecordedMediaFile: + epoch = _epoch_by_ordinal(manifest, ordinal) + if sequence < 1 or sequence > len(epoch.segments): + raise SessionIntegrityError("recorded media segment is outside the manifest") + segment = epoch.segments[sequence - 1] + if segment.sequence != sequence: + raise SessionIntegrityError("recorded media segment sequence is inconsistent") + return _validated_file( + segment.path, + parent=epoch.path / "segments", + media_type="video/iso.segment", + expected_bytes=segment.byte_length, + expected_sha256=segment.sha256, + filename=f"recorded-camera-{ordinal}-{sequence}.m4s", + ) + + +def validate_recorded_media_timeline( + manifests: tuple[RecordedMediaManifest, ...], + *, + recording_start_seconds: float, + recording_end_seconds: float, +) -> None: + """Require camera coverage to be reachable by the replay-v2 RRD timeline.""" + + if ( + not math.isfinite(recording_start_seconds) + or not math.isfinite(recording_end_seconds) + or recording_start_seconds < 0 + or recording_end_seconds < recording_start_seconds + ): + raise SessionIntegrityError("recording timeline is invalid") + tolerance = MEDIA_RECORDING_TIMELINE_TOLERANCE_SECONDS + for manifest in manifests: + if ( + manifest.timeline_start_seconds < recording_start_seconds - tolerance + or manifest.timeline_end_seconds > recording_end_seconds + tolerance + ): + raise SessionIntegrityError( + "recorded media timeline is outside the spatial recording timeline" + ) + + +def _prepare_sidecar_root(cache_root: Path | None) -> Path | None: + if cache_root is None: + return None + root = cache_root.expanduser() + root.mkdir(mode=0o700, parents=True, exist_ok=True) + try: + metadata = root.lstat() + except OSError as exc: + raise ValueError("recorded media cache root is unavailable") from exc + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise ValueError("recorded media cache root must be a real directory") + resolved = root.resolve(strict=True) + for child in resolved.iterdir(): + if _SIDECAR_TEMP_PATTERN.fullmatch(child.name) is None: + continue + try: + child.unlink() + except OSError: + continue + return resolved + + +def _sidecar_name(session_id: str, artifact_id: str) -> str: + key = f"{session_id}\0{artifact_id}".encode() + return f"{hashlib.sha256(key).hexdigest()}.json" + + +def _encode_prepared_sidecar( + manifest: RecordedMediaManifest, + identity: tuple[tuple[int, int, int, int], ...], +) -> bytes: + body: dict[str, Any] = { + "schema_version": RECORDED_MEDIA_PREPARATION_SCHEMA, + "session_id": manifest.session_id, + "public_source_id": manifest.public_source_id, + "artifact_id": manifest.artifact_id, + "source_identity": [list(item) for item in identity], + "manifest": _sidecar_manifest_document(manifest), + } + checksum = hashlib.sha256(_canonical_json(body)).hexdigest() + return _canonical_json({**body, "checksum_sha256": checksum}) + + +def _decode_prepared_sidecar(payload: bytes) -> dict[str, Any]: + value = json.loads(payload) + if not isinstance(value, dict): + raise SessionIntegrityError("recorded media preparation is not an object") + checksum = value.get("checksum_sha256") + if not isinstance(checksum, str) or _SHA256_PATTERN.fullmatch(checksum) is None: + raise SessionIntegrityError("recorded media preparation checksum is invalid") + body = {key: item for key, item in value.items() if key != "checksum_sha256"} + if hashlib.sha256(_canonical_json(body)).hexdigest() != checksum: + raise SessionIntegrityError("recorded media preparation checksum changed") + if value.get("schema_version") != RECORDED_MEDIA_PREPARATION_SCHEMA: + raise SessionIntegrityError("recorded media preparation schema is incompatible") + return value + + +def _sidecar_manifest_document(manifest: RecordedMediaManifest) -> dict[str, Any]: + return { + "schema_version": RECORDED_MEDIA_MANIFEST_SCHEMA, + "source_id": manifest.public_source_id, + "artifact_id": manifest.artifact_id, + "generation_sha256": manifest.generation_sha256, + "synchronization": manifest.synchronization, + "timeline_start_seconds": manifest.timeline_start_seconds, + "timeline_end_seconds": manifest.timeline_end_seconds, + "byte_length": manifest.byte_length, + "epochs": [ + { + "ordinal": epoch.ordinal, + "timeline_start_seconds": epoch.timeline_start_seconds, + "timeline_end_seconds": epoch.timeline_end_seconds, + "media_type": epoch.media_type, + "init_byte_length": epoch.init_byte_length, + "init_sha256": epoch.init_sha256, + "segments": [ + { + "sequence": segment.sequence, + "byte_length": segment.byte_length, + "sha256": segment.sha256, + } + for segment in epoch.segments + ], + } + for epoch in manifest.epochs + ], + } + + +def _manifest_from_sidecar( + document: dict[str, Any], + artifact: RecordedMediaArtifact, + epoch_paths: tuple[Path, ...], +) -> RecordedMediaManifest: + if ( + document.get("session_id") != artifact.session_id + or document.get("public_source_id") != artifact.public_source_id + or document.get("artifact_id") != artifact.artifact_id + ): + raise SessionIntegrityError("recorded media preparation identity is inconsistent") + descriptor = document.get("manifest") + if not isinstance(descriptor, dict): + raise SessionIntegrityError("recorded media preparation has no manifest") + if ( + descriptor.get("schema_version") != RECORDED_MEDIA_MANIFEST_SCHEMA + or descriptor.get("source_id") != artifact.public_source_id + or descriptor.get("artifact_id") != artifact.artifact_id + or descriptor.get("synchronization") != "host-arrival-best-effort" + ): + raise SessionIntegrityError("recorded media prepared manifest is incompatible") + epoch_documents = descriptor.get("epochs") + if ( + not isinstance(epoch_documents, list) + or not epoch_documents + or len(epoch_documents) != len(epoch_paths) + or len(epoch_documents) > MAX_MEDIA_EPOCHS + ): + raise SessionIntegrityError("recorded media prepared epochs are invalid") + epochs = tuple( + _epoch_from_sidecar(epoch_document, epoch_path, ordinal) + for ordinal, (epoch_document, epoch_path) in enumerate( + zip(epoch_documents, epoch_paths, strict=True), + start=1, + ) + ) + _validate_epoch_timeline(epochs) + byte_length = sum( + epoch.init_byte_length + + sum(segment.byte_length for segment in epoch.segments) + for epoch in epochs + ) + timeline_start_seconds = min(epoch.timeline_start_seconds for epoch in epochs) + timeline_end_seconds = max(epoch.timeline_end_seconds for epoch in epochs) + if ( + descriptor.get("byte_length") != byte_length + or descriptor.get("timeline_start_seconds") != timeline_start_seconds + or descriptor.get("timeline_end_seconds") != timeline_end_seconds + or not 0 < byte_length <= MAX_SAFE_INTEGER + ): + raise SessionIntegrityError("recorded media prepared aggregate is inconsistent") + generation_sha256 = _manifest_generation_sha256( + public_source_id=artifact.public_source_id, + artifact_id=artifact.artifact_id, + synchronization="host-arrival-best-effort", + epochs=epochs, + ) + if ( + descriptor.get("generation_sha256") != generation_sha256 + or _SHA256_PATTERN.fullmatch(generation_sha256) is None + ): + raise SessionIntegrityError("recorded media prepared generation is inconsistent") + return RecordedMediaManifest( + session_id=artifact.session_id, + public_source_id=artifact.public_source_id, + artifact_id=artifact.artifact_id, + synchronization="host-arrival-best-effort", + generation_sha256=generation_sha256, + timeline_start_seconds=timeline_start_seconds, + timeline_end_seconds=timeline_end_seconds, + byte_length=byte_length, + epochs=epochs, + ) + + +def _epoch_from_sidecar( + value: object, + epoch_path: Path, + ordinal: int, +) -> RecordedMediaEpoch: + if not isinstance(value, dict) or value.get("ordinal") != ordinal: + raise SessionIntegrityError("recorded media prepared epoch is inconsistent") + start = value.get("timeline_start_seconds") + end = value.get("timeline_end_seconds") + media_type = value.get("media_type") + init_byte_length = value.get("init_byte_length") + init_sha256 = value.get("init_sha256") + if ( + not _finite_non_negative_number(start) + or not _finite_non_negative_number(end) + or float(end) < float(start) + or not isinstance(media_type, str) + or not media_type.startswith("video/mp4") + or len(media_type) > 256 + or not _positive_int(init_byte_length) + or int(init_byte_length) > MAX_INIT_BYTES + or not isinstance(init_sha256, str) + or _SHA256_PATTERN.fullmatch(init_sha256) is None + ): + raise SessionIntegrityError("recorded media prepared epoch descriptor is invalid") + segment_documents = value.get("segments") + if ( + not isinstance(segment_documents, list) + or not 1 <= len(segment_documents) <= MAX_MEDIA_SEGMENTS + ): + raise SessionIntegrityError("recorded media prepared segments are invalid") + segments_root = epoch_path / "segments" + segments: list[RecordedMediaSegment] = [] + for sequence, segment in enumerate(segment_documents, start=1): + if not isinstance(segment, dict) or segment.get("sequence") != sequence: + raise SessionIntegrityError("recorded media prepared segment is inconsistent") + byte_length = segment.get("byte_length") + sha256 = segment.get("sha256") + if ( + not _positive_int(byte_length) + or int(byte_length) > MAX_MEDIA_SEGMENT_BYTES + or not isinstance(sha256, str) + or _SHA256_PATTERN.fullmatch(sha256) is None + ): + raise SessionIntegrityError("recorded media prepared segment is invalid") + segments.append( + RecordedMediaSegment( + sequence=sequence, + path=segments_root / f"{sequence}.m4s", + byte_length=int(byte_length), + sha256=sha256, + ) + ) + return RecordedMediaEpoch( + ordinal=ordinal, + path=epoch_path, + init_path=epoch_path / "init.mp4", + init_byte_length=int(init_byte_length), + init_sha256=init_sha256, + media_type=media_type, + timeline_start_seconds=float(start), + timeline_end_seconds=float(end), + segments=tuple(segments), + ) + + +def _sidecar_identity( + document: dict[str, Any], + *, + expected_length: int, +) -> tuple[tuple[int, int, int, int], ...]: + value = document.get("source_identity") + if ( + not isinstance(value, list) + or expected_length < 1 + or len(value) != expected_length + ): + raise SessionIntegrityError("recorded media preparation source identity is invalid") + identity: list[tuple[int, int, int, int]] = [] + for item in value: + if ( + not isinstance(item, list) + or len(item) != 4 + or any(not _non_negative_int(component) for component in item) + ): + raise SessionIntegrityError("recorded media preparation source identity is invalid") + identity.append((int(item[0]), int(item[1]), int(item[2]), int(item[3]))) + return tuple(identity) + + +def _canonical_json(value: object) -> bytes: + try: + encoded = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise SessionIntegrityError("recorded media preparation cannot be encoded") from exc + if not 0 < len(encoded) <= MAX_PREPARED_MEDIA_SIDECAR_BYTES: + raise SessionIntegrityError("recorded media preparation is outside bounds") + return encoded + + +def _write_sidecar_atomic(root: Path, filename: str, payload: bytes) -> None: + if _SIDECAR_NAME_PATTERN.fullmatch(filename) is None: + raise SessionIntegrityError("recorded media preparation filename is invalid") + directory_fd = os.open( + root, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + temporary = f".tmp-{secrets.token_hex(16)}" + descriptor: int | None = None + try: + descriptor = os.open( + temporary, + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_NOFOLLOW", 0), + 0o600, + dir_fd=directory_fd, + ) + remaining = memoryview(payload) + while remaining: + written = os.write(descriptor, remaining) + if written <= 0: + raise OSError("prepared media sidecar write made no progress") + remaining = remaining[written:] + os.fsync(descriptor) + os.close(descriptor) + descriptor = None + os.replace( + temporary, + filename, + src_dir_fd=directory_fd, + dst_dir_fd=directory_fd, + ) + os.fsync(directory_fd) + finally: + if descriptor is not None: + os.close(descriptor) + with suppress(FileNotFoundError): + os.unlink(temporary, dir_fd=directory_fd) + os.close(directory_fd) + + +def _finite_non_negative_number(value: object) -> TypeGuard[int | float]: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(float(value)) + and float(value) >= 0 + ) + + +def _epoch_paths(source_path: Path) -> tuple[Path, ...]: + try: + source = source_path.resolve(strict=True) + except OSError as exc: + raise SessionIntegrityError("recorded media source is missing") from exc + if not source.is_dir(): + raise SessionIntegrityError("recorded media source is not a directory") + epochs: list[tuple[int, Path]] = [] + for child in source.iterdir(): + match = _EPOCH_PATTERN.fullmatch(child.name) + if match is None or not child.is_dir(): + continue + try: + resolved = child.resolve(strict=True) + except OSError as exc: + raise SessionIntegrityError("recorded media epoch is unavailable") from exc + if resolved.parent != source: + raise SessionIntegrityError("recorded media epoch escapes its source") + epochs.append((int(match.group(1)), resolved)) + if not epochs: + raise SessionIntegrityError("recorded media source has no codec epochs") + if len(epochs) > MAX_MEDIA_EPOCHS: + raise SessionIntegrityError("recorded media source has too many codec epochs") + return tuple(path for _, path in sorted(epochs)) + + +def _prepared_source_identity( + raw_path: Path, + epoch_paths: tuple[Path, ...], + epochs: tuple[RecordedMediaEpoch, ...], +) -> tuple[tuple[int, int, int, int], ...]: + if len(epoch_paths) != len(epochs): + raise SessionIntegrityError("recorded media epoch identity is inconsistent") + identity: list[tuple[int, int, int, int]] = [] + for source_file in (raw_path, raw_path.with_name("mqtt.metadata.jsonl")): + metadata = _confined_file_stat(source_file, raw_path.parent) + identity.append( + (metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mtime_ns) + ) + for expected_ordinal, (epoch_path, epoch) in enumerate( + zip(epoch_paths, epochs, strict=True), + start=1, + ): + if epoch.ordinal != expected_ordinal or epoch.path != epoch_path: + raise SessionIntegrityError("recorded media epoch identity is inconsistent") + for filename in ("summary.json", "index.jsonl", "init.mp4"): + metadata = _confined_file_stat(epoch_path / filename, epoch_path) + identity.append( + (metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mtime_ns) + ) + try: + segments_root = (epoch_path / "segments").resolve(strict=True) + except OSError as exc: + raise SessionIntegrityError("recorded media segment directory is missing") from exc + if not segments_root.is_dir() or segments_root.parent != epoch_path: + raise SessionIntegrityError("recorded media segment directory is invalid") + for segment in epoch.segments: + metadata = _confined_file_stat( + segments_root / f"{segment.sequence}.m4s", + segments_root, + ) + identity.append( + (metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mtime_ns) + ) + return tuple(identity) + + +def _raw_timeline_origin(raw_path: Path) -> tuple[int, int]: + metadata_path = raw_path.with_name("mqtt.metadata.jsonl") + try: + line = _read_first_confined_line( + metadata_path, + raw_path.parent, + MAX_METADATA_LINE_CHARS, + ).decode("utf-8") + except (OSError, UnicodeDecodeError) as exc: + raise SessionIntegrityError("native capture timing metadata is unavailable") from exc + if not line or len(line) > MAX_METADATA_LINE_CHARS or not line.endswith(("\n", "\r")): + raise SessionIntegrityError("native capture timing origin is incomplete") + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise SessionIntegrityError("native capture timing origin is invalid") from exc + epoch_ns = record.get("received_at_epoch_ns") if isinstance(record, dict) else None + monotonic_ns = record.get("received_monotonic_ns") if isinstance(record, dict) else None + if ( + record.get("record_type") != "message" + or record.get("sequence") != 1 + or not _non_negative_int(epoch_ns) + or not _non_negative_int(monotonic_ns) + ): + raise SessionIntegrityError("native capture timing origin is invalid") + return int(epoch_ns), int(monotonic_ns) + + +def _read_manifest( + artifact: RecordedMediaArtifact, + epoch_paths: tuple[Path, ...], + *, + origin_epoch_ns: int, + origin_monotonic_ns: int, +) -> RecordedMediaManifest: + epochs = tuple( + _read_epoch( + path, + ordinal=index, + expected_source_name=artifact.source_path.name, + origin_epoch_ns=origin_epoch_ns, + origin_monotonic_ns=origin_monotonic_ns, + ) + for index, path in enumerate(epoch_paths, start=1) + ) + _validate_epoch_timeline(epochs) + byte_length = sum( + epoch.init_byte_length + + sum(segment.byte_length for segment in epoch.segments) + for epoch in epochs + ) + if not 0 < byte_length <= MAX_SAFE_INTEGER: + raise SessionIntegrityError("recorded media byte length is outside safe bounds") + timeline_start_seconds = min(epoch.timeline_start_seconds for epoch in epochs) + timeline_end_seconds = max(epoch.timeline_end_seconds for epoch in epochs) + generation_sha256 = _manifest_generation_sha256( + public_source_id=artifact.public_source_id, + artifact_id=artifact.artifact_id, + synchronization="host-arrival-best-effort", + epochs=epochs, + ) + return RecordedMediaManifest( + session_id=artifact.session_id, + public_source_id=artifact.public_source_id, + artifact_id=artifact.artifact_id, + synchronization="host-arrival-best-effort", + generation_sha256=generation_sha256, + timeline_start_seconds=timeline_start_seconds, + timeline_end_seconds=timeline_end_seconds, + byte_length=byte_length, + epochs=epochs, + ) + + +def _manifest_generation_sha256( + *, + public_source_id: str, + artifact_id: str, + synchronization: str, + epochs: tuple[RecordedMediaEpoch, ...], +) -> str: + """Digest the immutable, path-free descriptor generation.""" + + timeline_start_seconds = min(epoch.timeline_start_seconds for epoch in epochs) + timeline_end_seconds = max(epoch.timeline_end_seconds for epoch in epochs) + byte_length = sum( + epoch.init_byte_length + + sum(segment.byte_length for segment in epoch.segments) + for epoch in epochs + ) + descriptor = { + "schema_version": RECORDED_MEDIA_MANIFEST_SCHEMA, + "source_id": public_source_id, + "artifact_id": artifact_id, + "synchronization": synchronization, + "timeline_start_seconds": timeline_start_seconds, + "timeline_end_seconds": timeline_end_seconds, + "byte_length": byte_length, + "epochs": [ + { + "ordinal": epoch.ordinal, + "timeline_start_seconds": epoch.timeline_start_seconds, + "timeline_end_seconds": epoch.timeline_end_seconds, + "media_type": epoch.media_type, + "init_byte_length": epoch.init_byte_length, + "init_sha256": epoch.init_sha256, + "segments": [ + { + "sequence": segment.sequence, + "byte_length": segment.byte_length, + "sha256": segment.sha256, + } + for segment in epoch.segments + ], + } + for epoch in epochs + ], + } + payload = json.dumps( + descriptor, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _read_epoch( + epoch: Path, + *, + ordinal: int, + expected_source_name: str, + origin_epoch_ns: int, + origin_monotonic_ns: int, +) -> RecordedMediaEpoch: + summary = _read_json_object(epoch / "summary.json", MAX_MEDIA_INDEX_BYTES) + if ( + summary.get("schema_version") != CAMERA_ARCHIVE_SCHEMA + or summary.get("source_id") != expected_source_name + or summary.get("synchronization") != "host-arrival-best-effort" + ): + raise SessionIntegrityError("recorded media summary is incompatible") + segment_count = summary.get("segment_count") + if not _non_negative_int(segment_count) or not 1 <= int(segment_count) <= MAX_MEDIA_SEGMENTS: + raise SessionIntegrityError("recorded media segment count is invalid") + + try: + raw_lines = _read_confined_file( + epoch / "index.jsonl", + epoch, + MAX_MEDIA_INDEX_BYTES, + ).decode("utf-8").splitlines() + except UnicodeDecodeError as exc: + raise SessionIntegrityError("recorded media index is unavailable") from exc + if len(raw_lines) != int(segment_count): + raise SessionIntegrityError("recorded media index length does not match its summary") + entries: list[dict[str, Any]] = [] + for sequence, line in enumerate(raw_lines, start=1): + try: + entry = json.loads(line) + except json.JSONDecodeError as exc: + raise SessionIntegrityError("recorded media index contains invalid JSON") from exc + if not isinstance(entry, dict): + raise SessionIntegrityError("recorded media index entry is not an object") + if ( + entry.get("schema_version") != CAMERA_INDEX_SCHEMA + or entry.get("sequence") != sequence + or entry.get("kind") != "media" + or entry.get("path") != f"segments/{sequence}.m4s" + ): + raise SessionIntegrityError("recorded media index entry is inconsistent") + entries.append(entry) + + use_monotonic_clock = all( + _non_negative_int(entry.get("host_monotonic_ns")) for entry in entries + ) + timeline_points = tuple( + max( + 0.0, + _entry_session_seconds( + entry, + origin_epoch_ns=origin_epoch_ns, + origin_monotonic_ns=origin_monotonic_ns, + use_monotonic_clock=use_monotonic_clock, + ), + ) + for entry in entries + ) + if any(not math.isfinite(point) for point in timeline_points): + raise SessionIntegrityError("recorded media timeline contains a non-finite value") + if any( + current >= following + for current, following in zip( + timeline_points, + timeline_points[1:], + strict=False, + ) + ): + raise SessionIntegrityError("recorded media segment timeline is not strictly monotonic") + segments_root = (epoch / "segments").resolve(strict=True) + if not segments_root.is_dir() or segments_root.parent != epoch: + raise SessionIntegrityError("recorded media segment directory is invalid") + segments: list[RecordedMediaSegment] = [] + for sequence, entry in enumerate(entries, start=1): + byte_length = entry.get("length") + digest = entry.get("sha256") + if ( + not _positive_int(byte_length) + or int(byte_length) > MAX_MEDIA_SEGMENT_BYTES + or not isinstance(digest, str) + or _SHA256_PATTERN.fullmatch(digest) is None + ): + raise SessionIntegrityError("recorded media segment descriptor is invalid") + path = (segments_root / f"{sequence}.m4s").resolve() + metadata = _confined_file_stat( + segments_root / f"{sequence}.m4s", + segments_root, + ) + if metadata.st_size <= 0 or metadata.st_size > MAX_MEDIA_SEGMENT_BYTES: + raise SessionIntegrityError("recorded media segment length is outside bounds") + if metadata.st_size != int(byte_length): + raise SessionIntegrityError("recorded media segment length changed") + segments.append( + RecordedMediaSegment( + sequence=sequence, + path=path, + byte_length=int(byte_length), + sha256=digest, + ) + ) + + init_path = (epoch / "init.mp4").resolve() + init_payload = _read_confined_file(init_path, epoch, MAX_INIT_BYTES) + init_sha256 = hashlib.sha256(init_payload).hexdigest() + expected_init_sha256 = summary.get("init_sha256") + if ( + not isinstance(expected_init_sha256, str) + or expected_init_sha256 != init_sha256 + ): + raise SessionIntegrityError("recorded media init digest changed") + media_type = _mp4_media_type(init_payload) + timing = _mp4_video_timing(init_payload, _Mp4ParseBudget()) + duration_units: list[int] = [] + for segment in segments: + payload = _read_confined_file( + segment.path, + segments_root, + segment.byte_length, + ) + if hashlib.sha256(payload).hexdigest() != segment.sha256: + raise SessionIntegrityError("recorded media segment digest changed") + duration_units.append( + _mp4_video_fragment_duration_units( + payload, + timing, + _Mp4ParseBudget(), + ) + ) + first_duration_seconds = _checked_fragment_duration_seconds( + duration_units[0], + timing.timescale, + ) + total_duration_units = sum(duration_units) + if total_duration_units > MAX_SAFE_INTEGER: + raise SessionIntegrityError("recorded media epoch duration is outside safe bounds") + total_duration_seconds = total_duration_units / timing.timescale + timeline_start_seconds = max(0.0, timeline_points[0] - first_duration_seconds) + timeline_end_seconds = timeline_start_seconds + total_duration_seconds + if ( + not math.isfinite(timeline_end_seconds) + or timeline_end_seconds < timeline_start_seconds + ): + raise SessionIntegrityError("recorded media epoch timeline is invalid") + return RecordedMediaEpoch( + ordinal=ordinal, + path=epoch, + init_path=init_path, + init_byte_length=len(init_payload), + init_sha256=init_sha256, + media_type=media_type, + timeline_start_seconds=timeline_start_seconds, + timeline_end_seconds=timeline_end_seconds, + segments=tuple(segments), + ) + + +def _entry_session_seconds( + entry: dict[str, Any], + *, + origin_epoch_ns: int, + origin_monotonic_ns: int, + use_monotonic_clock: bool, +) -> float: + monotonic_ns = entry.get("host_monotonic_ns") + if use_monotonic_clock and _non_negative_int(monotonic_ns): + try: + return (int(monotonic_ns) - origin_monotonic_ns) / 1_000_000_000 + except OverflowError as exc: + raise SessionIntegrityError("recorded media timeline is outside bounds") from exc + epoch_ns = entry.get("host_epoch_ns") + if _non_negative_int(epoch_ns): + try: + return (int(epoch_ns) - origin_epoch_ns) / 1_000_000_000 + except OverflowError as exc: + raise SessionIntegrityError("recorded media timeline is outside bounds") from exc + raise SessionIntegrityError("recorded media has no host-arrival synchronization point") + + +def _validate_epoch_timeline(epochs: tuple[RecordedMediaEpoch, ...]) -> None: + previous_end: float | None = None + for expected_ordinal, epoch in enumerate(epochs, start=1): + start = epoch.timeline_start_seconds + end = epoch.timeline_end_seconds + if epoch.ordinal != expected_ordinal: + raise SessionIntegrityError("recorded media epoch ordinal is inconsistent") + if not math.isfinite(start) or not math.isfinite(end) or start < 0 or end < start: + raise SessionIntegrityError("recorded media epoch timeline is invalid") + if previous_end is not None and start < previous_end: + raise SessionIntegrityError("recorded media codec epochs overlap") + previous_end = end + + +def _mp4_fragment_duration_seconds(init_payload: bytes, fragment_payload: bytes) -> float: + """Return the bounded decoded duration of one ISO-BMFF video fragment. + + Camera timeline anchors are complete-fragment host-arrival observations. + This helper proves the exact decoded duration declared by one fragment. If + any timing field is absent or ambiguous the archive is not advertised as + replayable. + """ + + budget = _Mp4ParseBudget() + timing = _mp4_video_timing(init_payload, budget) + duration_units = _mp4_video_fragment_duration_units( + fragment_payload, + timing, + budget, + ) + return _checked_fragment_duration_seconds(duration_units, timing.timescale) + + +def _checked_fragment_duration_seconds(duration_units: int, timescale: int) -> float: + if duration_units <= 0: + raise SessionIntegrityError("recorded media fragment has no positive duration") + duration = duration_units / timescale + if ( + not math.isfinite(duration) + or duration <= 0 + or duration > MAX_MP4_FRAGMENT_DURATION_SECONDS + ): + raise SessionIntegrityError("recorded media fragment duration is outside bounds") + return duration + + +def _mp4_video_timing(payload: bytes, budget: _Mp4ParseBudget) -> _Mp4VideoTiming: + moov_payloads = [ + box_payload + for box_type, box_payload in _iter_mp4_boxes(payload, budget) + if box_type == b"moov" + ] + if len(moov_payloads) != 1: + raise SessionIntegrityError("recorded media init has no unique moov box") + moov = moov_payloads[0] + moov_boxes = tuple(_iter_mp4_boxes(moov, budget)) + defaults: dict[int, int] = {} + for box_type, box_payload in moov_boxes: + if box_type != b"mvex": + continue + for child_type, child_payload in _iter_mp4_boxes(box_payload, budget): + if child_type != b"trex": + continue + track_id, trex_default_duration = _parse_trex(child_payload) + if track_id in defaults: + raise SessionIntegrityError("recorded media init repeats a trex track") + defaults[track_id] = trex_default_duration + + video_tracks: list[tuple[int, int]] = [] + for box_type, trak_payload in moov_boxes: + if box_type != b"trak": + continue + track_id = _trak_track_id(trak_payload, budget) + media = _trak_media_timing(trak_payload, budget) + if media is not None: + video_tracks.append((track_id, media)) + if len(video_tracks) != 1: + raise SessionIntegrityError("recorded media init has no unique video track") + track_id, timescale = video_tracks[0] + default_duration = defaults.get(track_id) + return _Mp4VideoTiming( + track_id=track_id, + timescale=timescale, + default_sample_duration=( + default_duration if default_duration is not None and default_duration > 0 else None + ), + ) + + +def _trak_track_id(payload: bytes, budget: _Mp4ParseBudget) -> int: + track_ids = [ + _parse_tkhd_track_id(box_payload) + for box_type, box_payload in _iter_mp4_boxes(payload, budget) + if box_type == b"tkhd" + ] + if len(track_ids) != 1 or track_ids[0] <= 0: + raise SessionIntegrityError("recorded media track id is invalid") + return track_ids[0] + + +def _trak_media_timing(payload: bytes, budget: _Mp4ParseBudget) -> int | None: + media_boxes = [ + box_payload + for box_type, box_payload in _iter_mp4_boxes(payload, budget) + if box_type == b"mdia" + ] + if len(media_boxes) != 1: + raise SessionIntegrityError("recorded media track has no unique mdia box") + children = tuple(_iter_mp4_boxes(media_boxes[0], budget)) + handlers = [ + _parse_hdlr_type(box_payload) + for box_type, box_payload in children + if box_type == b"hdlr" + ] + if len(handlers) != 1: + raise SessionIntegrityError("recorded media track handler is ambiguous") + if handlers[0] != b"vide": + return None + timescales = [ + _parse_mdhd_timescale(box_payload) + for box_type, box_payload in children + if box_type == b"mdhd" + ] + if len(timescales) != 1 or timescales[0] <= 0: + raise SessionIntegrityError("recorded media video timescale is invalid") + return timescales[0] + + +def _mp4_video_fragment_duration_units( + payload: bytes, + timing: _Mp4VideoTiming, + budget: _Mp4ParseBudget, +) -> int: + moof_payloads = [ + box_payload + for box_type, box_payload in _iter_mp4_boxes(payload, budget) + if box_type == b"moof" + ] + if len(moof_payloads) != 1: + raise SessionIntegrityError("recorded media fragment has no unique moof box") + matching_durations: list[int] = [] + for box_type, traf_payload in _iter_mp4_boxes(moof_payloads[0], budget): + if box_type != b"traf": + continue + boxes = tuple(_iter_mp4_boxes(traf_payload, budget)) + tfhd_payloads = [box for kind, box in boxes if kind == b"tfhd"] + if len(tfhd_payloads) != 1: + raise SessionIntegrityError("recorded media fragment tfhd is ambiguous") + track_id, fragment_default_duration = _parse_tfhd(tfhd_payloads[0]) + if track_id != timing.track_id: + continue + trun_payloads = [box for kind, box in boxes if kind == b"trun"] + if not trun_payloads: + raise SessionIntegrityError("recorded media video fragment has no trun box") + default_duration = fragment_default_duration or timing.default_sample_duration + duration = sum( + _parse_trun_duration_units(trun, default_duration, budget) + for trun in trun_payloads + ) + matching_durations.append(duration) + if len(matching_durations) != 1: + raise SessionIntegrityError("recorded media fragment video track is ambiguous") + return matching_durations[0] + + +def _iter_mp4_boxes( + payload: bytes, + budget: _Mp4ParseBudget, +) -> Iterator[tuple[bytes, bytes]]: + offset = 0 + payload_length = len(payload) + while offset < payload_length: + if payload_length - offset < 8: + raise SessionIntegrityError("recorded media ISO-BMFF box is truncated") + size = int.from_bytes(payload[offset : offset + 4], "big") + box_type = payload[offset + 4 : offset + 8] + header_length = 8 + if size == 1: + if payload_length - offset < 16: + raise SessionIntegrityError("recorded media ISO-BMFF box is truncated") + size = int.from_bytes(payload[offset + 8 : offset + 16], "big") + header_length = 16 + elif size == 0: + size = payload_length - offset + if size < header_length or size > payload_length - offset: + raise SessionIntegrityError("recorded media ISO-BMFF box size is invalid") + budget.consume_box() + end = offset + size + yield box_type, payload[offset + header_length : end] + offset = end + + +def _parse_tkhd_track_id(payload: bytes) -> int: + version = _full_box_version(payload) + offset = 20 if version == 1 else 12 if version == 0 else -1 + return _read_u32(payload, offset, "tkhd track id") + + +def _parse_mdhd_timescale(payload: bytes) -> int: + version = _full_box_version(payload) + offset = 20 if version == 1 else 12 if version == 0 else -1 + return _read_u32(payload, offset, "mdhd timescale") + + +def _parse_hdlr_type(payload: bytes) -> bytes: + _full_box_version(payload) + if len(payload) < 12: + raise SessionIntegrityError("recorded media hdlr box is truncated") + return payload[8:12] + + +def _parse_trex(payload: bytes) -> tuple[int, int]: + _full_box_version(payload) + return ( + _read_u32(payload, 4, "trex track id"), + _read_u32(payload, 12, "trex default sample duration"), + ) + + +def _parse_tfhd(payload: bytes) -> tuple[int, int | None]: + flags = _full_box_flags(payload) + track_id = _read_u32(payload, 4, "tfhd track id") + cursor = 8 + for flag, width in ((0x000001, 8), (0x000002, 4)): + if flags & flag: + cursor = _advance_box_cursor(payload, cursor, width, "tfhd optional field") + default_duration: int | None = None + if flags & 0x000008: + default_duration = _read_u32(payload, cursor, "tfhd default sample duration") + cursor += 4 + for flag in (0x000010, 0x000020): + if flags & flag: + cursor = _advance_box_cursor(payload, cursor, 4, "tfhd optional field") + return track_id, default_duration if default_duration and default_duration > 0 else None + + +def _parse_trun_duration_units( + payload: bytes, + default_duration: int | None, + budget: _Mp4ParseBudget, +) -> int: + flags = _full_box_flags(payload) + sample_count = _read_u32(payload, 4, "trun sample count") + if sample_count < 1: + raise SessionIntegrityError("recorded media trun has no samples") + budget.consume_samples(sample_count) + cursor = 8 + if flags & 0x000001: + cursor = _advance_box_cursor(payload, cursor, 4, "trun data offset") + if flags & 0x000004: + cursor = _advance_box_cursor(payload, cursor, 4, "trun first sample flags") + per_sample_width = sum( + 4 for flag in (0x000100, 0x000200, 0x000400, 0x000800) if flags & flag + ) + if per_sample_width and sample_count > (len(payload) - cursor) // per_sample_width: + raise SessionIntegrityError("recorded media trun samples are truncated") + if flags & 0x000100: + duration = 0 + for _ in range(sample_count): + sample_duration = _read_u32(payload, cursor, "trun sample duration") + if sample_duration <= 0: + raise SessionIntegrityError("recorded media sample duration is invalid") + duration += sample_duration + cursor += per_sample_width + return duration + if default_duration is None or default_duration <= 0: + raise SessionIntegrityError("recorded media sample duration is unavailable") + if per_sample_width: + _advance_box_cursor( + payload, + cursor, + sample_count * per_sample_width, + "trun sample table", + ) + return sample_count * default_duration + + +def _full_box_version(payload: bytes) -> int: + if len(payload) < 4: + raise SessionIntegrityError("recorded media full box is truncated") + return payload[0] + + +def _full_box_flags(payload: bytes) -> int: + _full_box_version(payload) + return int.from_bytes(payload[1:4], "big") + + +def _read_u32(payload: bytes, offset: int, description: str) -> int: + if offset < 0 or offset + 4 > len(payload): + raise SessionIntegrityError(f"recorded media {description} is truncated") + return int.from_bytes(payload[offset : offset + 4], "big") + + +def _advance_box_cursor(payload: bytes, cursor: int, width: int, description: str) -> int: + if cursor < 0 or width < 0 or cursor + width > len(payload): + raise SessionIntegrityError(f"recorded media {description} is truncated") + return cursor + width + + +def _mp4_media_type(init_payload: bytes) -> str: + marker = init_payload.find(b"avcC") + if marker >= 0 and marker + 8 <= len(init_payload): + configuration_version = init_payload[marker + 4] + if configuration_version == 1: + profile = init_payload[marker + 5] + compatibility = init_payload[marker + 6] + level = init_payload[marker + 7] + return f'video/mp4; codecs="avc1.{profile:02X}{compatibility:02X}{level:02X}"' + # The archive is still valid evidence, but browser MSE cannot be opened + # safely without an explicit codec string. + return "video/mp4" + + +def _epoch_by_ordinal( + manifest: RecordedMediaManifest, + ordinal: int, +) -> RecordedMediaEpoch: + if ordinal < 1 or ordinal > len(manifest.epochs): + raise SessionIntegrityError("recorded media epoch is outside the manifest") + epoch = manifest.epochs[ordinal - 1] + if epoch.ordinal != ordinal: + raise SessionIntegrityError("recorded media epoch ordinal is inconsistent") + return epoch + + +def _validated_file( + path: Path, + *, + parent: Path, + media_type: str, + expected_bytes: int, + expected_sha256: str, + filename: str, +) -> RecordedMediaFile: + payload = _read_confined_file(path, parent, max(expected_bytes, 1)) + if len(payload) != expected_bytes or hashlib.sha256(payload).hexdigest() != expected_sha256: + raise SessionIntegrityError("recorded media payload changed after indexing") + return RecordedMediaFile( + payload=payload, + media_type=media_type, + byte_length=expected_bytes, + sha256=expected_sha256, + filename=filename, + ) + + +def _confined_file_stat(path: Path, parent: Path) -> os.stat_result: + try: + resolved_parent = parent.resolve(strict=True) + if path.parent.resolve(strict=True) != resolved_parent: + raise SessionIntegrityError("recorded media payload escapes its epoch") + parent_fd = os.open( + resolved_parent, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + except OSError as exc: + raise SessionIntegrityError("recorded media payload is missing") from exc + try: + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path.name, flags, dir_fd=parent_fd) + try: + stat_result = os.fstat(descriptor) + finally: + os.close(descriptor) + except OSError as exc: + raise SessionIntegrityError("recorded media payload is not a regular file") from exc + finally: + os.close(parent_fd) + if not stat.S_ISREG(stat_result.st_mode): + raise SessionIntegrityError("recorded media payload is not a regular file") + return stat_result + + +def _read_confined_file(path: Path, parent: Path, maximum_bytes: int) -> bytes: + try: + resolved_parent = parent.resolve(strict=True) + if path.parent.resolve(strict=True) != resolved_parent: + raise SessionIntegrityError("recorded media payload escapes its epoch") + parent_fd = os.open( + resolved_parent, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + except OSError as exc: + raise SessionIntegrityError("recorded media payload is missing") from exc + try: + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path.name, flags, dir_fd=parent_fd) + try: + stat_result = os.fstat(descriptor) + if ( + not stat.S_ISREG(stat_result.st_mode) + or not 0 < stat_result.st_size <= maximum_bytes + ): + raise SessionIntegrityError("recorded media payload is outside bounds") + chunks: list[bytes] = [] + remaining = stat_result.st_size + while remaining: + chunk = os.read(descriptor, min(remaining, 1024 * 1024)) + if not chunk: + raise SessionIntegrityError("recorded media payload was truncated") + chunks.append(chunk) + remaining -= len(chunk) + if os.read(descriptor, 1): + raise SessionIntegrityError("recorded media payload grew during validation") + after = os.fstat(descriptor) + if ( + after.st_dev != stat_result.st_dev + or after.st_ino != stat_result.st_ino + or after.st_size != stat_result.st_size + or after.st_mtime_ns != stat_result.st_mtime_ns + ): + raise SessionIntegrityError("recorded media payload changed during validation") + return b"".join(chunks) + finally: + os.close(descriptor) + except OSError as exc: + raise SessionIntegrityError("recorded media payload is unavailable") from exc + finally: + os.close(parent_fd) + + +def _read_first_confined_line(path: Path, parent: Path, maximum_bytes: int) -> bytes: + try: + resolved_parent = parent.resolve(strict=True) + if path.parent.resolve(strict=True) != resolved_parent: + raise SessionIntegrityError("recorded media timeline escapes its session") + parent_fd = os.open( + resolved_parent, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + except OSError as exc: + raise SessionIntegrityError("recorded media timeline is missing") from exc + try: + descriptor = os.open( + path.name, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + dir_fd=parent_fd, + ) + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise SessionIntegrityError("recorded media timeline is not regular") + payload = bytearray() + while len(payload) <= maximum_bytes: + chunk = os.read(descriptor, min(64 * 1024, maximum_bytes + 1 - len(payload))) + if not chunk: + break + newline = chunk.find(b"\n") + if newline >= 0: + payload.extend(chunk[: newline + 1]) + break + payload.extend(chunk) + if len(payload) > maximum_bytes: + raise SessionIntegrityError("recorded media timeline line exceeds its boundary") + return bytes(payload) + finally: + os.close(descriptor) + except OSError as exc: + raise SessionIntegrityError("recorded media timeline is unavailable") from exc + finally: + os.close(parent_fd) + + +def _read_json_object(path: Path, maximum_bytes: int) -> dict[str, Any]: + try: + value = json.loads(_read_confined_file(path, path.parent, maximum_bytes)) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise SessionIntegrityError("recorded media summary is invalid") from exc + if not isinstance(value, dict): + raise SessionIntegrityError("recorded media summary is not an object") + return value + + +def _non_negative_int(value: object) -> TypeGuard[int]: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def _positive_int(value: object) -> TypeGuard[int]: + return isinstance(value, int) and not isinstance(value, bool) and value > 0 diff --git a/src/k1link/sessions/models.py b/src/k1link/sessions/models.py new file mode 100644 index 0000000..18e4379 --- /dev/null +++ b/src/k1link/sessions/models.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +SessionStatus = Literal["ready", "interrupted", "failed"] +SessionModality = Literal["point-cloud", "trajectory", "video"] + + +class SessionStoreError(RuntimeError): + """Base error for the host-owned observation session store.""" + + +class SessionNotFoundError(SessionStoreError): + """The requested opaque session identifier is not registered.""" + + +class SessionNotReplayableError(SessionStoreError): + """The session has no reviewed replay source.""" + + +class SessionIntegrityError(SessionStoreError): + """A stored artifact no longer satisfies its confinement/integrity boundary.""" + + +class LayoutConflictError(SessionStoreError): + """A workspace layout revision changed since the caller loaded it.""" + + +@dataclass(frozen=True, slots=True) +class SessionSource: + source_id: str + semantic_channel_id: str + modality: SessionModality + status: str + seekable: bool + artifact_id: str + + def as_dict(self) -> dict[str, Any]: + return { + "source_id": self.source_id, + "semantic_channel_id": self.semantic_channel_id, + "modality": self.modality, + "status": self.status, + "seekable": self.seekable, + "artifact_id": self.artifact_id, + } + + +@dataclass(frozen=True, slots=True) +class SessionArtifact: + artifact_id: str + kind: str + media_type: str + byte_length: int + sha256: str | None + integrity_status: str + + def as_dict(self) -> dict[str, Any]: + return { + "artifact_id": self.artifact_id, + "kind": self.kind, + "media_type": self.media_type, + "byte_length": self.byte_length, + "sha256": self.sha256, + "integrity_status": self.integrity_status, + } + + +@dataclass(frozen=True, slots=True) +class SessionSummary: + session_id: str + display_name: str + status: SessionStatus + started_at_utc: str | None + completed_at_utc: str | None + duration_seconds: float | None + modalities: tuple[SessionModality, ...] + source_count: int + total_bytes: int + replayable: bool + origin: str + + def as_dict(self) -> dict[str, Any]: + return { + "schema_version": "missioncore.observation-session-summary/v1", + "session_id": self.session_id, + "display_name": self.display_name, + "status": self.status, + "started_at_utc": self.started_at_utc, + "completed_at_utc": self.completed_at_utc, + "duration_seconds": self.duration_seconds, + "modalities": list(self.modalities), + "source_count": self.source_count, + "total_bytes": self.total_bytes, + "replayable": self.replayable, + "origin": self.origin, + } + + +@dataclass(frozen=True, slots=True) +class SessionDetail: + summary: SessionSummary + sources: tuple[SessionSource, ...] + artifacts: tuple[SessionArtifact, ...] + + def as_dict(self) -> dict[str, Any]: + duration = self.summary.duration_seconds + return { + "schema_version": "missioncore.observation-session/v1", + **{ + key: value + for key, value in self.summary.as_dict().items() + if key != "schema_version" + }, + "timeline": { + "mode": "recorded" if self.summary.replayable else "unavailable", + "seekable": self.summary.replayable, + "start_seconds": 0.0 if self.summary.replayable else None, + "end_seconds": duration if self.summary.replayable else None, + "synchronization": "host-arrival-best-effort", + }, + "sources": [source.as_dict() for source in self.sources], + "artifacts": [artifact.as_dict() for artifact in self.artifacts], + } + + +@dataclass(frozen=True, slots=True) +class SessionPage: + items: tuple[SessionSummary, ...] + next_cursor: str | None + + def as_dict(self) -> dict[str, Any]: + return { + "schema_version": "missioncore.observation-session-list/v1", + "items": [item.as_dict() for item in self.items], + "next_cursor": self.next_cursor, + } + + +@dataclass(frozen=True, slots=True) +class WorkspaceLayout: + workspace_id: str + schema_version: int + revision: int + name: str + layout: dict[str, Any] + updated_at_utc: str + + def as_dict(self) -> dict[str, Any]: + return { + "schema_version": "missioncore.workspace-layout/v1", + "workspace_id": self.workspace_id, + "layout_schema_version": self.schema_version, + "revision": self.revision, + "name": self.name, + "layout": self.layout, + "updated_at_utc": self.updated_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class ReplayCommand: + """Internal-only replay command. ``source_path`` never enters an API DTO.""" + + session_id: str + source_path: Path + allowed_root: Path + session_root: Path + replay_byte_length: int + metadata_byte_length: int + expected_source_sha256: str | None + speed: float + loop: bool + + +@dataclass(frozen=True, slots=True) +class RecordedMediaArtifact: + """Internal handle for one confined archived camera source. + + The physical source id and filesystem locator never enter the public API. + ``public_source_id`` and ``artifact_id`` are opaque catalog identifiers. + """ + + session_id: str + public_source_id: str + artifact_id: str + source_path: Path + byte_length: int + + +@dataclass(frozen=True, slots=True) +class LegacySessionCandidate: + session_id: str + display_name: str + status: SessionStatus + started_at_utc: str | None + completed_at_utc: str | None + duration_seconds: float | None + modalities: tuple[SessionModality, ...] + replayable: bool + total_bytes: int + allowed_root: Path + session_root: Path + raw_path: Path + raw_byte_length: int + replay_raw_byte_length: int + replay_metadata_byte_length: int + raw_sha256: str | None + raw_integrity_status: str + media_sources: tuple[LegacyMediaSourceCandidate, ...] + + +@dataclass(frozen=True, slots=True) +class LegacyMediaSourceCandidate: + source_id: str + artifact_id: str + locator: Path + byte_length: int + epoch_count: int diff --git a/src/k1link/sessions/preparation.py b/src/k1link/sessions/preparation.py new file mode 100644 index 0000000..97c5fde --- /dev/null +++ b/src/k1link/sessions/preparation.py @@ -0,0 +1,617 @@ +from __future__ import annotations + +import os +import queue +import threading +from collections.abc import Callable +from dataclasses import dataclass, field, replace +from datetime import UTC, datetime +from functools import partial +from pathlib import Path +from time import monotonic +from typing import Literal, cast +from uuid import uuid4 + +from .media import RecordedMediaManifest, validate_recorded_media_timeline +from .models import ReplayCommand +from .recording import ( + MaterializedRecording, + RecordingMaterializationCancelled, + SessionRecordingMaterializer, +) + +PreparationState = Literal[ + "queued", + "validating", + "exporting", + "finalizing", + "ready", + "failed", + "cancelled", +] +ACTIVE_PREPARATION_STATES = frozenset({"queued", "validating", "exporting", "finalizing"}) +_PREPARATION_PHASE: dict[PreparationState, int] = { + "queued": 0, + "validating": 1, + "exporting": 2, + "finalizing": 3, + "ready": 4, + "failed": 4, + "cancelled": 4, +} + + +class RecordingPreparationQueueFull(RuntimeError): + """The bounded conversion queue cannot accept another recording.""" + + +@dataclass(frozen=True, slots=True) +class RecordingPreparationSnapshot: + preparation_id: str + session_id: str + state: PreparationState + progress: float + updated_at_utc: str + cancellable: bool + retryable: bool + error: str | None + command: ReplayCommand + recording: MaterializedRecording | None + recorded_media: tuple[RecordedMediaManifest, ...] | None + + +@dataclass(slots=True) +class _PreparationJob: + preparation_id: str + source_identity: tuple[object, ...] + # This is an immutable source-preparation command. Per-browser playback + # policy (speed/loop) never belongs to a shared conversion job. + command: ReplayCommand + state: PreparationState = "queued" + progress: float = 0.0 + updated_at_utc: str = field(default_factory=lambda: _utc_now_iso()) + error: str | None = None + recording: MaterializedRecording | None = None + recorded_media: tuple[RecordedMediaManifest, ...] | None = None + cancel_event: threading.Event = field(default_factory=threading.Event) + last_activity_monotonic: float = field(default_factory=monotonic) + interrupted_by_restart: bool = False + cancelled_by_operator: bool = False + + +@dataclass(slots=True) +class _WorkerGeneration: + generation_id: int + work_queue: queue.Queue[_PreparationJob] + stop_event: threading.Event = field(default_factory=threading.Event) + worker: threading.Thread | None = None + + +class SessionRecordingPreparationManager: + """One bounded, process-owned conversion worker for durable recordings. + + Jobs are keyed by the session plus an inexpensive source identity. They + intentionally outlive HTTP requests and browser tabs. Conversion stays + single-worker to bound Rerun's CPU, memory and temporary-disk pressure. + + Worker generations make lifespan restart safe even if a third-party + exporter ignores cancellation longer than ``close(timeout=...)``. New + jobs can queue immediately, but a successor worker starts only after the + previous generation has actually exited, so two writers never overlap. + """ + + def __init__( + self, + materializer: SessionRecordingMaterializer, + *, + queue_capacity: int = 128, + heartbeat_interval_seconds: float = 5.0, + ready_preparer: Callable[ + [ReplayCommand, MaterializedRecording], + tuple[RecordedMediaManifest, ...], + ] + | None = None, + ) -> None: + if queue_capacity < 1: + raise ValueError("recording preparation queue capacity must be positive") + if heartbeat_interval_seconds <= 0: + raise ValueError("recording preparation heartbeat interval must be positive") + self.materializer = materializer + self._queue_capacity = queue_capacity + # Activity callbacks from the real exporter are rate-limited to this + # interval. There is deliberately no independent fake heartbeat: a + # hung exporter must become observable to the browser stall detector. + self._heartbeat_interval_seconds = heartbeat_interval_seconds + self._ready_preparer = ready_preparer + self._guard = threading.RLock() + self._current_by_session: dict[str, _PreparationJob] = {} + self._closed = True + self._generation_counter = 0 + self._active_generation: _WorkerGeneration | None = None + self._pending_generation: _WorkerGeneration | None = None + self.start() + + def start(self) -> None: + """Start or restart the single worker for an application lifespan.""" + + with self._guard: + if not self._closed: + return + self._current_by_session = { + session_id: job + for session_id, job in self._current_by_session.items() + if job.state != "cancelled" + } + self._closed = False + active = self._active_generation + if active is not None and active.worker is not None and active.worker.is_alive(): + self._pending_generation = self._new_generation_locked() + return + self._active_generation = None + generation = self._pending_generation or self._new_generation_locked() + self._pending_generation = None + self._start_generation_locked(generation) + + def enqueue( + self, + command: ReplayCommand, + *, + retry_failed: bool = False, + retry_interrupted: bool = False, + ) -> RecordingPreparationSnapshot: + source_command = _source_command(command) + identity = _source_identity(source_command) + with self._guard: + if self._closed: + raise RuntimeError("recording preparation manager is closed") + current = self._current_by_session.get(command.session_id) + if current is not None and current.source_identity == identity: + retry_terminal = retry_failed and current.state in {"failed", "cancelled"} + retry_restart = ( + retry_interrupted + and current.state == "cancelled" + and current.interrupted_by_restart + ) + if not retry_terminal and not retry_restart: + return self._snapshot_locked(current) + + job = _PreparationJob( + preparation_id=uuid4().hex, + source_identity=identity, + command=source_command, + ) + self._current_by_session[command.session_id] = job + try: + self._intake_queue_locked().put_nowait(job) + except queue.Full as exc: + if self._current_by_session.get(command.session_id) is job: + if current is None: + self._current_by_session.pop(command.session_id, None) + else: + self._current_by_session[command.session_id] = current + raise RecordingPreparationQueueFull("recording preparation queue is full") from exc + return self._snapshot_locked(job) + + def resolve_cached( + self, + command: ReplayCommand, + ) -> RecordingPreparationSnapshot | None: + """Validate a published cache and register it as a ready job.""" + + # When launch preparation includes camera manifests, a disk-only RRD + # is not a complete ready result. The worker must validate both parts + # in one background transaction before publishing ``ready``. + if self._ready_preparer is not None: + return None + + source_command = _source_command(command) + identity = _source_identity(source_command) + recording = self.materializer.get_cached(source_command) + if recording is None: + with self._guard: + current = self._current_by_session.get(command.session_id) + if current is not None and current.state == "ready": + self._current_by_session.pop(command.session_id, None) + return None + with self._guard: + current = self._current_by_session.get(command.session_id) + if ( + current is not None + and current.source_identity == identity + and current.state in ACTIVE_PREPARATION_STATES | {"ready"} + ): + # A worker may have started between the disk lookup and this + # transaction. Let that single job publish its own result. + if current.state == "ready": + current.recording = recording + return self._snapshot_locked(current) + ready = _PreparationJob( + preparation_id=uuid4().hex, + source_identity=identity, + command=source_command, + state="ready", + progress=1.0, + recording=recording, + recorded_media=(), + ) + self._current_by_session[command.session_id] = ready + return self._snapshot_locked(ready) + + def resolve_cached_pinned( + self, + command: ReplayCommand, + ) -> tuple[RecordingPreparationSnapshot, Callable[[], None]] | None: + """Validate and lease a published cache without starting conversion.""" + + if self._ready_preparer is not None: + return None + + source_command = _source_command(command) + result = self.materializer.get_cached_pinned(source_command) + if result is None: + with self._guard: + current = self._current_by_session.get(command.session_id) + if current is not None and current.state == "ready": + self._current_by_session.pop(command.session_id, None) + return None + recording, release = result + identity = _source_identity(source_command) + with self._guard: + current = self._current_by_session.get(command.session_id) + if ( + current is None + or current.source_identity != identity + or current.state not in ACTIVE_PREPARATION_STATES | {"ready"} + ): + current = _PreparationJob( + preparation_id=uuid4().hex, + source_identity=identity, + command=source_command, + state="ready", + progress=1.0, + recording=recording, + recorded_media=(), + ) + self._current_by_session[command.session_id] = current + snapshot = self._snapshot_locked(current) + return snapshot, release + + def status(self, session_id: str) -> RecordingPreparationSnapshot | None: + with self._guard: + job = self._current_by_session.get(session_id) + if ( + job is not None + and job.state == "ready" + and ( + job.recording is None + or not self.materializer.is_recording_available(job.recording) + ) + ): + self._current_by_session.pop(session_id, None) + return None + return None if job is None else self._snapshot_locked(job) + + def reserve_cached( + self, + command: ReplayCommand, + *, + lease_seconds: float = 120.0, + ) -> RecordingPreparationSnapshot | None: + """Pin a ready artifact across the launch-document to file-GET gap.""" + + if lease_seconds <= 0: + raise ValueError("recording launch lease must be positive") + pinned = self.resolve_cached_pinned(command) + if pinned is None: + return None + snapshot, release = pinned + timer = threading.Timer(lease_seconds, release) + timer.name = f"missioncore-recording-launch-lease-{snapshot.preparation_id}" + timer.daemon = True + timer.start() + return snapshot + + def pin_ready( + self, + session_id: str, + *, + preparation_id: str | None = None, + ) -> tuple[RecordingPreparationSnapshot, Callable[[], None]] | None: + """Cheaply lease the exact already-validated ready generation.""" + + with self._guard: + job = self._current_by_session.get(session_id) + if ( + job is None + or job.state != "ready" + or job.recording is None + or ( + preparation_id is not None + and job.preparation_id != preparation_id + ) + ): + return None + recording = job.recording + pinned = self.materializer.pin_recording(recording) + if pinned is None: + if self._current_by_session.get(session_id) is job: + self._current_by_session.pop(session_id, None) + return None + snapshot = self._snapshot_locked(job) + return snapshot, pinned + + def reserve_ready( + self, + session_id: str, + *, + preparation_id: str | None = None, + lease_seconds: float = 120.0, + ) -> RecordingPreparationSnapshot | None: + """Hold a cheap launch lease without reopening or hashing the cache.""" + + if lease_seconds <= 0: + raise ValueError("recording launch lease must be positive") + pinned = self.pin_ready(session_id, preparation_id=preparation_id) + if pinned is None: + return None + snapshot, release = pinned + timer = threading.Timer(lease_seconds, release) + timer.name = f"missioncore-recording-launch-lease-{snapshot.preparation_id}" + timer.daemon = True + timer.start() + return snapshot + + def cancel(self, session_id: str, *, preparation_id: str | None = None) -> bool: + with self._guard: + job = self._current_by_session.get(session_id) + if ( + job is None + or job.state not in ACTIVE_PREPARATION_STATES + or (preparation_id is not None and job.preparation_id != preparation_id) + ): + return False + job.cancelled_by_operator = True + job.cancel_event.set() + if job.state == "queued": + self._transition_locked(job, "cancelled", job.progress) + return True + + def close(self, *, timeout: float = 5.0) -> None: + with self._guard: + if self._closed: + return + self._closed = True + for job in self._current_by_session.values(): + if job.state in ACTIVE_PREPARATION_STATES: + job.interrupted_by_restart = not job.cancelled_by_operator + job.cancel_event.set() + if job.state == "queued": + self._transition_locked(job, "cancelled", job.progress) + active = self._active_generation + if active is not None: + active.stop_event.set() + pending = self._pending_generation + if pending is not None: + pending.stop_event.set() + self._pending_generation = None + worker = None if active is None else active.worker + if worker is not None: + worker.join(timeout=max(0.0, timeout)) + + def _run_generation(self, generation: _WorkerGeneration) -> None: + work_queue = generation.work_queue + try: + while not generation.stop_event.is_set(): + try: + job = work_queue.get(timeout=0.1) + except queue.Empty: + continue + try: + if job.state == "cancelled" or job.cancel_event.is_set(): + with self._guard: + self._transition_locked(job, "cancelled", job.progress) + continue + with self._guard: + self._transition_locked(job, "validating", 0.05) + try: + recording = self.materializer.materialize( + job.command, + progress_callback=partial(self._progress, job), + cancel_event=job.cancel_event, + ) + except RecordingMaterializationCancelled: + with self._guard: + self._transition_locked(job, "cancelled", job.progress) + except Exception: + # The public status intentionally does not expose paths, + # broker payloads or exporter internals. + with self._guard: + if job.cancel_event.is_set(): + self._transition_locked(job, "cancelled", job.progress) + else: + self._transition_locked(job, "failed", job.progress) + job.error = "Не удалось подготовить запись сессии." + job.updated_at_utc = _utc_now_iso() + else: + with self._guard: + # Cancellation may race the final materializer + # return. Never publish ``ready`` after accepting a + # cancel request for this exact job. + if job.cancel_event.is_set(): + self._transition_locked(job, "cancelled", job.progress) + else: + self._transition_locked(job, "finalizing", 0.95) + if job.cancel_event.is_set(): + with self._guard: + self._transition_locked(job, "cancelled", job.progress) + continue + try: + recorded_media = ( + () + if self._ready_preparer is None + else self._ready_preparer(job.command, recording) + ) + validate_recorded_media_timeline( + recorded_media, + recording_start_seconds=( + recording.timeline_start_ns / 1_000_000_000 + ), + recording_end_seconds=( + recording.timeline_end_ns / 1_000_000_000 + ), + ) + except Exception: + with self._guard: + if job.cancel_event.is_set(): + self._transition_locked(job, "cancelled", job.progress) + else: + self._transition_locked(job, "failed", job.progress) + job.error = "Не удалось подготовить запись сессии." + job.updated_at_utc = _utc_now_iso() + continue + with self._guard: + if job.cancel_event.is_set(): + self._transition_locked(job, "cancelled", job.progress) + else: + job.recording = recording + job.recorded_media = recorded_media + self._transition_locked(job, "ready", 1.0) + finally: + work_queue.task_done() + finally: + self._generation_exited(generation) + + def _new_generation_locked(self) -> _WorkerGeneration: + self._generation_counter += 1 + return _WorkerGeneration( + generation_id=self._generation_counter, + work_queue=queue.Queue(maxsize=self._queue_capacity), + ) + + def _start_generation_locked(self, generation: _WorkerGeneration) -> None: + self._active_generation = generation + worker = threading.Thread( + target=self._run_generation, + args=(generation,), + name=f"missioncore-recording-preparation-{generation.generation_id}", + daemon=True, + ) + generation.worker = worker + worker.start() + + def _generation_exited(self, generation: _WorkerGeneration) -> None: + with self._guard: + if self._active_generation is not generation: + return + self._active_generation = None + if self._closed: + return + successor = self._pending_generation or self._new_generation_locked() + self._pending_generation = None + self._start_generation_locked(successor) + + def _intake_queue_locked(self) -> queue.Queue[_PreparationJob]: + pending = self._pending_generation + if pending is not None: + return pending.work_queue + active = self._active_generation + if active is None or active.stop_event.is_set(): + pending = self._new_generation_locked() + self._pending_generation = pending + return pending.work_queue + return active.work_queue + + def _progress(self, job: _PreparationJob, state: str, progress: float) -> None: + # Only the worker publishes ``ready`` after it owns the verified + # recording handle; accepting the final callback would expose a short + # ready-without-launch race to status polling. + if state not in {"validating", "exporting", "finalizing"}: + return + next_state = cast(PreparationState, state) + with self._guard: + if job.state in {"cancelled", "failed", "ready"}: + return + if _PREPARATION_PHASE[next_state] < _PREPARATION_PHASE[job.state]: + return + now = monotonic() + if ( + job.state == next_state + and progress <= job.progress + and now - job.last_activity_monotonic < self._heartbeat_interval_seconds + ): + return + self._transition_locked(job, next_state, progress) + + def _transition_locked( + self, + job: _PreparationJob, + state: PreparationState, + progress: float, + ) -> None: + if job.state not in ACTIVE_PREPARATION_STATES and state != job.state: + return + if _PREPARATION_PHASE[state] < _PREPARATION_PHASE[job.state]: + return + job.state = state + job.progress = max(job.progress, min(1.0, max(0.0, progress))) + job.updated_at_utc = _utc_now_iso() + job.last_activity_monotonic = monotonic() + + def _snapshot_locked(self, job: _PreparationJob) -> RecordingPreparationSnapshot: + return RecordingPreparationSnapshot( + preparation_id=job.preparation_id, + session_id=job.command.session_id, + state=job.state, + progress=job.progress, + updated_at_utc=job.updated_at_utc, + cancellable=( + job.state == "queued" + or ( + job.state in {"validating", "exporting", "finalizing"} + and self.materializer.supports_cooperative_cancellation + ) + ), + retryable=job.state in {"failed", "cancelled"}, + error=job.error, + command=job.command, + recording=job.recording, + recorded_media=job.recorded_media, + ) + + +def _source_identity(command: ReplayCommand) -> tuple[object, ...]: + """Build a non-blocking identity; full validation belongs to the worker.""" + + identities: list[object] = [ + command.session_id, + str(command.source_path), + command.replay_byte_length, + command.metadata_byte_length, + command.expected_source_sha256, + ] + for path in (command.source_path, command.source_path.with_name("mqtt.metadata.jsonl")): + try: + value = os.lstat(path) + except OSError: + identities.extend((str(Path(path)), None, None, None, None)) + else: + identities.extend( + ( + str(Path(path)), + value.st_size, + value.st_mtime_ns, + value.st_ctime_ns, + value.st_ino, + ) + ) + return tuple(identities) + + +def _source_command(command: ReplayCommand) -> ReplayCommand: + """Remove per-viewer launch policy from a shared preparation job.""" + + return replace(command, speed=1.0, loop=False) + + +def _utc_now_iso() -> str: + return datetime.now(UTC).isoformat().replace("+00:00", "Z") diff --git a/src/k1link/sessions/recording.py b/src/k1link/sessions/recording.py new file mode 100644 index 0000000..a351d72 --- /dev/null +++ b/src/k1link/sessions/recording.py @@ -0,0 +1,1345 @@ +from __future__ import annotations + +import hashlib +import inspect +import json +import os +import re +import shutil +import stat +import threading +from collections.abc import Callable, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Any, cast +from uuid import uuid4 + +from k1link.viewer.rrd_export import ( + RrdExportCancelled, + RrdExportError, + export_k1mqtt_to_rrd, +) + +from .models import ReplayCommand + +# v6 adds a real session_time=0 row to the RRD itself. Older sidecars can +# declare a zero start while their payload begins at the first decoded sensor +# frame, so accepting them would violate the browser playback contract. +CACHE_SCHEMA = "missioncore.derived-rerun-recording-cache/v6" +COMPATIBLE_CACHE_SCHEMAS = frozenset({CACHE_SCHEMA}) +RERUN_RECORDING_MEDIA_TYPE = "application/vnd.rerun.rrd" +RERUN_SESSION_TIMELINE = "session_time" +SESSION_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +DEFAULT_CACHE_MAX_BYTES = 8 * 1024 * 1024 * 1024 +DEFAULT_FREE_SPACE_RESERVE_BYTES = 2 * 1024 * 1024 * 1024 + +RrdExporter = Callable[..., Mapping[str, object]] +RecordingProgressCallback = Callable[[str, float], None] +DEFAULT_RRD_EXPORTER = cast(RrdExporter, export_k1mqtt_to_rrd) + + +class RecordingMaterializationError(RuntimeError): + """A seekable browser recording could not be prepared safely.""" + + +class RecordingMaterializationCancelled(RecordingMaterializationError): + """A background conversion was cooperatively cancelled before publication.""" + + +@dataclass(frozen=True, slots=True) +class MaterializedRecording: + """Internal handle for a private, derived Rerun recording.""" + + session_id: str + path: Path + media_type: str + byte_length: int + sha256: str + source_sha256: str + timeline: str + timeline_start_ns: int + timeline_end_ns: int + + +@dataclass(frozen=True, slots=True) +class _ValidatedMemoryEntry: + source_identity: tuple[object, ...] + recording_identity: tuple[int, int, int, int, int] + recording: MaterializedRecording + + +@dataclass(frozen=True, slots=True) +class _ValidatedSource: + source: Path + metadata: Path + source_stat: os.stat_result + metadata_stat: os.stat_result + replay_byte_length: int + metadata_byte_length: int + expected_source_sha256: str | None + + @property + def identity(self) -> tuple[object, ...]: + return ( + *_stat_identity(self.source_stat), + *_stat_identity(self.metadata_stat), + self.replay_byte_length, + self.metadata_byte_length, + ) + + +class SessionRecordingMaterializer: + """Build and validate a per-session seekable RRD under the private data root. + + The native ``.k1mqtt`` capture remains the source of record. Derived RRDs + live below ``data_dir/recordings`` and are reused only when both their + source identity and output digest still match an atomically written cache + sidecar. Calls for one session are serialized, so concurrent browser + requests cannot start duplicate exports or observe a half-published file. + """ + + def __init__( + self, + data_dir: Path, + *, + exporter: RrdExporter = DEFAULT_RRD_EXPORTER, + cache_max_bytes: int | None = None, + free_space_reserve_bytes: int | None = None, + ) -> None: + private_root = data_dir.expanduser().resolve() + private_root.mkdir(mode=0o700, parents=True, exist_ok=True) + _chmod_best_effort(private_root, 0o700) + recordings_root = private_root / "recordings" + recordings_root.mkdir(mode=0o700, parents=True, exist_ok=True) + if recordings_root.is_symlink(): + raise RecordingMaterializationError("recording cache root must not be a symlink") + _chmod_best_effort(recordings_root, 0o700) + self.recordings_root = recordings_root.resolve() + if not self.recordings_root.is_relative_to(private_root): + raise RecordingMaterializationError("recording cache escapes the private data root") + self._exporter = exporter + self._exporter_accepts_cancel = _callable_accepts_keyword(exporter, "cancel_event") + self._exporter_accepts_activity = _callable_accepts_keyword( + exporter, + "activity_callback", + ) + self.cache_max_bytes = _positive_configuration( + cache_max_bytes, + environment_name="MISSIONCORE_RRD_CACHE_MAX_BYTES", + default=DEFAULT_CACHE_MAX_BYTES, + ) + self.free_space_reserve_bytes = _non_negative_configuration( + free_space_reserve_bytes, + environment_name="MISSIONCORE_RRD_FREE_SPACE_RESERVE_BYTES", + default=DEFAULT_FREE_SPACE_RESERVE_BYTES, + ) + self._locks_guard = threading.Lock() + self._session_locks: dict[str, threading.Lock] = {} + self._memory_guard = threading.Lock() + self._validated_memory: dict[str, _ValidatedMemoryEntry] = {} + # Expensive RRD exports stay single-flight, while this much shorter + # guard protects cache publication, eviction and response leases. A + # cache hit must never wait behind a multi-minute export of another + # session. + self._export_lock = threading.Lock() + self._cache_guard = threading.RLock() + self._pinned_sessions: dict[str, int] = {} + self._interprocess_lock_path = self.recordings_root / ".export.lock" + + @property + def supports_cooperative_cancellation(self) -> bool: + """Whether the configured exporter observes a cancellation event.""" + + return self._exporter_accepts_cancel + + def is_recording_available(self, recording: MaterializedRecording) -> bool: + """Cheap no-follow check for a previously validated ready handle.""" + + path = recording.path + try: + path_stat = path.lstat() + except OSError: + return False + return ( + path.parent.parent == self.recordings_root + and path.name == "scene.rrd" + and stat.S_ISREG(path_stat.st_mode) + and not stat.S_ISLNK(path_stat.st_mode) + and path_stat.st_size == recording.byte_length + ) + + def pin_recording( + self, + recording: MaterializedRecording, + ) -> Callable[[], None] | None: + """Lease an in-memory validated handle using only cheap file metadata.""" + + if SESSION_ID_PATTERN.fullmatch(recording.session_id) is None: + return None + with self._cache_guard: + if not self.is_recording_available(recording): + return None + self._increment_pin_locked(recording.session_id) + return self._release_callback(recording.session_id) + + def __call__(self, command: ReplayCommand) -> MaterializedRecording: + """Alias for :meth:`materialize`, suitable for the web API protocol.""" + + return self.materialize(command) + + def get_cached(self, command: ReplayCommand) -> MaterializedRecording | None: + """Return a validated cache hit without starting an export.""" + + session_id = _validate_command_shape(command) + if not self._has_compatible_cache_candidate(session_id): + return None + with self._lock_for(session_id): + return self._load_cached_recording(session_id, _validate_source(command)) + + def get_cached_pinned( + self, + command: ReplayCommand, + ) -> tuple[MaterializedRecording, Callable[[], None]] | None: + """Lease a validated cache hit without starting an export.""" + + session_id = _validate_command_shape(command) + if not self._has_compatible_cache_candidate(session_id): + return None + with self._lock_for(session_id): + recording = self._load_cached_recording( + session_id, + _validate_source(command), + pin=True, + ) + if recording is None: + return None + return recording, self._release_callback(session_id) + + def _has_compatible_cache_candidate(self, session_id: str) -> bool: + _root, recording_path, sidecar_path = self._cache_paths(session_id) + if ( + recording_path.is_symlink() + or sidecar_path.is_symlink() + or not recording_path.is_file() + or not sidecar_path.is_file() + ): + return False + try: + value = json.loads(sidecar_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return False + return isinstance(value, dict) and value.get("schema_version") in COMPATIBLE_CACHE_SCHEMAS + + def source_identity(self, command: ReplayCommand) -> tuple[object, ...]: + """Return the validated source identity used for job deduplication.""" + + session_id = _validate_command_shape(command) + return (session_id, *_validate_source(command).identity) + + def materialize( + self, + command: ReplayCommand, + *, + progress_callback: RecordingProgressCallback | None = None, + cancel_event: threading.Event | None = None, + ) -> MaterializedRecording: + session_id = _validate_command_shape(command) + session_lock = self._lock_for(session_id) + # Per-session locks deduplicate callers for one recording. Validate a + # ready cache before entering the global export gate so another large + # conversion cannot block playback of this already prepared session. + with session_lock: + _report_progress(progress_callback, "validating", 0.05) + _raise_if_cancelled(cancel_event) + source = _validate_source(command) + cached = self._load_cached_recording(session_id, source) + if cached is not None: + _report_progress(progress_callback, "ready", 1.0) + return cached + # One global export gate bounds peak CPU, RAM and temporary disk + # use. Revalidate after waiting because the native capture may + # have changed while another session was being converted. + with self._export_lock, _exclusive_file_lock(self._interprocess_lock_path): + self._scavenge_export_artifacts_locked() + _raise_if_cancelled(cancel_event) + source = _validate_source(command) + cached = self._load_cached_recording(session_id, source) + if cached is not None: + _report_progress(progress_callback, "ready", 1.0) + return cached + _report_progress(progress_callback, "exporting", 0.2) + return self._export_recording_locked( + session_id, + source, + progress_callback=progress_callback, + cancel_event=cancel_event, + ) + + def materialize_pinned( + self, + command: ReplayCommand, + ) -> tuple[MaterializedRecording, Callable[[], None]]: + """Materialize and lease one cache entry for the response lifetime.""" + + session_id = _validate_command_shape(command) + session_lock = self._lock_for(session_id) + with session_lock: + source = _validate_source(command) + recording = self._load_cached_recording(session_id, source, pin=True) + if recording is None: + with self._export_lock, _exclusive_file_lock(self._interprocess_lock_path): + self._scavenge_export_artifacts_locked() + source = _validate_source(command) + recording = self._load_cached_recording(session_id, source, pin=True) + if recording is None: + recording = self._export_recording_locked(session_id, source) + with self._cache_guard: + self._increment_pin_locked(session_id) + + return recording, self._release_callback(session_id) + + def _release_callback(self, session_id: str) -> Callable[[], None]: + released = False + release_guard = threading.Lock() + + def release() -> None: + nonlocal released + with release_guard: + if released: + return + released = True + with self._cache_guard: + count = self._pinned_sessions.get(session_id, 0) + if count <= 1: + self._pinned_sessions.pop(session_id, None) + else: + self._pinned_sessions[session_id] = count - 1 + + return release + + def _scavenge_export_artifacts_locked(self) -> None: + """Remove crash leftovers while holding the cross-process writer lock.""" + + try: + session_roots = tuple(self.recordings_root.iterdir()) + except OSError as exc: + raise RecordingMaterializationError( + "recording cache could not be scavenged" + ) from exc + for session_root in session_roots: + try: + root_stat = session_root.lstat() + except OSError: + continue + if stat.S_ISLNK(root_stat.st_mode) or not stat.S_ISDIR(root_stat.st_mode): + continue + try: + children = tuple(session_root.iterdir()) + except OSError: + continue + for child in children: + name = child.name + stale = ( + (name.startswith(".scene.") and name.endswith(".candidate.rrd")) + or ( + name.startswith("..scene.") + and ".candidate.rrd." in name + and name.endswith(".tmp") + ) + or (name.startswith(".source.") and name.endswith(".tmp")) + or ( + name.startswith(".scene.rrd.cache.json.") + and name.endswith(".tmp") + ) + ) + if not stale: + continue + try: + child_stat = child.lstat() + if stat.S_ISDIR(child_stat.st_mode) and not stat.S_ISLNK( + child_stat.st_mode + ): + shutil.rmtree(child) + else: + child.unlink() + except FileNotFoundError: + continue + except OSError as exc: + raise RecordingMaterializationError( + "stale recording export artifact could not be removed" + ) from exc + + def _lock_for(self, session_id: str) -> threading.Lock: + with self._locks_guard: + lock = self._session_locks.get(session_id) + if lock is None: + lock = threading.Lock() + self._session_locks[session_id] = lock + return lock + + def _cache_paths( + self, + session_id: str, + ) -> tuple[Path, Path, Path]: + session_root = self.recordings_root / session_id + session_root.mkdir(mode=0o700, parents=True, exist_ok=True) + if session_root.is_symlink(): + raise RecordingMaterializationError("session recording cache must not be a symlink") + resolved_session_root = session_root.resolve() + if not resolved_session_root.is_relative_to(self.recordings_root): + raise RecordingMaterializationError("recording cache escapes the private data root") + _chmod_best_effort(resolved_session_root, 0o700) + return ( + resolved_session_root, + resolved_session_root / "scene.rrd", + resolved_session_root / "scene.rrd.cache.json", + ) + + def _load_cached_recording( + self, + session_id: str, + source: _ValidatedSource, + *, + pin: bool = False, + ) -> MaterializedRecording | None: + # Cache validation and the optional lease are one transaction with + # eviction. This makes it safe for FileResponse to open the path after + # this method returns even while another session is being exported. + with self._cache_guard: + resolved_session_root, recording_path, sidecar_path = self._cache_paths(session_id) + memory_cached = self._load_memory_cache( + session_id=session_id, + source=source, + recording_path=recording_path, + ) + if memory_cached is not None: + if pin: + self._increment_pin_locked(session_id) + _touch_lru(resolved_session_root) + return memory_cached + cached = self._load_valid_cache( + session_id=session_id, + source=source, + recording_path=recording_path, + sidecar_path=sidecar_path, + ) + if cached is None: + return None + self._remember(cached, source, _regular_file_stat(recording_path, "recording")) + if pin: + self._increment_pin_locked(session_id) + _touch_lru(resolved_session_root) + return cached + + def _increment_pin_locked(self, session_id: str) -> None: + self._pinned_sessions[session_id] = self._pinned_sessions.get(session_id, 0) + 1 + + def _export_recording_locked( + self, + session_id: str, + source: _ValidatedSource, + *, + progress_callback: RecordingProgressCallback | None = None, + cancel_event: threading.Event | None = None, + ) -> MaterializedRecording: + with self._cache_guard: + resolved_session_root, recording_path, sidecar_path = self._cache_paths(session_id) + self._ensure_cache_capacity( + required_bytes=max(source.replay_byte_length * 2, 1) + 4 * 1024, + protected_session_id=session_id, + ) + + # Keep a valid old artifact and sidecar published while rebuilding. + # The unique confined candidate is atomically swapped in only after + # export, source-stability and digest validation have all succeeded. + candidate_path = resolved_session_root / f".scene.{uuid4().hex}.candidate.rrd" + try: + candidate_path.touch(mode=0o600, exist_ok=False) + except OSError as exc: + raise RecordingMaterializationError( + "derived recording candidate could not be reserved" + ) from exc + + staged_root: Path | None = None + export_source = source.source + try: + if ( + source.replay_byte_length != source.source_stat.st_size + or source.metadata_byte_length != source.metadata_stat.st_size + ): + staged_root, export_source = _stage_replay_prefix( + resolved_session_root, + source, + cancel_event=cancel_event, + activity_callback=lambda: _report_progress( + progress_callback, + "exporting", + 0.3, + ), + ) + summary = self._invoke_exporter( + export_source, + candidate_path, + cancel_event=cancel_event, + activity_callback=lambda: _report_progress( + progress_callback, + "exporting", + 0.5, + ), + ) + _raise_if_cancelled(cancel_event) + _report_progress(progress_callback, "finalizing", 0.9) + except RrdExportCancelled as exc: + candidate_path.unlink(missing_ok=True) + raise RecordingMaterializationCancelled( + "recording preparation was cancelled" + ) from exc + except RrdExportError as exc: + candidate_path.unlink(missing_ok=True) + raise RecordingMaterializationError("native capture could not be exported") from exc + except OSError as exc: + candidate_path.unlink(missing_ok=True) + raise RecordingMaterializationError("derived recording could not be written") from exc + except BaseException: + candidate_path.unlink(missing_ok=True) + raise + finally: + if staged_root is not None: + shutil.rmtree(staged_root, ignore_errors=True) + + try: + source_after = _validate_source_state(source) + if source_after.identity != source.identity: + raise RecordingMaterializationError("native capture changed during RRD export") + _chmod_best_effort(candidate_path, 0o600) + source_sha256 = _sha256_prefix_stable( + source.source, + source.source_stat, + source.replay_byte_length, + ) + if ( + source.expected_source_sha256 is not None + and source_sha256 != source.expected_source_sha256 + ): + raise RecordingMaterializationError( + "native capture digest no longer matches catalog" + ) + candidate = _materialized_from_export( + session_id=session_id, + source_sha256=source_sha256, + recording_path=candidate_path, + summary=summary, + ) + except BaseException: + candidate_path.unlink(missing_ok=True) + raise + try: + with self._cache_guard: + candidate_stat = _regular_file_stat( + candidate_path, + "derived recording candidate", + ) + if candidate_stat.st_size > self.cache_max_bytes: + raise RecordingMaterializationError("derived recording exceeds the cache quota") + self._ensure_cache_capacity( + required_bytes=4 * 1024, + protected_session_id=session_id, + ) + _raise_if_cancelled(cancel_event) + os.replace(candidate_path, recording_path) + _fsync_directory(resolved_session_root) + recording_stat = _regular_file_stat(recording_path, "derived recording") + materialized = MaterializedRecording( + session_id=candidate.session_id, + path=recording_path, + media_type=candidate.media_type, + byte_length=candidate.byte_length, + sha256=candidate.sha256, + source_sha256=candidate.source_sha256, + timeline=candidate.timeline, + timeline_start_ns=candidate.timeline_start_ns, + timeline_end_ns=candidate.timeline_end_ns, + ) + document = _cache_document(materialized, source, recording_stat) + _write_json_atomic(sidecar_path, document) + _chmod_best_effort(sidecar_path, 0o600) + self._remember(materialized, source, recording_stat) + _touch_lru(resolved_session_root) + finally: + candidate_path.unlink(missing_ok=True) + _report_progress(progress_callback, "ready", 1.0) + return materialized + + def _invoke_exporter( + self, + source: Path, + destination: Path, + *, + cancel_event: threading.Event | None, + activity_callback: Callable[[], None], + ) -> Mapping[str, object]: + kwargs: dict[str, object] = {} + if self._exporter_accepts_cancel: + kwargs["cancel_event"] = cancel_event + if self._exporter_accepts_activity: + kwargs["activity_callback"] = activity_callback + return self._exporter(source, destination, **kwargs) + + def _load_memory_cache( + self, + *, + session_id: str, + source: _ValidatedSource, + recording_path: Path, + ) -> MaterializedRecording | None: + with self._memory_guard: + entry = self._validated_memory.get(session_id) + if entry is None or entry.source_identity != source.identity: + return None + if recording_path.is_symlink(): + return None + try: + recording_stat = _regular_file_stat(recording_path, "derived recording") + except RecordingMaterializationError: + return None + if entry.recording_identity != _stat_identity(recording_stat): + return None + return entry.recording + + def _remember( + self, + recording: MaterializedRecording, + source: _ValidatedSource, + recording_stat: os.stat_result, + ) -> None: + entry = _ValidatedMemoryEntry( + source_identity=source.identity, + recording_identity=_stat_identity(recording_stat), + recording=recording, + ) + with self._memory_guard: + self._validated_memory[recording.session_id] = entry + + def _load_valid_cache( + self, + *, + session_id: str, + source: _ValidatedSource, + recording_path: Path, + sidecar_path: Path, + ) -> MaterializedRecording | None: + if recording_path.is_symlink() or sidecar_path.is_symlink(): + return None + try: + raw_document = json.loads(sidecar_path.read_text(encoding="utf-8")) + document = _validate_cache_document(raw_document, session_id) + recording_stat = _regular_file_stat(recording_path, "derived recording") + except (OSError, ValueError, json.JSONDecodeError, RecordingMaterializationError): + return None + + if document["source_file_byte_length"] != source.source_stat.st_size: + return None + if document["source_mtime_ns"] != source.source_stat.st_mtime_ns: + return None + if document["source_ctime_ns"] != source.source_stat.st_ctime_ns: + return None + if document["source_replay_byte_length"] != source.replay_byte_length: + return None + if document["metadata_file_byte_length"] != source.metadata_stat.st_size: + return None + if document["metadata_mtime_ns"] != source.metadata_stat.st_mtime_ns: + return None + if document["metadata_ctime_ns"] != source.metadata_stat.st_ctime_ns: + return None + if document["metadata_replay_byte_length"] != source.metadata_byte_length: + return None + if document["recording_byte_length"] != recording_stat.st_size: + return None + if document["recording_mtime_ns"] != recording_stat.st_mtime_ns: + return None + + source_sha256 = _sha256_prefix_stable( + source.source, + source.source_stat, + source.replay_byte_length, + ) + if ( + source.expected_source_sha256 is not None + and source_sha256 != source.expected_source_sha256 + ): + raise RecordingMaterializationError( + "native capture digest no longer matches catalog" + ) + if source_sha256 != document["source_sha256"]: + return None + metadata_sha256 = _sha256_prefix_stable( + source.metadata, + source.metadata_stat, + source.metadata_byte_length, + ) + if metadata_sha256 != document["metadata_sha256"]: + return None + recording_sha256 = _sha256_stable(recording_path, recording_stat) + if recording_sha256 != document["recording_sha256"]: + return None + _chmod_best_effort(recording_path, 0o600) + _chmod_best_effort(sidecar_path, 0o600) + if document["schema_version"] != CACHE_SCHEMA: + migrated = dict(document) + migrated["schema_version"] = CACHE_SCHEMA + _write_json_atomic(sidecar_path, migrated) + _chmod_best_effort(sidecar_path, 0o600) + return MaterializedRecording( + session_id=session_id, + path=recording_path, + media_type=RERUN_RECORDING_MEDIA_TYPE, + byte_length=recording_stat.st_size, + sha256=recording_sha256, + source_sha256=source_sha256, + timeline=RERUN_SESSION_TIMELINE, + timeline_start_ns=document["timeline_start_ns"], + timeline_end_ns=document["timeline_end_ns"], + ) + + def _ensure_cache_capacity( + self, + *, + required_bytes: int, + protected_session_id: str, + ) -> None: + if required_bytes < 0: + raise RecordingMaterializationError("cache reservation is invalid") + while True: + total_bytes, entries = _cache_entries(self.recordings_root) + free_bytes = shutil.disk_usage(self.recordings_root).free + quota_ok = total_bytes + required_bytes <= self.cache_max_bytes + reserve_ok = free_bytes >= self.free_space_reserve_bytes + required_bytes + if quota_ok and reserve_ok: + return + victim = next( + ( + entry + for entry in entries + if entry[0].name != protected_session_id + and not self._session_is_pinned(entry[0].name) + ), + None, + ) + if victim is None: + reason = "cache quota" if not quota_ok else "free-space reserve" + raise RecordingMaterializationError( + f"derived recording cannot satisfy the {reason}" + ) + victim_path, _victim_bytes, _victim_mtime = victim + _remove_cache_session(self.recordings_root, victim_path) + with self._memory_guard: + self._validated_memory.pop(victim_path.name, None) + + def _session_is_pinned(self, session_id: str) -> bool: + with self._cache_guard: + return self._pinned_sessions.get(session_id, 0) > 0 + + +def _validate_command_shape(command: ReplayCommand) -> str: + session_id = getattr(command, "session_id", None) + if not isinstance(session_id, str) or SESSION_ID_PATTERN.fullmatch(session_id) is None: + raise RecordingMaterializationError("session id has an invalid shape") + return session_id + + +def _report_progress( + callback: RecordingProgressCallback | None, + state: str, + progress: float, +) -> None: + if callback is not None: + callback(state, progress) + + +def _raise_if_cancelled(cancel_event: threading.Event | None) -> None: + if cancel_event is not None and cancel_event.is_set(): + raise RecordingMaterializationCancelled("recording preparation was cancelled") + + +def _validate_source(command: ReplayCommand) -> _ValidatedSource: + source_path = getattr(command, "source_path", None) + allowed_root = getattr(command, "allowed_root", None) + session_root = getattr(command, "session_root", None) + replay_byte_length = getattr(command, "replay_byte_length", None) + metadata_byte_length = getattr(command, "metadata_byte_length", None) + expected_source_sha256 = getattr(command, "expected_source_sha256", None) + if not isinstance(source_path, Path): + raise RecordingMaterializationError("replay command has no native capture") + try: + if not isinstance(allowed_root, Path) or not isinstance(session_root, Path): + raise RecordingMaterializationError("replay command has no confinement roots") + allowed = allowed_root.expanduser().resolve(strict=True) + session = session_root.expanduser().resolve(strict=True) + source = source_path.expanduser().absolute() + source_parent = source.parent.resolve(strict=True) + except OSError as exc: + raise RecordingMaterializationError("native capture is missing") from exc + if ( + not allowed.is_dir() + or not session.is_dir() + or not session.is_relative_to(allowed) + or not source_parent.is_relative_to(session) + ): + raise RecordingMaterializationError("native capture escapes its allowed session root") + if source.suffix.casefold() != ".k1mqtt": + raise RecordingMaterializationError("native capture has an unsupported format") + source_stat = _regular_file_stat_nofollow(source, "native capture") + metadata = source.with_name("mqtt.metadata.jsonl") + if not metadata.parent.resolve(strict=True).is_relative_to(session): + raise RecordingMaterializationError("native metadata escapes its allowed session root") + metadata_stat = _regular_file_stat_nofollow(metadata, "native metadata") + if ( + not isinstance(replay_byte_length, int) + or isinstance(replay_byte_length, bool) + or not 1 <= replay_byte_length <= source_stat.st_size + ): + raise RecordingMaterializationError("native capture replay boundary is invalid") + if ( + not isinstance(metadata_byte_length, int) + or isinstance(metadata_byte_length, bool) + or not 1 <= metadata_byte_length <= metadata_stat.st_size + ): + raise RecordingMaterializationError("native metadata replay boundary is invalid") + if expected_source_sha256 is not None: + if not isinstance(expected_source_sha256, str) or not _is_sha256(expected_source_sha256): + raise RecordingMaterializationError("native capture expected digest is invalid") + if replay_byte_length != source_stat.st_size: + raise RecordingMaterializationError( + "a full-capture digest cannot describe a replay prefix" + ) + return _ValidatedSource( + source=source, + metadata=metadata, + source_stat=source_stat, + metadata_stat=metadata_stat, + replay_byte_length=replay_byte_length, + metadata_byte_length=metadata_byte_length, + expected_source_sha256=expected_source_sha256, + ) + + +def _validate_source_state(source: _ValidatedSource) -> _ValidatedSource: + return _ValidatedSource( + source=source.source, + metadata=source.metadata, + source_stat=_regular_file_stat_nofollow(source.source, "native capture"), + metadata_stat=_regular_file_stat_nofollow(source.metadata, "native metadata"), + replay_byte_length=source.replay_byte_length, + metadata_byte_length=source.metadata_byte_length, + expected_source_sha256=source.expected_source_sha256, + ) + + +def _materialized_from_export( + *, + session_id: str, + source_sha256: str, + recording_path: Path, + summary: Mapping[str, object], +) -> MaterializedRecording: + recording_stat = _regular_file_stat(recording_path, "derived recording") + exported_source_sha256 = _require_sha256(summary, "source_sha256") + recording_sha256 = _require_sha256(summary, "rrd_sha256") + if exported_source_sha256 != source_sha256: + raise RecordingMaterializationError("export source digest does not match native capture") + if recording_sha256 != _sha256_stable(recording_path, recording_stat): + raise RecordingMaterializationError("export digest does not match derived recording") + if _require_int(summary, "rrd_bytes") != recording_stat.st_size: + raise RecordingMaterializationError("export byte length does not match derived recording") + if summary.get("timeline") != RERUN_SESSION_TIMELINE: + raise RecordingMaterializationError("export uses an unsupported playback timeline") + timeline_start_ns = _require_int(summary, "timeline_start_ns") + timeline_end_ns = _require_int(summary, "timeline_end_ns") + if timeline_start_ns < 0 or timeline_end_ns < timeline_start_ns: + raise RecordingMaterializationError("export timeline bounds are invalid") + return MaterializedRecording( + session_id=session_id, + path=recording_path, + media_type=RERUN_RECORDING_MEDIA_TYPE, + byte_length=recording_stat.st_size, + sha256=recording_sha256, + source_sha256=exported_source_sha256, + timeline=RERUN_SESSION_TIMELINE, + timeline_start_ns=timeline_start_ns, + timeline_end_ns=timeline_end_ns, + ) + + +def _cache_document( + recording: MaterializedRecording, + source: _ValidatedSource, + recording_stat: os.stat_result, +) -> dict[str, object]: + return { + "schema_version": CACHE_SCHEMA, + "session_id": recording.session_id, + "source_file_byte_length": source.source_stat.st_size, + "source_replay_byte_length": source.replay_byte_length, + "source_mtime_ns": source.source_stat.st_mtime_ns, + "source_ctime_ns": source.source_stat.st_ctime_ns, + "source_sha256": recording.source_sha256, + "metadata_file_byte_length": source.metadata_stat.st_size, + "metadata_replay_byte_length": source.metadata_byte_length, + "metadata_mtime_ns": source.metadata_stat.st_mtime_ns, + "metadata_ctime_ns": source.metadata_stat.st_ctime_ns, + "metadata_sha256": _sha256_prefix_stable( + source.metadata, + source.metadata_stat, + source.metadata_byte_length, + ), + "recording_byte_length": recording.byte_length, + "recording_mtime_ns": recording_stat.st_mtime_ns, + "recording_sha256": recording.sha256, + "timeline": recording.timeline, + "timeline_start_ns": recording.timeline_start_ns, + "timeline_end_ns": recording.timeline_end_ns, + } + + +def _validate_cache_document(value: object, session_id: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise ValueError("recording cache sidecar is not an object") + expected_keys = { + "schema_version", + "session_id", + "source_file_byte_length", + "source_replay_byte_length", + "source_mtime_ns", + "source_ctime_ns", + "source_sha256", + "metadata_file_byte_length", + "metadata_replay_byte_length", + "metadata_mtime_ns", + "metadata_ctime_ns", + "metadata_sha256", + "recording_byte_length", + "recording_mtime_ns", + "recording_sha256", + "timeline", + "timeline_start_ns", + "timeline_end_ns", + } + if set(value) != expected_keys: + raise ValueError("recording cache sidecar has an unsupported shape") + if value["schema_version"] not in COMPATIBLE_CACHE_SCHEMAS or value["session_id"] != session_id: + raise ValueError("recording cache sidecar identity does not match") + if value["timeline"] != RERUN_SESSION_TIMELINE: + raise ValueError("recording cache timeline is unsupported") + for key in ( + "source_file_byte_length", + "source_replay_byte_length", + "source_mtime_ns", + "source_ctime_ns", + "metadata_file_byte_length", + "metadata_replay_byte_length", + "metadata_mtime_ns", + "metadata_ctime_ns", + "recording_byte_length", + "recording_mtime_ns", + "timeline_start_ns", + "timeline_end_ns", + ): + if not isinstance(value[key], int) or isinstance(value[key], bool) or value[key] < 0: + raise ValueError("recording cache contains an invalid integer") + for key in ("source_sha256", "metadata_sha256", "recording_sha256"): + digest = value[key] + if not isinstance(digest, str) or not _is_sha256(digest): + raise ValueError("recording cache contains an invalid digest") + if value["timeline_end_ns"] < value["timeline_start_ns"]: + raise ValueError("recording cache timeline bounds are invalid") + return cast(dict[str, Any], value) + + +def _regular_file_stat(path: Path, label: str) -> os.stat_result: + try: + stat_result = path.stat() + except OSError as exc: + raise RecordingMaterializationError(f"{label} is missing") from exc + if not path.is_file(): + raise RecordingMaterializationError(f"{label} is not a regular file") + return stat_result + + +def _regular_file_stat_nofollow(path: Path, label: str) -> os.stat_result: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise RecordingMaterializationError(f"{label} is missing or unsafe") from exc + try: + result = os.fstat(descriptor) + if not stat.S_ISREG(result.st_mode): + raise RecordingMaterializationError(f"{label} is not a regular file") + current = os.lstat(path) + if stat.S_ISLNK(current.st_mode) or (current.st_dev, current.st_ino) != ( + result.st_dev, + result.st_ino, + ): + raise RecordingMaterializationError(f"{label} changed during no-follow open") + return result + finally: + os.close(descriptor) + + +def _stat_identity(value: os.stat_result) -> tuple[int, int, int, int, int]: + return (value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns, value.st_ctime_ns) + + +def _sha256_stable(path: Path, expected_stat: os.stat_result) -> str: + return _sha256_prefix_stable(path, expected_stat, expected_stat.st_size) + + +def _sha256_prefix_stable( + path: Path, + expected_stat: os.stat_result, + byte_length: int, +) -> str: + if not 0 <= byte_length <= expected_stat.st_size: + raise RecordingMaterializationError("recording digest boundary is invalid") + return _sha256_prefix_cached( + str(path), + _stat_identity(expected_stat), + byte_length, + ) + + +@lru_cache(maxsize=128) +def _sha256_prefix_cached( + path_text: str, + expected_identity: tuple[int, int, int, int, int], + byte_length: int, +) -> str: + path = Path(path_text) + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise RecordingMaterializationError( + "recording artifact could not be opened without following links" + ) from exc + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode) or _stat_identity(before) != expected_identity: + raise RecordingMaterializationError("recording artifact changed before validation") + digest = hashlib.sha256() + remaining = byte_length + while remaining: + chunk = os.read(descriptor, min(1024 * 1024, remaining)) + if not chunk: + raise RecordingMaterializationError("recording artifact is truncated") + digest.update(chunk) + remaining -= len(chunk) + after = os.fstat(descriptor) + current = os.lstat(path) + if ( + _stat_identity(after) != expected_identity + or stat.S_ISLNK(current.st_mode) + or (current.st_dev, current.st_ino) != (after.st_dev, after.st_ino) + ): + raise RecordingMaterializationError("recording artifact changed during validation") + return digest.hexdigest() + except OSError as exc: + raise RecordingMaterializationError("recording artifact could not be validated") from exc + finally: + os.close(descriptor) + + +def _stage_replay_prefix( + session_cache_root: Path, + source: _ValidatedSource, + *, + cancel_event: threading.Event | None = None, + activity_callback: Callable[[], None] | None = None, +) -> tuple[Path, Path]: + staged_root = session_cache_root / f".source.{uuid4().hex}.tmp" + try: + staged_root.mkdir(mode=0o700) + staged_source = staged_root / "mqtt.raw.k1mqtt" + staged_metadata = staged_root / "mqtt.metadata.jsonl" + _copy_prefix_nofollow( + source.source, + source.source_stat, + staged_source, + source.replay_byte_length, + cancel_event=cancel_event, + activity_callback=activity_callback, + ) + _copy_prefix_nofollow( + source.metadata, + source.metadata_stat, + staged_metadata, + source.metadata_byte_length, + cancel_event=cancel_event, + activity_callback=activity_callback, + ) + _fsync_directory(staged_root) + return staged_root, staged_source + except BaseException: + shutil.rmtree(staged_root, ignore_errors=True) + raise + + +def _copy_prefix_nofollow( + source: Path, + expected_stat: os.stat_result, + destination: Path, + byte_length: int, + *, + cancel_event: threading.Event | None = None, + activity_callback: Callable[[], None] | None = None, +) -> None: + source_flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + source_descriptor = os.open(source, source_flags) + destination_descriptor = os.open( + destination, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0), + 0o600, + ) + try: + before = os.fstat(source_descriptor) + if not stat.S_ISREG(before.st_mode) or _stat_identity(before) != _stat_identity( + expected_stat + ): + raise RecordingMaterializationError("native capture changed before prefix copy") + remaining = byte_length + while remaining: + _raise_if_cancelled(cancel_event) + if activity_callback is not None: + activity_callback() + chunk = os.read(source_descriptor, min(1024 * 1024, remaining)) + if not chunk: + raise RecordingMaterializationError("native capture prefix is truncated") + view = memoryview(chunk) + while view: + written = os.write(destination_descriptor, view) + view = view[written:] + remaining -= len(chunk) + os.fsync(destination_descriptor) + after = os.fstat(source_descriptor) + if _stat_identity(after) != _stat_identity(expected_stat): + raise RecordingMaterializationError("native capture changed during prefix copy") + finally: + os.close(source_descriptor) + os.close(destination_descriptor) + + +def _cache_entries(root: Path) -> tuple[int, list[tuple[Path, int, int]]]: + total = 0 + entries: list[tuple[Path, int, int]] = [] + try: + children = tuple(root.iterdir()) + except OSError as exc: + raise RecordingMaterializationError("recording cache could not be inspected") from exc + for child in children: + if child.name == ".export.lock": + continue + try: + child_stat = child.lstat() + except OSError: + continue + byte_length = _cache_path_bytes(child) + total += byte_length + entries.append((child, byte_length, child_stat.st_mtime_ns)) + entries.sort(key=lambda item: (item[2], item[0].name)) + return total, entries + + +def _cache_path_bytes(path: Path) -> int: + try: + path_stat = path.lstat() + except OSError: + return 0 + if stat.S_ISLNK(path_stat.st_mode) or stat.S_ISREG(path_stat.st_mode): + return path_stat.st_size + if not stat.S_ISDIR(path_stat.st_mode): + return 0 + total = path_stat.st_size + try: + children = tuple(path.iterdir()) + except OSError: + return total + return total + sum(_cache_path_bytes(child) for child in children) + + +def _remove_cache_file(path: Path) -> None: + try: + path_stat = path.lstat() + except FileNotFoundError: + return + except OSError as exc: + raise RecordingMaterializationError("recording cache entry could not be inspected") from exc + if stat.S_ISDIR(path_stat.st_mode): + raise RecordingMaterializationError("recording cache file was replaced by a directory") + try: + path.unlink() + except OSError as exc: + raise RecordingMaterializationError("recording cache entry could not be removed") from exc + + +def _remove_cache_session(root: Path, path: Path) -> None: + if path.parent != root or path.name in {"", ".", ".."}: + raise RecordingMaterializationError("cache eviction target escapes recording root") + try: + path_stat = path.lstat() + if stat.S_ISLNK(path_stat.st_mode) or not stat.S_ISDIR(path_stat.st_mode): + path.unlink() + else: + shutil.rmtree(path) + _fsync_directory(root) + except FileNotFoundError: + return + except OSError as exc: + raise RecordingMaterializationError("derived cache eviction failed") from exc + + +def _touch_lru(path: Path) -> None: + try: + os.utime(path, None, follow_symlinks=False) + except OSError: + return + + +def _positive_configuration( + configured: int | None, + *, + environment_name: str, + default: int, +) -> int: + value = configured + if value is None: + raw = os.environ.get(environment_name, "").strip() + value = int(raw) if raw else default + if isinstance(value, bool) or value <= 0: + raise ValueError(f"{environment_name} must be a positive integer") + return value + + +def _non_negative_configuration( + configured: int | None, + *, + environment_name: str, + default: int, +) -> int: + value = configured + if value is None: + raw = os.environ.get(environment_name, "").strip() + value = int(raw) if raw else default + if isinstance(value, bool) or value < 0: + raise ValueError(f"{environment_name} must be a non-negative integer") + return value + + +def _callable_accepts_keyword(callback: Callable[..., object], keyword: str) -> bool: + try: + parameters = inspect.signature(callback).parameters + except (TypeError, ValueError): + return False + return keyword in parameters or any( + parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in parameters.values() + ) + + +@contextmanager +def _exclusive_file_lock(path: Path) -> Any: + """Serialize exporters across Mission Core processes on POSIX hosts.""" + + try: + import fcntl + except ImportError as exc: # pragma: no cover - production targets are POSIX + raise RecordingMaterializationError( + "cross-process recording lock is unavailable" + ) from exc + + flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) | getattr( + os, + "O_NOFOLLOW", + 0, + ) + try: + descriptor = os.open(path, flags, 0o600) + except OSError as exc: + raise RecordingMaterializationError( + "cross-process recording lock could not be opened" + ) from exc + try: + lock_stat = os.fstat(descriptor) + if not stat.S_ISREG(lock_stat.st_mode): + raise RecordingMaterializationError( + "cross-process recording lock is not a regular file" + ) + fcntl.flock(descriptor, fcntl.LOCK_EX) + yield + except OSError as exc: + raise RecordingMaterializationError( + "cross-process recording lock failed" + ) from exc + finally: + try: + fcntl.flock(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) + + +def _require_int(summary: Mapping[str, object], key: str) -> int: + value = summary.get(key) + if not isinstance(value, int) or isinstance(value, bool): + raise RecordingMaterializationError(f"RRD export summary has no valid {key}") + return value + + +def _require_sha256(summary: Mapping[str, object], key: str) -> str: + value = summary.get(key) + if not isinstance(value, str) or not _is_sha256(value): + raise RecordingMaterializationError(f"RRD export summary has no valid {key}") + return value + + +def _is_sha256(value: str) -> bool: + return len(value) == 64 and all(character in "0123456789abcdef" for character in value) + + +def _write_json_atomic(path: Path, document: Mapping[str, object]) -> None: + temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp") + payload = (json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n").encode() + try: + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + # fdopen owns the descriptor after successful construction. + raise + os.replace(temporary, path) + _fsync_directory(path.parent) + except OSError as exc: + raise RecordingMaterializationError( + "recording cache sidecar could not be published" + ) from exc + finally: + temporary.unlink(missing_ok=True) + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _chmod_best_effort(path: Path, mode: int) -> None: + try: + if path.stat().st_mode & 0o777 != mode: + path.chmod(mode) + except OSError: + return diff --git a/src/k1link/sessions/store.py b/src/k1link/sessions/store.py new file mode 100644 index 0000000..baedcd0 --- /dev/null +++ b/src/k1link/sessions/store.py @@ -0,0 +1,715 @@ +from __future__ import annotations + +import json +import os +import re +import sqlite3 +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any, cast + +from k1link.artifacts import utc_now_iso + +from .legacy import discover_legacy_viewer_sessions +from .models import ( + LayoutConflictError, + LegacySessionCandidate, + RecordedMediaArtifact, + ReplayCommand, + SessionArtifact, + SessionDetail, + SessionIntegrityError, + SessionModality, + SessionNotFoundError, + SessionNotReplayableError, + SessionPage, + SessionSource, + SessionStatus, + SessionSummary, + WorkspaceLayout, +) + +IDENTIFIER_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +MAX_LAYOUT_BYTES = 256 * 1024 +DATABASE_NAME = "mission-core.sqlite3" + +SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS observation_sessions ( + session_id TEXT PRIMARY KEY, + display_name TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('ready', 'interrupted', 'failed')), + started_at_utc TEXT, + completed_at_utc TEXT, + duration_seconds REAL, + modalities_json TEXT NOT NULL, + replayable INTEGER NOT NULL CHECK (replayable IN (0, 1)), + origin TEXT NOT NULL, + source_count INTEGER NOT NULL, + total_bytes INTEGER NOT NULL, + replay_raw_bytes INTEGER NOT NULL DEFAULT 0, + replay_metadata_bytes INTEGER NOT NULL DEFAULT 0, + allowed_root TEXT NOT NULL, + session_root TEXT NOT NULL, + created_at_utc TEXT NOT NULL, + updated_at_utc TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS observation_sessions_recent +ON observation_sessions(started_at_utc DESC, session_id DESC); + +CREATE TABLE IF NOT EXISTS observation_session_artifacts ( + session_id TEXT NOT NULL REFERENCES observation_sessions(session_id) ON DELETE CASCADE, + artifact_id TEXT NOT NULL, + kind TEXT NOT NULL, + media_type TEXT NOT NULL, + byte_length INTEGER NOT NULL, + sha256 TEXT, + integrity_status TEXT NOT NULL, + locator TEXT NOT NULL, + PRIMARY KEY (session_id, artifact_id) +); + +CREATE TABLE IF NOT EXISTS observation_session_sources ( + session_id TEXT NOT NULL REFERENCES observation_sessions(session_id) ON DELETE CASCADE, + source_id TEXT NOT NULL, + semantic_channel_id TEXT NOT NULL, + modality TEXT NOT NULL, + status TEXT NOT NULL, + seekable INTEGER NOT NULL CHECK (seekable IN (0, 1)), + artifact_id TEXT NOT NULL, + PRIMARY KEY (session_id, source_id), + FOREIGN KEY (session_id, artifact_id) + REFERENCES observation_session_artifacts(session_id, artifact_id) + ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS workspace_layouts ( + workspace_id TEXT PRIMARY KEY, + layout_schema_version INTEGER NOT NULL, + revision INTEGER NOT NULL, + name TEXT NOT NULL, + layout_json TEXT NOT NULL, + updated_at_utc TEXT NOT NULL +); +""" + + +def resolve_missioncore_data_dir(repository_root: Path) -> Path: + configured = os.environ.get("MISSIONCORE_DATA_DIR", "").strip() + if configured: + return Path(configured).expanduser().resolve() + return (repository_root.expanduser().resolve() / ".runtime" / "mission-core").resolve() + + +def resolve_missioncore_evidence_dir(repository_root: Path) -> Path: + """Return the private source-of-record root for new observation sessions.""" + + configured = os.environ.get("MISSIONCORE_EVIDENCE_DIR", "").strip() + if configured: + return Path(configured).expanduser().resolve() + return (resolve_missioncore_data_dir(repository_root) / "evidence" / "sessions").resolve() + + +class SessionStore: + """SQLite catalog plus confined filesystem references for host observation sessions.""" + + def __init__(self, repository_root: Path, *, data_dir: Path | None = None) -> None: + self.repository_root = repository_root.expanduser().resolve() + self.data_dir = ( + data_dir.expanduser().resolve() + if data_dir is not None + else resolve_missioncore_data_dir(self.repository_root) + ) + self.data_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + with _ignore_os_error(): + self.data_dir.chmod(0o700) + self.database_path = self.data_dir / DATABASE_NAME + self._lock = threading.RLock() + self._initialize() + + def import_legacy_viewer_live(self, root: Path) -> tuple[str, ...]: + allowed_root = root.expanduser().resolve() + candidates = discover_legacy_viewer_sessions(allowed_root) + imported: list[str] = [] + for candidate in candidates: + self._upsert_legacy(candidate) + imported.append(candidate.session_id) + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + discovered = set(imported) + indexed = connection.execute( + "SELECT session_id FROM observation_sessions " + "WHERE origin = 'legacy-viewer-live' AND allowed_root = ?", + (str(allowed_root),), + ).fetchall() + stale = [row["session_id"] for row in indexed if row["session_id"] not in discovered] + connection.executemany( + "DELETE FROM observation_sessions WHERE session_id = ?", + ((session_id,) for session_id in stale), + ) + connection.commit() + return tuple(imported) + + def list_recent(self, *, limit: int = 20, cursor: str | None = None) -> SessionPage: + if not 1 <= limit <= 100: + raise ValueError("limit must be within 1..100") + parameters: list[object] = [] + where = "" + with self._connect() as connection: + if cursor is not None: + _validate_identifier(cursor, "session cursor") + cursor_row = connection.execute( + "SELECT started_at_utc, session_id FROM observation_sessions " + "WHERE session_id = ?", + (cursor,), + ).fetchone() + if cursor_row is None: + raise SessionNotFoundError("observation session cursor was not found") + where = ( + "WHERE (COALESCE(started_at_utc, ''), session_id) < " + "(COALESCE(?, ''), ?)" + ) + parameters.extend((cursor_row["started_at_utc"], cursor_row["session_id"])) + parameters.append(limit + 1) + rows = connection.execute( + f"SELECT * FROM observation_sessions {where} " # noqa: S608 - static clause + "ORDER BY COALESCE(started_at_utc, '') DESC, session_id DESC LIMIT ?", + parameters, + ).fetchall() + has_more = len(rows) > limit + selected = rows[:limit] + items = tuple(_summary_from_row(row) for row in selected) + next_cursor = items[-1].session_id if has_more and items else None + return SessionPage(items=items, next_cursor=next_cursor) + + def get_session(self, session_id: str) -> SessionDetail: + _validate_identifier(session_id, "session id") + with self._connect() as connection: + row = connection.execute( + "SELECT * FROM observation_sessions WHERE session_id = ?", + (session_id,), + ).fetchone() + if row is None: + raise SessionNotFoundError("observation session was not found") + source_rows = connection.execute( + "SELECT source_id, semantic_channel_id, modality, status, seekable, artifact_id " + "FROM observation_session_sources WHERE session_id = ? ORDER BY source_id", + (session_id,), + ).fetchall() + artifact_rows = connection.execute( + "SELECT artifact_id, kind, media_type, byte_length, sha256, integrity_status " + "FROM observation_session_artifacts WHERE session_id = ? ORDER BY artifact_id", + (session_id,), + ).fetchall() + sources = tuple( + SessionSource( + source_id=source["source_id"], + semantic_channel_id=source["semantic_channel_id"], + modality=cast(SessionModality, source["modality"]), + status=source["status"], + seekable=bool(source["seekable"]), + artifact_id=source["artifact_id"], + ) + for source in source_rows + ) + artifacts = tuple( + SessionArtifact( + artifact_id=artifact["artifact_id"], + kind=artifact["kind"], + media_type=artifact["media_type"], + byte_length=artifact["byte_length"], + sha256=artifact["sha256"], + integrity_status=artifact["integrity_status"], + ) + for artifact in artifact_rows + ) + return SessionDetail(summary=_summary_from_row(row), sources=sources, artifacts=artifacts) + + def prepare_replay( + self, + session_id: str, + *, + speed: float = 1.0, + loop: bool = False, + ) -> ReplayCommand: + detail = self.get_session(session_id) + if not detail.summary.replayable: + raise SessionNotReplayableError("observation session has no replayable spatial source") + if not 0 <= speed <= 100: + raise ValueError("speed must be within 0..100") + with self._connect() as connection: + row = connection.execute( + "SELECT s.allowed_root, s.session_root, s.replay_raw_bytes, " + "s.replay_metadata_bytes, a.locator, a.sha256 " + "FROM observation_sessions AS s " + "JOIN observation_session_artifacts AS a ON a.session_id = s.session_id " + "WHERE s.session_id = ? AND a.artifact_id = 'raw-mqtt'", + (session_id,), + ).fetchone() + if row is None: + raise SessionNotReplayableError("observation session replay artifact is unavailable") + source_path = _resolve_confined_artifact( + Path(row["allowed_root"]), + Path(row["session_root"]), + Path(row["locator"]), + ) + return ReplayCommand( + session_id=session_id, + source_path=source_path, + allowed_root=Path(row["allowed_root"]), + session_root=Path(row["session_root"]), + replay_byte_length=int(row["replay_raw_bytes"]), + metadata_byte_length=int(row["replay_metadata_bytes"]), + expected_source_sha256=row["sha256"], + speed=float(speed), + loop=loop, + ) + + def list_recorded_media(self, session_id: str) -> tuple[RecordedMediaArtifact, ...]: + """Return confined archived-video handles without exposing device ids.""" + + _validate_identifier(session_id, "session id") + with self._connect() as connection: + session = connection.execute( + "SELECT allowed_root, session_root FROM observation_sessions " + "WHERE session_id = ?", + (session_id,), + ).fetchone() + if session is None: + raise SessionNotFoundError("observation session was not found") + rows = connection.execute( + "SELECT DISTINCT a.artifact_id, a.locator, a.byte_length " + "FROM observation_session_artifacts AS a " + "JOIN observation_session_sources AS source " + "ON source.session_id = a.session_id AND source.artifact_id = a.artifact_id " + "WHERE a.session_id = ? AND a.kind = 'recorded-video' " + "AND source.modality = 'video' ORDER BY a.artifact_id", + (session_id,), + ).fetchall() + return tuple( + _recorded_media_from_row( + session_id=session_id, + allowed_root=Path(session["allowed_root"]), + session_root=Path(session["session_root"]), + artifact_id=row["artifact_id"], + locator=Path(row["locator"]), + byte_length=row["byte_length"], + ) + for row in rows + ) + + def get_recorded_media( + self, + session_id: str, + artifact_id: str, + ) -> RecordedMediaArtifact: + _validate_identifier(artifact_id, "recorded media artifact id") + matches = tuple( + artifact + for artifact in self.list_recorded_media(session_id) + if artifact.artifact_id == artifact_id + ) + if len(matches) != 1: + raise SessionNotFoundError("recorded media source was not found") + return matches[0] + + def get_layout(self, workspace_id: str) -> WorkspaceLayout: + _validate_identifier(workspace_id, "workspace id") + with self._connect() as connection: + row = connection.execute( + "SELECT * FROM workspace_layouts WHERE workspace_id = ?", + (workspace_id,), + ).fetchone() + if row is None: + raise SessionNotFoundError("workspace layout was not found") + return _layout_from_row(row) + + def save_layout( + self, + workspace_id: str, + *, + schema_version: int, + expected_revision: int, + name: str, + layout: dict[str, Any], + ) -> WorkspaceLayout: + _validate_identifier(workspace_id, "workspace id") + if schema_version != 1: + raise ValueError("only workspace layout schema version 1 is supported") + if expected_revision < 0: + raise ValueError("expected revision must be non-negative") + normalized_name = name.strip() + if not 1 <= len(normalized_name) <= 160: + raise ValueError("layout name must contain 1..160 characters") + serialized = _serialize_layout(layout) + updated_at = utc_now_iso() + + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + current = connection.execute( + "SELECT revision FROM workspace_layouts WHERE workspace_id = ?", + (workspace_id,), + ).fetchone() + current_revision = 0 if current is None else int(current["revision"]) + if current_revision != expected_revision: + connection.rollback() + raise LayoutConflictError("workspace layout revision changed") + revision = current_revision + 1 + connection.execute( + "INSERT INTO workspace_layouts " + "(workspace_id, layout_schema_version, revision, name, layout_json, " + "updated_at_utc) VALUES (?, ?, ?, ?, ?, ?) " + "ON CONFLICT(workspace_id) DO UPDATE SET " + "layout_schema_version = excluded.layout_schema_version, " + "revision = excluded.revision, name = excluded.name, " + "layout_json = excluded.layout_json, updated_at_utc = excluded.updated_at_utc", + ( + workspace_id, + schema_version, + revision, + normalized_name, + serialized, + updated_at, + ), + ) + connection.commit() + return WorkspaceLayout( + workspace_id=workspace_id, + schema_version=schema_version, + revision=revision, + name=normalized_name, + layout=json.loads(serialized), + updated_at_utc=updated_at, + ) + + def _initialize(self) -> None: + with self._connect() as connection: + connection.executescript(SCHEMA_SQL) + columns = { + row["name"] + for row in connection.execute("PRAGMA table_info(observation_sessions)") + } + if "replay_raw_bytes" not in columns: + connection.execute( + "ALTER TABLE observation_sessions " + "ADD COLUMN replay_raw_bytes INTEGER NOT NULL DEFAULT 0" + ) + if "replay_metadata_bytes" not in columns: + connection.execute( + "ALTER TABLE observation_sessions " + "ADD COLUMN replay_metadata_bytes INTEGER NOT NULL DEFAULT 0" + ) + connection.commit() + with _ignore_os_error(): + self.database_path.chmod(0o600) + + def _upsert_legacy(self, candidate: LegacySessionCandidate) -> None: + _validate_identifier(candidate.session_id, "legacy session id") + allowed_root = candidate.allowed_root.resolve() + session_root = candidate.session_root.resolve() + if not session_root.is_relative_to(allowed_root): + raise SessionIntegrityError("legacy session root escapes its allowed root") + sources = _legacy_sources(candidate) + artifacts = _legacy_artifacts(candidate, session_root) + now = utc_now_iso() + modalities_json = json.dumps(list(candidate.modalities), separators=(",", ":")) + + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + existing = connection.execute( + "SELECT origin, allowed_root, session_root, created_at_utc " + "FROM observation_sessions WHERE session_id = ?", + (candidate.session_id,), + ).fetchone() + if existing is not None and ( + existing["origin"] != "legacy-viewer-live" + or Path(existing["allowed_root"]).resolve() != allowed_root + or Path(existing["session_root"]).resolve() != session_root + ): + connection.rollback() + raise SessionIntegrityError("session id is already bound to another origin") + created_at = existing["created_at_utc"] if existing is not None else now + connection.execute( + "INSERT INTO observation_sessions " + "(session_id, display_name, status, started_at_utc, completed_at_utc, " + "duration_seconds, modalities_json, replayable, origin, source_count, " + "total_bytes, replay_raw_bytes, replay_metadata_bytes, allowed_root, " + "session_root, created_at_utc, updated_at_utc) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " + "ON CONFLICT(session_id) DO UPDATE SET " + "display_name = excluded.display_name, status = excluded.status, " + "started_at_utc = excluded.started_at_utc, " + "completed_at_utc = excluded.completed_at_utc, " + "duration_seconds = excluded.duration_seconds, " + "modalities_json = excluded.modalities_json, " + "replayable = excluded.replayable, source_count = excluded.source_count, " + "total_bytes = excluded.total_bytes, " + "replay_raw_bytes = excluded.replay_raw_bytes, " + "replay_metadata_bytes = excluded.replay_metadata_bytes, " + "updated_at_utc = excluded.updated_at_utc", + ( + candidate.session_id, + candidate.display_name, + candidate.status, + candidate.started_at_utc, + candidate.completed_at_utc, + candidate.duration_seconds, + modalities_json, + int(candidate.replayable), + "legacy-viewer-live", + len(sources), + candidate.total_bytes, + candidate.replay_raw_byte_length, + candidate.replay_metadata_byte_length, + str(allowed_root), + str(session_root), + created_at, + now, + ), + ) + connection.execute( + "DELETE FROM observation_session_sources WHERE session_id = ?", + (candidate.session_id,), + ) + connection.execute( + "DELETE FROM observation_session_artifacts WHERE session_id = ?", + (candidate.session_id,), + ) + connection.executemany( + "INSERT INTO observation_session_artifacts " + "(session_id, artifact_id, kind, media_type, byte_length, sha256, " + "integrity_status, locator) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + [ + (candidate.session_id, *artifact) + for artifact in artifacts + ], + ) + connection.executemany( + "INSERT INTO observation_session_sources " + "(session_id, source_id, semantic_channel_id, modality, status, seekable, " + "artifact_id) VALUES (?, ?, ?, ?, ?, ?, ?)", + [ + ( + candidate.session_id, + source.source_id, + source.semantic_channel_id, + source.modality, + source.status, + int(source.seekable), + source.artifact_id, + ) + for source in sources + ], + ) + connection.commit() + + @contextmanager + def _connect(self) -> Iterator[sqlite3.Connection]: + connection = sqlite3.connect(self.database_path, timeout=5.0) + connection.row_factory = sqlite3.Row + try: + connection.execute("PRAGMA foreign_keys = ON") + connection.execute("PRAGMA synchronous = FULL") + connection.execute("PRAGMA busy_timeout = 5000") + connection.execute("PRAGMA journal_mode = WAL") + yield connection + finally: + connection.close() + + +def _legacy_sources(candidate: LegacySessionCandidate) -> tuple[SessionSource, ...]: + rows: list[SessionSource] = [] + if "point-cloud" in candidate.modalities: + rows.append( + SessionSource( + source_id="sensor.lidar.primary", + semantic_channel_id="spatial.point-cloud.recorded", + modality="point-cloud", + status="recorded", + seekable=candidate.replayable, + artifact_id="raw-mqtt", + ) + ) + if "trajectory" in candidate.modalities: + rows.append( + SessionSource( + source_id="spatial.trajectory", + semantic_channel_id="spatial.pose.recorded", + modality="trajectory", + status="recorded", + seekable=candidate.replayable, + artifact_id="raw-mqtt", + ) + ) + rows.extend( + SessionSource( + source_id=media.source_id, + semantic_channel_id="camera.video.recorded", + modality="video", + status="recorded", + seekable=True, + artifact_id=media.artifact_id, + ) + for media in candidate.media_sources + ) + if len({source.source_id for source in rows}) != len(rows): + raise SessionIntegrityError("legacy session contains duplicate source identifiers") + return tuple(rows) + + +def _legacy_artifacts( + candidate: LegacySessionCandidate, + session_root: Path, +) -> tuple[tuple[str, str, str, int, str | None, str, str], ...]: + artifacts: list[tuple[str, str, str, int, str | None, str, str]] = [ + ( + "raw-mqtt", + "raw-transport", + "application/x-nodedc-k1mqtt", + candidate.raw_byte_length, + candidate.raw_sha256, + candidate.raw_integrity_status, + str(candidate.raw_path), + ) + ] + for media in candidate.media_sources: + try: + locator = media.locator.resolve(strict=True) + except OSError as exc: + raise SessionIntegrityError("legacy video artifact is missing") from exc + if not locator.is_dir() or not locator.is_relative_to(session_root): + raise SessionIntegrityError("legacy video artifact escapes its session root") + artifacts.append( + ( + media.artifact_id, + "recorded-video", + "video/mp4", + media.byte_length, + None, + "validated-structure", + str(locator), + ) + ) + if len({artifact[0] for artifact in artifacts}) != len(artifacts): + raise SessionIntegrityError("legacy session contains duplicate artifact identifiers") + return tuple(artifacts) + + +def _summary_from_row(row: sqlite3.Row) -> SessionSummary: + raw_modalities = json.loads(row["modalities_json"]) + modalities = tuple(cast(SessionModality, value) for value in raw_modalities) + return SessionSummary( + session_id=row["session_id"], + display_name=row["display_name"], + status=cast(SessionStatus, row["status"]), + started_at_utc=row["started_at_utc"], + completed_at_utc=row["completed_at_utc"], + duration_seconds=row["duration_seconds"], + modalities=modalities, + source_count=row["source_count"], + total_bytes=row["total_bytes"], + replayable=bool(row["replayable"]), + origin=row["origin"], + ) + + +def _layout_from_row(row: sqlite3.Row) -> WorkspaceLayout: + payload = json.loads(row["layout_json"]) + if not isinstance(payload, dict): + raise SessionIntegrityError("stored workspace layout is not a JSON object") + return WorkspaceLayout( + workspace_id=row["workspace_id"], + schema_version=row["layout_schema_version"], + revision=row["revision"], + name=row["name"], + layout=payload, + updated_at_utc=row["updated_at_utc"], + ) + + +def _resolve_confined_artifact( + allowed_root: Path, + session_root: Path, + locator: Path, +) -> Path: + try: + allowed = allowed_root.resolve(strict=True) + session = session_root.resolve(strict=True) + artifact = locator.resolve(strict=True) + except OSError as exc: + raise SessionIntegrityError("session replay artifact is missing") from exc + if ( + not allowed.is_dir() + or not session.is_dir() + or not artifact.is_file() + or not session.is_relative_to(allowed) + or not artifact.is_relative_to(session) + ): + raise SessionIntegrityError("session replay artifact escapes its allowed root") + return artifact + + +def _recorded_media_from_row( + *, + session_id: str, + allowed_root: Path, + session_root: Path, + artifact_id: str, + locator: Path, + byte_length: int, +) -> RecordedMediaArtifact: + _validate_identifier(artifact_id, "recorded media artifact id") + try: + allowed = allowed_root.resolve(strict=True) + session = session_root.resolve(strict=True) + media_root = (session / "media").resolve(strict=True) + artifact = locator.resolve(strict=True) + except OSError as exc: + raise SessionIntegrityError("recorded media artifact is missing") from exc + if ( + not allowed.is_dir() + or not session.is_dir() + or not media_root.is_dir() + or not artifact.is_dir() + or not session.is_relative_to(allowed) + or not media_root.is_relative_to(session) + or artifact.parent != media_root + ): + raise SessionIntegrityError("recorded media artifact escapes its session root") + if not isinstance(byte_length, int) or isinstance(byte_length, bool) or byte_length < 1: + raise SessionIntegrityError("recorded media artifact has an invalid byte length") + public_suffix = artifact_id.removeprefix("recorded-video-") + public_source_id = f"recorded.camera.{public_suffix}" + _validate_identifier(public_source_id, "recorded media source id") + return RecordedMediaArtifact( + session_id=session_id, + public_source_id=public_source_id, + artifact_id=artifact_id, + source_path=artifact, + byte_length=byte_length, + ) + + +def _serialize_layout(layout: dict[str, Any]) -> str: + try: + serialized = json.dumps(layout, ensure_ascii=False, separators=(",", ":")) + except (TypeError, ValueError) as exc: + raise ValueError("layout must be a JSON object") from exc + if len(serialized.encode("utf-8")) > MAX_LAYOUT_BYTES: + raise ValueError("layout exceeds the 256 KiB storage boundary") + return serialized + + +def _validate_identifier(value: str, label: str) -> None: + if not IDENTIFIER_PATTERN.fullmatch(value): + raise ValueError(f"{label} has an invalid shape") + + +@contextmanager +def _ignore_os_error() -> Iterator[None]: + try: + yield + except OSError: + return diff --git a/src/k1link/viewer/replay.py b/src/k1link/viewer/replay.py index b1489d5..7a701c6 100644 --- a/src/k1link/viewer/replay.py +++ b/src/k1link/viewer/replay.py @@ -53,11 +53,17 @@ def _iter_native_capture(path: Path) -> Iterator[StreamMessage]: epoch_ns = fallback_epoch_ns + frame.sequence - 1 monotonic_ns: int | None = None if metadata_stream is not None: - epoch_ns, monotonic_ns = _read_native_timing( + timing = _read_native_timing( metadata_stream, expected_sequence=frame.sequence, fallback_epoch_ns=epoch_ns, ) + if timing is None: + # A crash may leave one raw frame ahead of the last fully + # committed metadata line. Only the aligned prefix has a + # trustworthy source timeline and is safe to replay. + return + epoch_ns, monotonic_ns = timing yield StreamMessage( sequence=frame.sequence, topic=frame.topic, @@ -78,15 +84,19 @@ def _read_native_timing( *, expected_sequence: int, fallback_epoch_ns: int, -) -> tuple[int, int | None]: +) -> tuple[int, int | None] | None: line = stream.readline(MAX_METADATA_LINE_CHARS + 1) if not line: - return fallback_epoch_ns, None + return None if len(line) > MAX_METADATA_LINE_CHARS: raise ReplayFormatError("native metadata line exceeds the reviewed bound") try: record = json.loads(line) except json.JSONDecodeError as exc: + if not line.endswith(("\n", "\r")): + # A non-newline final tail is the only tolerated corruption: the + # writer may have crashed between raw and metadata group commits. + return None raise ReplayFormatError( f"native metadata line {expected_sequence} is not valid JSON" ) from exc diff --git a/src/k1link/viewer/rrd_export.py b/src/k1link/viewer/rrd_export.py new file mode 100644 index 0000000..1380fa7 --- /dev/null +++ b/src/k1link/viewer/rrd_export.py @@ -0,0 +1,595 @@ +from __future__ import annotations + +import hashlib +import math +import os +import threading +from collections.abc import Callable +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path +from typing import TypedDict +from uuid import UUID, uuid4 + +import numpy as np +import rerun as rr +from rerun import blueprint as rrb + +from k1link.data_plane import DecodedPointCloudView, DecodedPoseView, NormalizationError +from k1link.protocol.normalizer import normalize_k1_message +from k1link.viewer.replay import ReplayFormatError, detect_replay_format, iter_replay_messages +from k1link.viewer.rerun_bridge import ( + MAX_TRAJECTORY_POSES, + TRAJECTORY_APPEND_INTERVAL_NS, + TRAJECTORY_FORCE_APPEND_NS, + TRAJECTORY_MIN_DISTANCE_METERS, + TRAJECTORY_PUBLISH_INTERVAL_NS, + RerunSceneSettings, + _parse_hex_color, + _point_colors, +) + +APPLICATION_ID = "nodedc_mission_core_recorded" +SESSION_TIMELINE = "session_time" +CAPTURE_TIMELINE = "capture_time" +JS_MAX_SAFE_INTEGER = (1 << 53) - 1 + +# Rerun keys viewer state by these IDs. Reusing them for every settings-only +# blueprint update makes the update overwrite the existing scene instead of +# creating a fresh view/container (which would also reset the operator's eye +# position and layout). +RECORDED_SPATIAL_VIEW_ID = UUID("5f5f11d5-3b0a-4a81-887b-2be767cba1c0") +RECORDED_ROOT_CONTAINER_ID = UUID("b02f2aca-8471-4dcb-b786-53df5a320fc8") +RECORDED_POINTS_VISUALIZER_ID = UUID("ca037ec0-8761-4417-86ee-846fa2875303") + + +class RrdExportSummary(TypedDict): + schema_version: int + input_path: str + output_path: str + recording_id: str + timeline: str + capture_timeline: str + source_messages: int + decoded_messages: int + point_frames: int + pose_frames: int + ignored_messages: int + points: int + trajectory_poses: int + trajectory_updates: int + session_origin_monotonic_ns: int + timeline_start_ns: int + timeline_end_ns: int + timeline_span_ns: int + first_decoded_time_ns: int + last_decoded_time_ns: int + source_sha256: str + rrd_sha256: str + rrd_bytes: int + + +class RrdExportError(RuntimeError): + """A raw capture could not be converted into a complete durable RRD.""" + + +class RrdExportCancelled(RrdExportError): + """A background RRD export was cooperatively cancelled.""" + + +@dataclass(slots=True) +class _ExportCounters: + source_messages: int = 0 + point_frames: int = 0 + pose_frames: int = 0 + ignored_messages: int = 0 + points: int = 0 + first_decoded_time_ns: int | None = None + last_decoded_time_ns: int | None = None + + @property + def decoded_messages(self) -> int: + return self.point_frames + self.pose_frames + + def observe_decoded(self, session_time_ns: int) -> None: + if self.first_decoded_time_ns is None: + self.first_decoded_time_ns = session_time_ns + self.last_decoded_time_ns = session_time_ns + + +@dataclass(slots=True) +class _TrajectoryBuffer: + positions: list[tuple[float, float, float]] + last_append_time_ns: int | None = None + last_publish_time_ns: int | None = None + updates: int = 0 + + @classmethod + def empty(cls) -> _TrajectoryBuffer: + return cls(positions=[]) + + def process( + self, + recording: rr.RecordingStream, + position: tuple[float, float, float], + session_time_ns: int, + ) -> None: + if not self._append(position, session_time_ns): + return + if ( + self.last_publish_time_ns is not None + and session_time_ns - self.last_publish_time_ns < TRAJECTORY_PUBLISH_INTERVAL_NS + ): + return + self.last_publish_time_ns = session_time_ns + self.updates += 1 + recording.log( + "/world/trajectory", + rr.LineStrips3D( + [list(self.positions)], + colors=[247, 248, 244, 255], + radii=rr.Radius.ui_points(2.0), + ), + ) + + def _append(self, position: tuple[float, float, float], session_time_ns: int) -> bool: + if not self.positions: + self.positions.append(position) + self.last_append_time_ns = session_time_ns + return True + + assert self.last_append_time_ns is not None + elapsed_ns = session_time_ns - self.last_append_time_ns + if elapsed_ns < TRAJECTORY_APPEND_INTERVAL_NS: + return False + if ( + math.dist(self.positions[-1], position) < TRAJECTORY_MIN_DISTANCE_METERS + and elapsed_ns < TRAJECTORY_FORCE_APPEND_NS + ): + return False + + self.positions.append(position) + self.last_append_time_ns = session_time_ns + if len(self.positions) > MAX_TRAJECTORY_POSES: + last = self.positions[-1] + self.positions = self.positions[::2] + if self.positions[-1] != last: + self.positions.append(last) + return True + + +def export_k1mqtt_to_rrd( + input_path: Path, + output_path: Path, + *, + cancel_event: threading.Event | None = None, + activity_callback: Callable[[], None] | None = None, +) -> RrdExportSummary: + """Losslessly project every decodable K1 data-plane frame into one RRD. + + The raw capture remains the source of record. The derived RRD uses a + recording-local duration timeline whose zero is the first raw message's + receive-monotonic timestamp. It never traverses the bounded live-preview + queue, so export throughput cannot drop point or pose frames. + + The destination is replaced only after the temporary RRD has been closed, + flushed and fsynced. Any decode, timing, sink or rename failure therefore + leaves an existing destination artifact untouched. + """ + + _raise_if_cancelled(cancel_event) + source = input_path.expanduser().resolve() + destination = output_path.expanduser().resolve() + _validate_paths(source, destination) + destination.parent.mkdir(parents=True, exist_ok=True) + + recording_id = str(uuid4()) + temporary = destination.with_name(f".{destination.name}.{recording_id}.tmp") + source_sha256 = _sha256_file( + source, + cancel_event=cancel_event, + activity_callback=activity_callback, + ) + settings = RerunSceneSettings() + blueprint = _recorded_blueprint(settings) + recording: rr.RecordingStream | None = None + recording_closed = False + published = False + + counters = _ExportCounters() + trajectory = _TrajectoryBuffer.empty() + session_origin_ns: int | None = None + previous_monotonic_ns: int | None = None + + try: + recording = rr.RecordingStream(APPLICATION_ID, recording_id=recording_id) + recording.set_sinks( + rr.FileSink(temporary, write_footer=True), + default_blueprint=blueprint, + ) + _log_static_scene(recording) + _log_session_origin(recording) + + for message in iter_replay_messages(source): + _raise_if_cancelled(cancel_event) + _notify_activity(activity_callback) + counters.source_messages += 1 + monotonic_ns = message.received_monotonic_ns + if monotonic_ns is None: + raise RrdExportError( + "native capture metadata must provide received_monotonic_ns " + f"for message {message.sequence}" + ) + if previous_monotonic_ns is not None and monotonic_ns < previous_monotonic_ns: + raise RrdExportError( + f"native capture monotonic time decreases at message {message.sequence}" + ) + if session_origin_ns is None: + session_origin_ns = monotonic_ns + session_time_ns = monotonic_ns - session_origin_ns + if session_time_ns > JS_MAX_SAFE_INTEGER: + raise RrdExportError( + "session duration exceeds the exact JavaScript nanosecond range" + ) + previous_monotonic_ns = monotonic_ns + + try: + decoded = normalize_k1_message( + message, + processing_started_monotonic_ns=monotonic_ns, + ) + except NormalizationError as exc: + raise RrdExportError( + f"known K1 frame {message.sequence} failed normalization" + ) from exc + if decoded is None: + counters.ignored_messages += 1 + continue + + _set_frame_time( + recording, + decoded.context.sequence, + session_time_ns, + decoded.context.captured_at_epoch_ns, + ) + counters.observe_decoded(session_time_ns) + if isinstance(decoded, DecodedPointCloudView): + _log_points(recording, decoded, settings) + counters.point_frames += 1 + counters.points += decoded.point_count + elif isinstance(decoded, DecodedPoseView): + position = ( + float(decoded.position_xyz[0]), + float(decoded.position_xyz[1]), + float(decoded.position_xyz[2]), + ) + _log_pose(recording, decoded, position) + trajectory.process(recording, position, session_time_ns) + counters.pose_frames += 1 + else: + counters.ignored_messages += 1 + + if session_origin_ns is None: + raise RrdExportError("native capture contains no messages") + if counters.decoded_messages == 0: + raise RrdExportError("native capture contains no decodable point or pose frames") + assert counters.first_decoded_time_ns is not None + assert counters.last_decoded_time_ns is not None + + _raise_if_cancelled(cancel_event) + recording.flush(timeout_sec=30.0) + recording.disconnect() + recording_closed = True + _raise_if_cancelled(cancel_event) + _fsync_file(temporary) + rrd_bytes = temporary.stat().st_size + if rrd_bytes <= 0: + raise RrdExportError("Rerun produced an empty recording") + rrd_sha256 = _sha256_file( + temporary, + cancel_event=cancel_event, + activity_callback=activity_callback, + ) + + summary = RrdExportSummary( + schema_version=1, + input_path=str(source), + output_path=str(destination), + recording_id=recording_id, + timeline=SESSION_TIMELINE, + capture_timeline=CAPTURE_TIMELINE, + source_messages=counters.source_messages, + decoded_messages=counters.decoded_messages, + point_frames=counters.point_frames, + pose_frames=counters.pose_frames, + ignored_messages=counters.ignored_messages, + points=counters.points, + trajectory_poses=len(trajectory.positions), + trajectory_updates=trajectory.updates, + session_origin_monotonic_ns=session_origin_ns, + timeline_start_ns=0, + # Playback completeness is defined by data actually written to + # the RRD. K1 status/heartbeat packets may continue long after the + # final point or pose frame; advertising that raw tail as the RRD + # end makes a strict browser buffering gate wait forever. + timeline_end_ns=counters.last_decoded_time_ns, + timeline_span_ns=counters.last_decoded_time_ns, + first_decoded_time_ns=counters.first_decoded_time_ns, + last_decoded_time_ns=counters.last_decoded_time_ns, + source_sha256=source_sha256, + rrd_sha256=rrd_sha256, + rrd_bytes=rrd_bytes, + ) + _raise_if_cancelled(cancel_event) + os.replace(temporary, destination) + _fsync_directory(destination.parent) + published = True + return summary + except (ReplayFormatError, OSError) as exc: + raise RrdExportError(f"RRD export failed: {exc}") from exc + finally: + if recording is not None and not recording_closed: + with suppress(BaseException): + recording.disconnect() + if not published: + temporary.unlink(missing_ok=True) + + +def _validate_paths(source: Path, destination: Path) -> None: + if not source.is_file(): + raise RrdExportError(f"raw capture does not exist: {source}") + if source.suffix.casefold() != ".k1mqtt": + raise RrdExportError("RRD export accepts only native .k1mqtt captures") + if destination.suffix.casefold() != ".rrd": + raise RrdExportError("RRD destination must use the .rrd suffix") + if source == destination: + raise RrdExportError("raw capture and RRD destination must be different files") + try: + replay_format = detect_replay_format(source) + except ReplayFormatError as exc: + raise RrdExportError(str(exc)) from exc + if replay_format != "k1mqtt": + raise RrdExportError("RRD export accepts only native K1MQTT captures") + + +def _recorded_blueprint( + settings: RerunSceneSettings, + *, + include_initial_playback_state: bool = True, +) -> rrb.Blueprint: + accumulation = max(0.0, settings.accumulation_seconds) + # An omitted range means Rerun's native latest-at query: the most recent + # LiDAR frame at the cursor. A zero-width range is *not* equivalent; it + # only matches rows stamped at the cursor's exact nanosecond and therefore + # makes an ordinary recorded scan appear empty. + time_ranges: list[rr.VisibleTimeRange] | None = None + if accumulation > 0: + time_ranges = [ + rr.VisibleTimeRange( + SESSION_TIMELINE, + start=rr.TimeRangeBoundary.cursor_relative(seconds=-accumulation), + end=rr.TimeRangeBoundary.cursor_relative(), + ) + ] + point_visualizer = rr.Points3D.from_fields( + # A negative Radius value is Rerun's serialized representation + # for UI points. Blueprint overrides broadcast this singleton + # value across every recorded point without rewriting the store. + radii=rr.Radius.ui_points(settings.point_size), + # Recorded height/intensity/distance/RGB palettes are baked into + # each Points3D row. A uniform custom color is the one color mode + # that can be replaced safely by a singleton blueprint override. + colors=( + [_parse_hex_color(settings.custom_color)] + if settings.palette == "custom" + else None + ), + ).visualizer() + point_visualizer.id = RECORDED_POINTS_VISUALIZER_ID + spatial_view = rrb.Spatial3DView( + origin="/world", + name="Пространственная сцена", + background=[7, 8, 10, 255], + line_grid=rrb.LineGrid3D( + visible=settings.show_grid, + color=[86, 91, 99, 110], + stroke_width=0.75, + ), + overrides={ + # EntityBehavior is evaluated from the blueprint store and can + # therefore hide/reveal already-recorded entities without + # rewriting the data RRD. Keep the point visualizer alongside it + # so size/color overrides remain active for the same entity. + "/world/points": [ + rrb.EntityBehavior(visible=settings.show_points), + point_visualizer, + ], + "/world/trajectory": rrb.EntityBehavior( + visible=settings.show_trajectory, + ), + }, + # A positive window accumulates historical frames. With no + # window, latest-at deliberately keeps one current LiDAR frame. + time_ranges=time_ranges, + ) + spatial_view.id = RECORDED_SPATIAL_VIEW_ID + root_container = rrb.Tabs(spatial_view) + root_container.id = RECORDED_ROOT_CONTAINER_ID + + if include_initial_playback_state: + # This state is appropriate only while opening a newly exported RRD. + # Settings-only blueprint messages must not pause an already playing + # recording or mutate the host-owned panel state. + return rrb.Blueprint( + root_container, + rrb.TimePanel( + timeline=SESSION_TIMELINE, + play_state="paused", + state="hidden", + ), + auto_layout=False, + auto_views=False, + collapse_panels=True, + ) + + return rrb.Blueprint( + root_container, + auto_layout=False, + auto_views=False, + collapse_panels=False, + ) + + +def recorded_blueprint_rrd( + settings: RerunSceneSettings, + *, + application_id: str = APPLICATION_ID, + recording_id: str, +) -> bytes: + """Serialize a small active blueprint update for an already-open recording. + + The returned RRD contains blueprint-store messages only; it never copies the + recorded data store and is therefore safe to push through a WebViewer log + channel when operator display settings change. + """ + + recording = rr.RecordingStream( + application_id, + recording_id=recording_id, + send_properties=False, + ) + stream = rr.binary_stream(recording) + try: + recording.send_blueprint( + _recorded_blueprint( + settings, + include_initial_playback_state=False, + ), + make_active=True, + make_default=False, + ) + payload = stream.read(flush=True, flush_timeout_sec=5.0) + except Exception as exc: + raise RrdExportError("failed to serialize recorded blueprint") from exc + finally: + with suppress(Exception): + recording.disconnect() + if not payload or not payload.startswith(b"RRF2") or len(payload) > 1_048_576: + raise RrdExportError("serialized recorded blueprint is invalid") + return payload + + +def _log_static_scene(recording: rr.RecordingStream) -> None: + recording.log("/world", rr.ViewCoordinates.RIGHT_HAND_Z_UP, static=True) + recording.log( + "/world/sensor_pose", + rr.TransformAxes3D(axis_length=0.45, show_frame=False), + static=True, + ) + + +def _log_session_origin(recording: rr.RecordingStream) -> None: + """Materialize the declared zero of ``session_time`` outside the 3D scene.""" + + recording.set_time( + SESSION_TIMELINE, + duration=np.timedelta64(0, "ns"), + ) + recording.log( + "/__mission_core/session_origin", + rr.AnyValues(session_origin=True), + ) + + +def _set_frame_time( + recording: rr.RecordingStream, + sequence: int, + session_time_ns: int, + capture_time_ns: int, +) -> None: + recording.set_time( + SESSION_TIMELINE, + duration=np.timedelta64(session_time_ns, "ns"), + ) + recording.set_time( + CAPTURE_TIMELINE, + timestamp=np.datetime64(capture_time_ns, "ns"), + ) + recording.set_time("message_sequence", sequence=sequence) + + +def _log_points( + recording: rr.RecordingStream, + frame: DecodedPointCloudView, + settings: RerunSceneSettings, +) -> None: + positions = np.asarray(frame.positions_xyz, dtype=np.float32).reshape((-1, 3)) + if frame.intensities is None: + intensities = np.full(frame.point_count, 255, dtype=np.uint8) + else: + intensities = np.frombuffer(frame.intensities, dtype=np.uint8) + rgb = ( + None + if frame.colors_rgb is None + else np.frombuffer(frame.colors_rgb, dtype=np.uint8).reshape((-1, 3)) + ) + recording.log( + "/world/points", + rr.Points3D( + positions, + colors=_point_colors(positions, intensities, rgb, settings), + radii=rr.Radius.ui_points(settings.point_size), + ), + ) + + +def _log_pose( + recording: rr.RecordingStream, + frame: DecodedPoseView, + position: tuple[float, float, float], +) -> None: + recording.log( + "/world/sensor_pose", + rr.Transform3D( + translation=position, + quaternion=rr.Quaternion(xyzw=frame.orientation_xyzw), + ), + ) + + +def _fsync_file(path: Path) -> None: + with path.open("rb") as stream: + os.fsync(stream.fileno()) + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _sha256_file( + path: Path, + *, + cancel_event: threading.Event | None = None, + activity_callback: Callable[[], None] | None = None, +) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + _raise_if_cancelled(cancel_event) + _notify_activity(activity_callback) + digest.update(chunk) + return digest.hexdigest() + + +def _raise_if_cancelled(cancel_event: threading.Event | None) -> None: + if cancel_event is not None and cancel_event.is_set(): + raise RrdExportCancelled("RRD export was cancelled") + + +def _notify_activity(callback: Callable[[], None] | None) -> None: + if callback is not None: + callback() diff --git a/src/k1link/web/camera_archive.py b/src/k1link/web/camera_archive.py new file mode 100644 index 0000000..50e8296 --- /dev/null +++ b/src/k1link/web/camera_archive.py @@ -0,0 +1,1098 @@ +from __future__ import annotations + +import hashlib +import importlib +import json +import os +import re +import secrets +import stat +import threading +import time +from pathlib import Path +from typing import IO, Any, Literal, cast + +from k1link.artifacts import utc_now_iso + +CameraArchiveKind = Literal["init", "media"] +CameraArchiveStatus = Literal["complete", "interrupted", "failed"] + +CAMERA_ARCHIVE_SCHEMA = "missioncore.camera-recording/v1" +CAMERA_INDEX_SCHEMA = "missioncore.camera-recording-index/v1" +CAMERA_COMMIT_POLICY = "per-segment-fsync" + +_SAFE_COMPONENT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +_EPOCH_DIRECTORY = re.compile(r"^epoch-([1-9][0-9]*)$") +_SEGMENT_FILE = re.compile(r"^([1-9][0-9]*)\.m4s$") +_DEFAULT_COMMIT_INTERVAL_SECONDS = 0.25 +_DEFAULT_COMMIT_BYTES = 4 * 1024 * 1024 +_MAX_RECOVERY_INDEX_BYTES = 32 * 1024 * 1024 +_MAX_RECOVERY_SEGMENT_BYTES = 8 * 1024 * 1024 +_MAX_RECOVERY_SEGMENTS = 500_000 + +_ACTIVE_ARCHIVES_LOCK = threading.Lock() +_ACTIVE_ARCHIVES: set[Path] = set() + + +class CameraArchiveError(RuntimeError): + """A durable camera artifact could not be committed or recovered.""" + + +class CameraArchiveWriter: + """Write a canonical, seekable fMP4 camera epoch. + + ``init.mp4`` and every ``segments/N.m4s`` file are individually fsynced and + atomically published before the matching JSONL index record. The index and + an interrupted checkpoint summary are then fsynced before :meth:`append` + returns. This deliberately uses a per-segment durability boundary: the old + group-commit tuning arguments remain source-compatible but never weaken the + segment-bound RPO. + + A process crash can interrupt the small multi-file commit window. The + module-level :func:`recover_incomplete_camera_archives` reconciles that + window from the independently durable fragments without discarding bytes. + """ + + def __init__( + self, + session_dir: Path, + source_id: str, + generation: int, + *, + commit_interval_seconds: float = _DEFAULT_COMMIT_INTERVAL_SECONDS, + commit_bytes: int = _DEFAULT_COMMIT_BYTES, + ) -> None: + if not _SAFE_COMPONENT.fullmatch(source_id): + raise ValueError("camera source id is not a safe storage identifier") + if generation < 1: + raise ValueError("camera generation must be positive") + if commit_interval_seconds < 0: + raise ValueError("camera commit interval must be non-negative") + if commit_bytes < 1: + raise ValueError("camera commit byte threshold must be positive") + + root = session_dir.expanduser().resolve() + if not root.is_dir(): + raise ValueError("observation session directory does not exist") + media_root = root / "media" + source_dir = media_root / source_id + archive_dir = source_dir / f"epoch-{generation}" + + root_fd = _open_directory_fd(root) + if root_fd is None: + raise CameraArchiveError("camera session root failed no-follow validation") + media_fd: int | None = None + source_fd: int | None = None + archive_fd: int | None = None + segments_fd: int | None = None + index_stream: IO[bytes] | None = None + storage_ready = False + try: + media_fd = _create_or_open_private_directory_at(root_fd, "media") + source_fd = _create_or_open_private_directory_at(media_fd, source_id) + archive_fd = _create_or_open_private_directory_at( + source_fd, + f"epoch-{generation}", + exclusive=True, + ) + segments_fd = _create_or_open_private_directory_at( + archive_fd, + "segments", + exclusive=True, + ) + index_stream = _open_private_binary_at(archive_fd, "index.jsonl") + storage_ready = True + finally: + for descriptor in (source_fd, media_fd, root_fd): + if descriptor is not None: + os.close(descriptor) + if not storage_ready: + if index_stream is not None: + index_stream.close() + for descriptor in (segments_fd, archive_fd): + if descriptor is not None: + os.close(descriptor) + if index_stream is None or archive_fd is None or segments_fd is None: + raise CameraArchiveError("camera index could not be created") + + self.session_dir = root + self.source_id = source_id + self.generation = generation + self.archive_dir = archive_dir + self.init_path = self.archive_dir / "init.mp4" + self.segments_dir = self.archive_dir / "segments" + self.index_path = self.archive_dir / "index.jsonl" + self.summary_path = self.archive_dir / "summary.json" + self._archive_fd = archive_fd + self._segments_fd = segments_fd + self._index = index_stream + self._lock = threading.Lock() + self._sequence = 0 + self._committed_media_segments = 0 + self._committed_bytes = 0 + self._init_written = False + self._closed = False + self._closed_summary: dict[str, Any] | None = None + self._failed = False + self._started_at_utc = utc_now_iso() + self._started_monotonic_ns = time.monotonic_ns() + self._stream_sha256 = hashlib.sha256() + self._index_sha256 = hashlib.sha256() + self._init_sha256: str | None = None + self._legacy_commit_interval_seconds = commit_interval_seconds + self._legacy_commit_bytes = commit_bytes + with _ACTIVE_ARCHIVES_LOCK: + _ACTIVE_ARCHIVES.add(self.archive_dir) + + def append( + self, + kind: CameraArchiveKind, + payload: bytes, + *, + host_epoch_ns: int | None = None, + host_monotonic_ns: int | None = None, + ) -> dict[str, Any]: + if kind not in {"init", "media"}: + raise ValueError("camera archive kind must be init or media") + if not payload: + raise ValueError("camera archive payload must not be empty") + epoch_ns = host_epoch_ns if host_epoch_ns is not None else time.time_ns() + monotonic_ns = ( + host_monotonic_ns if host_monotonic_ns is not None else time.monotonic_ns() + ) + if epoch_ns < 0 or monotonic_ns < 0: + raise ValueError("camera archive timestamps must be non-negative") + + with self._lock: + self._require_open_locked() + if kind == "init": + return self._append_init_locked(payload, epoch_ns, monotonic_ns) + if not self._init_written: + raise CameraArchiveError("camera media arrived before its init segment") + return self._append_media_locked(payload, epoch_ns, monotonic_ns) + + def flush(self) -> None: + """Reassert the current segment-bound checkpoint durability.""" + + with self._lock: + self._require_open_locked() + try: + self._index.flush() + os.fsync(self._index.fileno()) + self._write_summary_locked( + status="interrupted", + failure_code="process-crash-before-clean-close", + ) + except (OSError, CameraArchiveError) as exc: + self._failed = True + raise CameraArchiveError("camera archive durable commit failed") from exc + + def close( + self, + *, + status: CameraArchiveStatus = "complete", + failure_code: str | None = None, + ) -> dict[str, Any]: + with self._lock: + if self._closed: + if self._closed_summary is None: + raise CameraArchiveError("camera archive close previously failed") + return dict(self._closed_summary) + effective_status: CameraArchiveStatus = "failed" if self._failed else status + close_error: OSError | None = None + try: + self._index.flush() + os.fsync(self._index.fileno()) + except OSError as exc: + self._failed = True + effective_status = "failed" + close_error = exc + finally: + try: + self._index.close() + except OSError as exc: + close_error = close_error or exc + + summary = self._summary_locked( + status=effective_status, + failure_code=( + failure_code + or ("storage-commit-failed" if close_error is not None else None) + ), + ) + try: + _write_json_atomic_at(self._archive_fd, "summary.json", summary) + except (OSError, CameraArchiveError) as exc: + self._failed = True + raise CameraArchiveError("camera archive summary commit failed") from exc + finally: + self._closed = True + os.close(self._segments_fd) + os.close(self._archive_fd) + with _ACTIVE_ARCHIVES_LOCK: + _ACTIVE_ARCHIVES.discard(self.archive_dir) + self._closed_summary = dict(summary) + if close_error is not None: + raise CameraArchiveError("camera archive final commit failed") from close_error + return summary + + def _append_init_locked( + self, + payload: bytes, + epoch_ns: int, + monotonic_ns: int, + ) -> dict[str, Any]: + if self._init_written: + raise CameraArchiveError("camera archive already contains an init segment") + digest = hashlib.sha256(payload).hexdigest() + try: + _write_bytes_atomic_at(self._archive_fd, "init.mp4", payload) + self._init_written = True + self._init_sha256 = digest + self._committed_bytes = len(payload) + self._stream_sha256.update(payload) + self._write_summary_locked( + status="interrupted", + failure_code="process-crash-before-clean-close", + ) + except (OSError, CameraArchiveError) as exc: + self._failed = True + raise CameraArchiveError("camera archive durable commit failed") from exc + return { + "schema_version": CAMERA_INDEX_SCHEMA, + "sequence": 0, + "kind": "init", + "path": "init.mp4", + "length": len(payload), + "sha256": digest, + "host_epoch_ns": epoch_ns, + "host_monotonic_ns": monotonic_ns, + "session_monotonic_ns": max(0, monotonic_ns - self._started_monotonic_ns), + } + + def _append_media_locked( + self, + payload: bytes, + epoch_ns: int, + monotonic_ns: int, + ) -> dict[str, Any]: + sequence = self._sequence + 1 + relative_path = f"segments/{sequence}.m4s" + segment_name = f"{sequence}.m4s" + digest = hashlib.sha256(payload).hexdigest() + entry = { + "schema_version": CAMERA_INDEX_SCHEMA, + "sequence": sequence, + "kind": "media", + "path": relative_path, + "length": len(payload), + "sha256": digest, + "host_epoch_ns": epoch_ns, + "host_monotonic_ns": monotonic_ns, + "session_monotonic_ns": max(0, monotonic_ns - self._started_monotonic_ns), + } + encoded = _encode_index_entry(entry) + index_offset = self._index.tell() + published_segment = False + try: + _write_bytes_atomic_at(self._segments_fd, segment_name, payload) + published_segment = True + self._index.write(encoded) + self._index.flush() + os.fsync(self._index.fileno()) + except (OSError, CameraArchiveError) as exc: + self._failed = True + # Best-effort rollback keeps the last fully checkpointed prefix + # structurally valid. Recovery handles a crash before this block. + with _ignore_os_error(): + self._index.seek(index_offset) + self._index.truncate() + self._index.flush() + os.fsync(self._index.fileno()) + if published_segment: + with _ignore_os_error(): + os.unlink(segment_name, dir_fd=self._segments_fd) + os.fsync(self._segments_fd) + raise CameraArchiveError("camera archive durable commit failed") from exc + + self._sequence = sequence + self._committed_media_segments += 1 + self._committed_bytes += len(payload) + self._stream_sha256.update(payload) + self._index_sha256.update(encoded) + try: + self._write_summary_locked( + status="interrupted", + failure_code="process-crash-before-clean-close", + ) + except (OSError, CameraArchiveError) as exc: + self._failed = True + raise CameraArchiveError("camera archive durable checkpoint failed") from exc + return entry + + def _write_summary_locked( + self, + *, + status: CameraArchiveStatus, + failure_code: str | None, + ) -> None: + _write_json_atomic_at( + self._archive_fd, + "summary.json", + self._summary_locked(status=status, failure_code=failure_code), + ) + + def _summary_locked( + self, + *, + status: CameraArchiveStatus, + failure_code: str | None, + ) -> dict[str, Any]: + return { + "schema_version": CAMERA_ARCHIVE_SCHEMA, + "source_id": self.source_id, + "codec_epoch": self.generation, + "status": status, + "started_at_utc": self._started_at_utc, + "completed_at_utc": utc_now_iso(), + "segment_count": self._committed_media_segments, + "entry_count": self._committed_media_segments, + "media_segment_count": self._committed_media_segments, + "valid_bytes": self._committed_bytes, + "init_sha256": self._init_sha256, + "stream_sha256": self._stream_sha256.hexdigest(), + "index_sha256": self._index_sha256.hexdigest(), + "synchronization": "host-arrival-best-effort", + "commit_policy": CAMERA_COMMIT_POLICY, + "failure_code": failure_code, + "artifacts": { + "init": "init.mp4", + "segments": "segments", + "index": "index.jsonl", + }, + } + + def _require_open_locked(self) -> None: + if self._closed: + raise CameraArchiveError("camera archive is already closed") + if self._failed: + raise CameraArchiveError("camera archive is in a failed state") + + +def recover_incomplete_camera_archives( + sessions_root: Path, +) -> tuple[dict[str, Any], ...]: + """Seal incomplete camera epochs beneath a trusted sessions root. + + The scan is confined to direct, safe session children and canonical + ``media//epoch-N`` trees. In-process active writers are skipped. + Complete, structurally valid summaries are idempotent no-ops. A no-follow + advisory lease at ``sessions_root/.camera-recovery.lock`` serializes worker + startup recovery across processes. For an incomplete epoch, independently + durable, contiguous ``N.m4s`` fragments are re-indexed; non-contiguous/orphan + evidence is moved (never deleted) to ``recovery-orphans``; and an + ``interrupted`` summary is atomically written. + + This is intentionally a startup/catalog-refresh operation, not a hot-path + operation: it hashes media fragments. The callable is safe to repeat, but a + composition layer should normally execute it once per server process before + the first catalog import. + """ + + root = sessions_root.expanduser().resolve() + if not root.is_dir(): + return () + root_fd = _open_directory_fd(root) + if root_fd is None: + raise CameraArchiveError("camera sessions root failed no-follow validation") + lock_fd: int | None = None + try: + lock_fd = _open_recovery_lock_at(root_fd) + _lock_recovery_file(lock_fd) + recovered: list[dict[str, Any]] = [] + for session_path in sorted(root.iterdir()): + if not session_path.is_dir() or not _SAFE_COMPONENT.fullmatch(session_path.name): + continue + session = _resolved_confined_directory(session_path, root) + if session is None: + continue + media_root = _resolved_confined_directory(session / "media", session) + if media_root is None: + continue + for source_path in sorted(media_root.iterdir()): + if not source_path.is_dir() or not _SAFE_COMPONENT.fullmatch(source_path.name): + continue + source = _resolved_confined_directory(source_path, media_root) + if source is None: + continue + for epoch_path in sorted(source.iterdir()): + match = _EPOCH_DIRECTORY.fullmatch(epoch_path.name) + if not epoch_path.is_dir() or match is None: + continue + epoch = _resolved_confined_directory(epoch_path, source) + if epoch is None: + continue + with _ACTIVE_ARCHIVES_LOCK: + active = epoch in _ACTIVE_ARCHIVES + if active: + continue + summary = _recover_epoch( + epoch, + source_id=source.name, + generation=int(match.group(1)), + ) + if summary is not None: + recovered.append(summary) + return tuple(recovered) + finally: + if lock_fd is not None: + _unlock_recovery_file(lock_fd) + os.close(lock_fd) + os.close(root_fd) + + +def _recover_epoch( + epoch: Path, + *, + source_id: str, + generation: int, +) -> dict[str, Any] | None: + epoch_fd = _open_directory_fd(epoch) + if epoch_fd is None: + return None + try: + init = _read_regular_at(epoch_fd, "init.mp4", _MAX_RECOVERY_SEGMENT_BYTES) + if not init: + return None + segments_fd = _open_directory_at(epoch_fd, "segments") + if segments_fd is None: + return None + try: + old_index = _read_regular_at( + epoch_fd, + "index.jsonl", + _MAX_RECOVERY_INDEX_BYTES, + allow_empty=True, + ) + old_summary = _read_regular_at( + epoch_fd, + "summary.json", + _MAX_RECOVERY_INDEX_BYTES, + allow_empty=False, + ) + segment_payloads, segment_timestamps, orphans = _read_recovery_segments( + segments_fd + ) + if _sealed_epoch_is_valid( + source_id=source_id, + summary_bytes=old_summary, + index_bytes=old_index, + segment_payloads=segment_payloads, + orphans=orphans, + ): + return None + + old_entries = _parse_index_prefix(old_index) + old_by_sequence = { + int(entry["sequence"]): entry + for entry in old_entries + if isinstance(entry.get("sequence"), int) + } + recovered_entries: list[dict[str, Any]] = [] + stream_hash = hashlib.sha256(init) + valid_bytes = len(init) + expected = 1 + while expected <= _MAX_RECOVERY_SEGMENTS: + payload = segment_payloads.get(expected) + if payload is None: + break + digest = hashlib.sha256(payload).hexdigest() + previous = old_by_sequence.get(expected) + if _index_entry_matches(previous, expected, len(payload), digest): + entry = dict(cast(dict[str, Any], previous)) + else: + entry = { + "schema_version": CAMERA_INDEX_SCHEMA, + "sequence": expected, + "kind": "media", + "path": f"segments/{expected}.m4s", + "length": len(payload), + "sha256": digest, + "host_epoch_ns": segment_timestamps[expected], + "host_monotonic_ns": None, + "session_monotonic_ns": None, + "recovered": True, + } + recovered_entries.append(entry) + stream_hash.update(payload) + valid_bytes += len(payload) + expected += 1 + + valid_sequences = {int(entry["sequence"]) for entry in recovered_entries} + orphans.extend( + f"{sequence}.m4s" + for sequence in segment_payloads + if sequence not in valid_sequences + ) + orphans = sorted(set(orphans)) + if not recovered_entries: + return None + + encoded_index = b"".join( + _encode_index_entry(entry) for entry in recovered_entries + ) + needs_recovery_dir = bool( + orphans + or (old_index is not None and old_index != encoded_index) + or old_summary + ) + recovery_fd: int | None = None + if needs_recovery_dir: + recovery_fd = _ensure_recovery_directory_at(epoch_fd) + try: + if ( + recovery_fd is not None + and old_index is not None + and old_index != encoded_index + ): + _preserve_recovery_artifact_at( + recovery_fd, + "index.pre-recovery.jsonl", + old_index, + ) + if recovery_fd is not None and old_summary: + _preserve_recovery_artifact_at( + recovery_fd, + "summary.pre-recovery.json", + old_summary, + ) + if recovery_fd is not None: + for orphan in orphans: + _quarantine_orphan_at(segments_fd, recovery_fd, orphan) + + _write_bytes_atomic_at(epoch_fd, "index.jsonl", encoded_index) + started_at = _summary_timestamp_bytes( + old_summary, + "started_at_utc", + ) or utc_now_iso() + index_hash = hashlib.sha256(encoded_index).hexdigest() + summary = { + "schema_version": CAMERA_ARCHIVE_SCHEMA, + "source_id": source_id, + "codec_epoch": generation, + "status": "interrupted", + "started_at_utc": started_at, + "completed_at_utc": utc_now_iso(), + "segment_count": len(recovered_entries), + "entry_count": len(recovered_entries), + "media_segment_count": len(recovered_entries), + "valid_bytes": valid_bytes, + "init_sha256": hashlib.sha256(init).hexdigest(), + "stream_sha256": stream_hash.hexdigest(), + "index_sha256": index_hash, + "synchronization": "host-arrival-best-effort", + "commit_policy": CAMERA_COMMIT_POLICY, + "failure_code": "server-process-interrupted", + "recovered_at_utc": utc_now_iso(), + "artifacts": { + "init": "init.mp4", + "segments": "segments", + "index": "index.jsonl", + }, + } + _write_json_atomic_at(epoch_fd, "summary.json", summary) + return summary + finally: + if recovery_fd is not None: + os.close(recovery_fd) + finally: + os.close(segments_fd) + finally: + os.close(epoch_fd) + + +def _sealed_epoch_is_valid( + *, + source_id: str, + summary_bytes: bytes | None, + index_bytes: bytes | None, + segment_payloads: dict[int, bytes], + orphans: list[str], +) -> bool: + if not summary_bytes or index_bytes is None or orphans: + return False + try: + summary = json.loads(summary_bytes) + except (UnicodeDecodeError, json.JSONDecodeError): + return False + if not isinstance(summary, dict) or summary.get("source_id") != source_id: + return False + segment_count = summary.get("segment_count") + if ( + not isinstance(segment_count, int) + or isinstance(segment_count, bool) + or segment_count < 1 + or set(segment_payloads) != set(range(1, segment_count + 1)) + ): + return False + entries = _parse_index_prefix(index_bytes) + if len(entries) != segment_count: + return False + return all( + _index_entry_matches( + entry, + sequence, + len(segment_payloads[sequence]), + hashlib.sha256(segment_payloads[sequence]).hexdigest(), + ) + for sequence, entry in enumerate(entries, start=1) + ) + + +def _read_recovery_segments( + segments_fd: int, +) -> tuple[dict[int, bytes], dict[int, int], list[str]]: + payloads: dict[int, bytes] = {} + timestamps: dict[int, int] = {} + orphans: list[str] = [] + try: + names = sorted(os.listdir(segments_fd)) + except OSError as exc: + raise CameraArchiveError("camera segment directory cannot be enumerated") from exc + for name in names: + match = _SEGMENT_FILE.fullmatch(name) + if match is None: + continue + sequence = int(match.group(1)) + if name != f"{sequence}.m4s" or sequence in payloads: + orphans.append(name) + continue + read_result = _read_regular_with_metadata_at( + segments_fd, + name, + _MAX_RECOVERY_SEGMENT_BYTES, + allow_empty=False, + ) + if read_result is None: + orphans.append(name) + continue + payload, metadata = read_result + payloads[sequence] = payload + timestamps[sequence] = metadata.st_mtime_ns + return payloads, timestamps, orphans + + +def _parse_index_prefix(payload: bytes | None) -> list[dict[str, Any]]: + if not payload: + return [] + result: list[dict[str, Any]] = [] + for raw_line in payload.splitlines(keepends=True): + if not raw_line.endswith(b"\n"): + break + try: + value = json.loads(raw_line) + except (UnicodeDecodeError, json.JSONDecodeError): + break + if not isinstance(value, dict): + break + sequence = value.get("sequence") + if ( + not isinstance(sequence, int) + or isinstance(sequence, bool) + or sequence != len(result) + 1 + ): + break + result.append(value) + return result + + +def _index_entry_matches( + entry: object, + sequence: int, + length: int, + digest: str, +) -> bool: + return ( + isinstance(entry, dict) + and entry.get("sequence") == sequence + and entry.get("kind") == "media" + and entry.get("path") == f"segments/{sequence}.m4s" + and entry.get("length") == length + and entry.get("sha256") == digest + ) + + +def _encode_index_entry(entry: dict[str, Any]) -> bytes: + return ( + json.dumps(entry, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n" + ).encode("utf-8") + + +def _open_private_binary_at(parent_fd: int, name: str) -> IO[bytes]: + if "/" in name or name in {"", ".", ".."}: + raise CameraArchiveError("camera artifact has an unsafe name") + descriptor = os.open( + name, + os.O_RDWR | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + dir_fd=parent_fd, + ) + return os.fdopen(descriptor, "w+b") + + +def _resolved_confined_directory(path: Path, parent: Path) -> Path | None: + try: + metadata = path.lstat() + if not stat.S_ISDIR(metadata.st_mode): + return None + resolved = path.resolve(strict=True) + except OSError: + return None + return resolved if resolved.is_dir() and resolved.is_relative_to(parent) else None + + +def _open_directory_fd(path: Path) -> int | None: + try: + before = path.lstat() + if not stat.S_ISDIR(before.st_mode): + return None + descriptor = os.open( + path, + os.O_RDONLY + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + except OSError: + return None + opened = os.fstat(descriptor) + if (opened.st_dev, opened.st_ino) != (before.st_dev, before.st_ino): + os.close(descriptor) + return None + return descriptor + + +def _open_directory_at(parent_fd: int, name: str) -> int | None: + if "/" in name or name in {"", ".", ".."}: + return None + try: + before = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if not stat.S_ISDIR(before.st_mode): + return None + descriptor = os.open( + name, + os.O_RDONLY + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0), + dir_fd=parent_fd, + ) + except OSError: + return None + opened = os.fstat(descriptor) + if (opened.st_dev, opened.st_ino) != (before.st_dev, before.st_ino): + os.close(descriptor) + return None + return descriptor + + +def _create_or_open_private_directory_at( + parent_fd: int, + name: str, + *, + exclusive: bool = False, +) -> int: + if "/" in name or name in {"", ".", ".."}: + raise CameraArchiveError("camera storage directory has an unsafe name") + try: + os.mkdir(name, 0o700, dir_fd=parent_fd) + os.fsync(parent_fd) + except FileExistsError: + if exclusive: + raise + except OSError as exc: + raise CameraArchiveError("camera storage directory cannot be created") from exc + descriptor = _open_directory_at(parent_fd, name) + if descriptor is None: + raise CameraArchiveError("camera storage directory failed no-follow validation") + try: + fchmod = getattr(os, "fchmod", None) + if callable(fchmod): + fchmod(descriptor, 0o700) + return descriptor + except OSError as exc: + os.close(descriptor) + raise CameraArchiveError("camera storage permissions cannot be enforced") from exc + + +def _open_recovery_lock_at(root_fd: int) -> int: + name = ".camera-recovery.lock" + descriptor: int | None = None + try: + try: + before = os.stat(name, dir_fd=root_fd, follow_symlinks=False) + except FileNotFoundError: + before = None + if before is not None and not stat.S_ISREG(before.st_mode): + raise OSError("recovery lock is not a regular file") + descriptor = os.open( + name, + os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), + 0o600, + dir_fd=root_fd, + ) + metadata = os.fstat(descriptor) + after = os.stat(name, dir_fd=root_fd, follow_symlinks=False) + if ( + not stat.S_ISREG(metadata.st_mode) + or not stat.S_ISREG(after.st_mode) + or (metadata.st_dev, metadata.st_ino) != (after.st_dev, after.st_ino) + or ( + before is not None + and (metadata.st_dev, metadata.st_ino) != (before.st_dev, before.st_ino) + ) + ): + raise OSError("recovery lock is not a regular file") + fchmod = getattr(os, "fchmod", None) + if callable(fchmod): + fchmod(descriptor, 0o600) + if metadata.st_size == 0: + os.write(descriptor, b"\0") + os.fsync(descriptor) + return descriptor + except OSError as exc: + if descriptor is not None: + os.close(descriptor) + raise CameraArchiveError("camera recovery lock failed no-follow validation") from exc + + +def _lock_recovery_file(descriptor: int) -> None: + try: + if os.name == "posix": + fcntl = importlib.import_module("fcntl") + fcntl.flock(descriptor, fcntl.LOCK_EX) + return + msvcrt = importlib.import_module("msvcrt") + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_LOCK, 1) + except (ImportError, OSError) as exc: + raise CameraArchiveError("camera recovery lease could not be acquired") from exc + + +def _unlock_recovery_file(descriptor: int) -> None: + try: + if os.name == "posix": + fcntl = importlib.import_module("fcntl") + fcntl.flock(descriptor, fcntl.LOCK_UN) + return + msvcrt = importlib.import_module("msvcrt") + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_UNLCK, 1) + except (ImportError, OSError): + # Closing the descriptor below releases an OS advisory lease even if an + # explicit unlock fails during shutdown. + return + + +def _read_regular_at( + parent_fd: int, + name: str, + limit: int, + *, + allow_empty: bool = False, +) -> bytes | None: + result = _read_regular_with_metadata_at( + parent_fd, + name, + limit, + allow_empty=allow_empty, + ) + return result[0] if result is not None else None + + +def _read_regular_with_metadata_at( + parent_fd: int, + name: str, + limit: int, + *, + allow_empty: bool = False, +) -> tuple[bytes, os.stat_result] | None: + if "/" in name or name in {"", ".", ".."}: + return None + try: + before = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size > limit + or (before.st_size == 0 and not allow_empty) + ): + return None + descriptor = os.open( + name, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + dir_fd=parent_fd, + ) + except OSError: + return None + try: + opened = os.fstat(descriptor) + if ( + not stat.S_ISREG(opened.st_mode) + or (opened.st_dev, opened.st_ino) != (before.st_dev, before.st_ino) + or opened.st_size > limit + ): + return None + chunks: list[bytes] = [] + remaining = opened.st_size + while remaining: + chunk = os.read(descriptor, min(1024 * 1024, remaining)) + if not chunk: + return None + chunks.append(chunk) + remaining -= len(chunk) + payload = b"".join(chunks) + after = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if ( + (after.st_dev, after.st_ino) != (opened.st_dev, opened.st_ino) + or after.st_size != len(payload) + ): + return None + if not payload and not allow_empty: + return None + return payload, opened + except OSError: + return None + finally: + os.close(descriptor) + + +def _ensure_recovery_directory_at(epoch_fd: int) -> int: + name = "recovery-orphans" + try: + metadata = os.stat(name, dir_fd=epoch_fd, follow_symlinks=False) + except FileNotFoundError: + try: + os.mkdir(name, 0o700, dir_fd=epoch_fd) + os.fsync(epoch_fd) + except OSError as exc: + raise CameraArchiveError("camera recovery directory cannot be created") from exc + except OSError as exc: + raise CameraArchiveError("camera recovery directory cannot be inspected") from exc + else: + if not stat.S_ISDIR(metadata.st_mode): + raise CameraArchiveError("camera recovery directory is not a real directory") + descriptor = _open_directory_at(epoch_fd, name) + if descriptor is None: + raise CameraArchiveError("camera recovery directory failed no-follow validation") + return descriptor + + +def _preserve_recovery_artifact_at( + recovery_fd: int, + name: str, + payload: bytes, +) -> None: + if not payload: + return + destination = _unique_name_at(recovery_fd, name) + _write_bytes_atomic_at(recovery_fd, destination, payload) + + +def _quarantine_orphan_at( + segments_fd: int, + recovery_fd: int, + name: str, +) -> None: + if "/" in name or name in {"", ".", ".."}: + raise CameraArchiveError("camera orphan has an unsafe name") + destination = _unique_name_at(recovery_fd, name) + try: + # dir_fd-relative rename moves the directory entry itself. A malicious + # symlink is quarantined without ever following or reading its target. + os.rename( + name, + destination, + src_dir_fd=segments_fd, + dst_dir_fd=recovery_fd, + ) + os.fsync(segments_fd) + os.fsync(recovery_fd) + except OSError as exc: + raise CameraArchiveError("camera orphan cannot be quarantined safely") from exc + + +def _unique_name_at(directory_fd: int, base: str) -> str: + candidate = base + suffix = 1 + while True: + try: + os.stat(candidate, dir_fd=directory_fd, follow_symlinks=False) + except FileNotFoundError: + return candidate + except OSError as exc: + raise CameraArchiveError("camera recovery destination cannot be inspected") from exc + candidate = f"{base}.{suffix}" + suffix += 1 + + +def _write_bytes_atomic_at(directory_fd: int, name: str, payload: bytes) -> None: + if "/" in name or name in {"", ".", ".."}: + raise CameraArchiveError("camera recovery artifact has an unsafe name") + temp_name = f".{name}.{secrets.token_hex(8)}.tmp" + descriptor: int | None = None + try: + descriptor = os.open( + temp_name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, + dir_fd=directory_fd, + ) + view = memoryview(payload) + while view: + written = os.write(descriptor, view) + if written <= 0: + raise OSError("short write") + view = view[written:] + os.fsync(descriptor) + os.close(descriptor) + descriptor = None + os.replace( + temp_name, + name, + src_dir_fd=directory_fd, + dst_dir_fd=directory_fd, + ) + os.fsync(directory_fd) + except OSError as exc: + raise CameraArchiveError("camera recovery artifact cannot be committed") from exc + finally: + if descriptor is not None: + os.close(descriptor) + try: + os.unlink(temp_name, dir_fd=directory_fd) + except FileNotFoundError: + pass + except OSError: + pass + + +def _write_json_atomic_at(directory_fd: int, name: str, payload: dict[str, Any]) -> None: + encoded = (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode("utf-8") + _write_bytes_atomic_at(directory_fd, name, encoded) + + +def _summary_timestamp_bytes(payload: bytes | None, field: str) -> str | None: + if not payload: + return None + try: + summary = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError): + return None + value = summary.get(field) if isinstance(summary, dict) else None + return value if isinstance(value, str) and len(value) <= 64 else None + + +class _ignore_os_error: + def __enter__(self) -> None: + return None + + def __exit__(self, exc_type: object, exc: object, traceback: object) -> bool: + return isinstance(exc, OSError) diff --git a/src/k1link/web/session_api.py b/src/k1link/web/session_api.py new file mode 100644 index 0000000..aa52b6f --- /dev/null +++ b/src/k1link/web/session_api.py @@ -0,0 +1,1350 @@ +from __future__ import annotations + +import inspect +import re +from collections.abc import Awaitable, Callable, Mapping +from threading import Lock +from typing import Annotated, Any, Literal, Protocol +from urllib.parse import quote + +from fastapi import APIRouter, Body, Header, HTTPException, Query, Response +from fastapi.responses import FileResponse, JSONResponse +from pydantic import BaseModel, ConfigDict, Field, StrictBool, field_validator, model_validator +from starlette.concurrency import run_in_threadpool +from starlette.types import Receive, Scope, Send + +from k1link.sessions import ( + RECORDED_MEDIA_MANIFEST_SCHEMA, + LayoutConflictError, + MaterializedRecording, + RecordedMediaFile, + RecordedMediaInspector, + RecordedMediaManifest, + RecordingMaterializationError, + RecordingPreparationQueueFull, + RecordingPreparationSnapshot, + ReplayCommand, + SessionIntegrityError, + SessionNotFoundError, + SessionNotReplayableError, + SessionRecordingPreparationManager, + SessionStore, + validate_recorded_media_timeline, +) +from k1link.viewer.rerun_bridge import RerunSceneSettings +from k1link.viewer.rrd_export import ( + APPLICATION_ID as RECORDED_APPLICATION_ID, +) +from k1link.viewer.rrd_export import ( + RrdExportError, + recorded_blueprint_rrd, +) + +DEFAULT_REPLAY_PLUGIN_ID = "nodedc.device.xgrids-lixelkity-k1" +DEFAULT_REPLAY_ACTION_ID = "stream.start-replay" +SAFE_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$") +SAFE_SHA256 = re.compile(r"^[a-f0-9]{64}$") +MAX_SAFE_INTEGER = 9_007_199_254_740_991 +REVALIDATED_RECORDING_CACHE_CONTROL = "private, no-cache, no-transform" +IMMUTABLE_RECORDING_CACHE_CONTROL = ( + "private, max-age=31536000, immutable, no-transform" +) + + +class _ReleasingFileResponse(FileResponse): + """Release a cache pin exactly once after every ASGI completion path.""" + + def __init__( + self, + *args: Any, + release: Callable[[], None], + **kwargs: Any, + ) -> None: + super().__init__(*args, **kwargs) + self._release = release + self._release_guard = Lock() + self._released = False + + def _release_once(self) -> None: + with self._release_guard: + if self._released: + return + self._released = True + self._release() + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + try: + await super().__call__(scope, receive, send) + finally: + # Starlette background tasks do not run when `send` is cancelled + # or raises. The finally block makes the response lifetime the + # authoritative pin lifetime for success, disconnect and failure. + self._release_once() + + +class StrictApiModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class ReplayRequest(StrictApiModel): + speed: float = Field(default=1.0, ge=0.0, le=100.0) + loop: bool = False + + +class RecordedBlueprintRequest(StrictApiModel): + application_id: Literal["nodedc_mission_core_recorded"] + recording_id: str = Field( + min_length=1, + max_length=128, + pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$", + ) + accumulation_seconds: float = Field(strict=True, ge=0.0, le=3600.0) + show_points: StrictBool + show_trajectory: StrictBool + show_grid: StrictBool + point_size: float = Field(default=2.5, strict=True, ge=0.1, le=32.0) + palette: Literal["turbo", "viridis", "plasma", "grayscale", "custom"] = "turbo" + custom_color: str = Field(default="#f7f8f4", pattern=r"^#[0-9A-Fa-f]{6}$") + + +class SceneSettingsDocument(StrictApiModel): + projection: Literal["3d", "2d", "map"] + point_size: float = Field(ge=0.1, le=32.0) + color_mode: Literal["intensity", "height", "distance", "rgb", "class"] + palette: Literal["turbo", "viridis", "plasma", "grayscale", "custom"] + custom_color: str = Field(pattern=r"^#[0-9A-Fa-f]{6}$") + accumulation_seconds: float = Field(ge=0.0, le=3600.0) + show_points: bool + show_trajectory: bool + show_grid: bool + show_labels: bool + show_camera_frustums: bool + + +class ToolWindowsDocument(StrictApiModel): + sources_open: bool + display_open: bool + layers_open: bool + order: list[Literal["sources", "display", "layers"]] + + @field_validator("order") + @classmethod + def validate_order(cls, value: list[str]) -> list[str]: + if len(value) != 3 or len(value) != len(set(value)): + raise ValueError("tool window order must be a permutation of all three tools") + return value + + +class NormalizedWindowRect(StrictApiModel): + x: float = Field(ge=0.0, le=1.0) + y: float = Field(ge=0.0, le=1.0) + width: float = Field(gt=0.0, le=1.0) + height: float = Field(gt=0.0, le=1.0) + + @model_validator(mode="after") + def validate_bounds(self) -> NormalizedWindowRect: + epsilon = 1e-9 + if self.x + self.width > 1.0 + epsilon or self.y + self.height > 1.0 + epsilon: + raise ValueError("normalized window rectangle escapes the viewport") + return self + + +class ViewportSize(StrictApiModel): + width: float = Field(gt=0.0, le=100_000.0) + height: float = Field(gt=0.0, le=100_000.0) + + +class LayoutPutRequest(StrictApiModel): + version: Literal[1] + revision: int = Field(ge=0, le=MAX_SAFE_INTEGER) + workspace_id: Literal["observation.spatial"] + scene_settings: SceneSettingsDocument + tool_windows: ToolWindowsDocument + visible_source_ids: list[str] + active_floating_source_id: str | None + window_rects: dict[str, NormalizedWindowRect] + viewport_size: ViewportSize + + @field_validator("visible_source_ids") + @classmethod + def validate_visible_source_ids(cls, value: list[str]) -> list[str]: + if len(value) > 256 or len(value) != len(set(value)): + raise ValueError("visible source identifiers must be unique and bounded") + for source_id in value: + _validate_source_id(source_id) + return value + + @field_validator("active_floating_source_id") + @classmethod + def validate_active_source_id(cls, value: str | None) -> str | None: + if value is not None: + _validate_source_id(value) + return value + + @field_validator("window_rects") + @classmethod + def validate_window_rect_ids( + cls, + value: dict[str, NormalizedWindowRect], + ) -> dict[str, NormalizedWindowRect]: + if len(value) > 256: + raise ValueError("window rectangle catalog exceeds its boundary") + for source_id in value: + _validate_source_id(source_id) + return value + + @model_validator(mode="after") + def validate_active_source_visibility(self) -> LayoutPutRequest: + if ( + self.active_floating_source_id is not None + and self.active_floating_source_id not in self.visible_source_ids + ): + raise ValueError("active floating source must be visible") + return self + + +class ReplayLauncher(Protocol): + def __call__( + self, + command: ReplayCommand, + ) -> Mapping[str, Any] | Awaitable[Mapping[str, Any]]: ... + + +class RecordingMaterializer(Protocol): + def __call__(self, command: ReplayCommand) -> MaterializedRecording: ... + + +class CatalogRefresher(Protocol): + """Synchronous app-owned recovery + discovery transaction hook. + + The host composes hardware-specific recovery before generic session + discovery, then passes the resulting callable here. Return values stay + internal and are intentionally ignored by the API. + """ + + def __call__(self) -> object: ... + + +def build_session_router( + store: SessionStore, + *, + catalog_refresher: CatalogRefresher | None = None, + replay_launcher: ReplayLauncher | None = None, + recording_materializer: RecordingMaterializer | None = None, + recording_preparation_manager: SessionRecordingPreparationManager | None = None, + media_inspector: RecordedMediaInspector | None = None, + allow_synchronous_recording_fallback: bool = False, + replay_plugin_id: str = DEFAULT_REPLAY_PLUGIN_ID, + replay_action_id: str = DEFAULT_REPLAY_ACTION_ID, +) -> APIRouter: + """Build generic host APIs without exposing filesystem locators to clients.""" + + router = APIRouter(tags=["observation-sessions"]) + recorded_media_inspector = media_inspector or RecordedMediaInspector() + + @router.get("/api/v1/observation-sessions") + def list_observation_sessions( + limit: int = Query(default=20, ge=1, le=100), + cursor: str | None = Query(default=None, max_length=128), + ) -> dict[str, Any]: + try: + _refresh_catalog(catalog_refresher) + page = store.list_recent(limit=limit, cursor=cursor) + return { + "items": [ + { + "id": item.session_id, + "label": item.display_name, + "started_at_utc": item.started_at_utc, + "completed_at_utc": item.completed_at_utc, + "status": item.status, + "modalities": list(item.modalities), + "duration_seconds": item.duration_seconds or 0.0, + "replayable": item.replayable, + **( + { + "preparation": _catalog_preparation_document( + store, + recording_preparation_manager, + item.session_id, + item.replayable, + ) + } + if recording_preparation_manager is not None + else {} + ), + } + for item in page.items + if item.started_at_utc is not None + ] + } + except SessionNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=422, detail="Некорректные параметры запроса.") from exc + + @router.get("/api/v1/observation-sessions/{session_id}") + def get_observation_session(session_id: str) -> dict[str, Any]: + try: + _refresh_catalog(catalog_refresher) + return store.get_session(session_id).as_dict() + except SessionNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException( + status_code=422, + detail="Некорректный идентификатор сессии.", + ) from exc + + @router.post("/api/v1/observation-sessions/{session_id}/replay") + async def replay_observation_session( + session_id: str, + request: Annotated[ReplayRequest | None, Body()] = None, + ) -> Any: + replay_request = request or ReplayRequest() + try: + command = await run_in_threadpool( + _prepare_replay, + store, + catalog_refresher, + session_id, + replay_request.speed, + replay_request.loop, + True, + ) + except SessionNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except SessionNotReplayableError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except SessionIntegrityError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=422, detail="Некорректные параметры повтора.") from exc + + if recording_preparation_manager is not None: + try: + # Enqueue performs only catalog identity/lstat checks. A cold + # RRD cache and every camera index are validated exclusively + # by the process-owned worker. + snapshot = recording_preparation_manager.enqueue( + command, + retry_failed=True, + ) + except RecordingPreparationQueueFull as exc: + raise HTTPException( + status_code=503, + detail="Очередь подготовки записей переполнена.", + ) from exc + if snapshot.state != "ready" or snapshot.recording is None: + return _preparation_response(snapshot) + reserved = recording_preparation_manager.reserve_ready( + command.session_id, + preparation_id=snapshot.preparation_id, + ) + if reserved is None or reserved.recording is None: + try: + replacement = recording_preparation_manager.enqueue( + command, + retry_failed=True, + ) + except RecordingPreparationQueueFull as exc: + raise HTTPException( + status_code=503, + detail="Очередь подготовки записей переполнена.", + ) from exc + return _preparation_response(replacement) + return _recording_launch_document( + reserved.recording, + command, + reserved.recorded_media or (), + ) + + if recording_materializer is not None and allow_synchronous_recording_fallback: + recording = await _materialize(recording_materializer, command) + try: + recorded_media = await run_in_threadpool( + _inspect_recorded_media, + store, + recorded_media_inspector, + command, + recording, + ) + except SessionIntegrityError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + return _recording_launch_document(recording, command, recorded_media) + if recording_materializer is not None: + raise HTTPException( + status_code=503, + detail="Сервис фоновой подготовки записей не настроен.", + ) + + runtime_state: dict[str, Any] | None = None + dispatched = replay_launcher is not None + if replay_launcher is not None: + launched = replay_launcher(command) + result = await launched if inspect.isawaitable(launched) else launched + runtime_state = dict(result) + return { + "schema_version": "missioncore.observation-session-replay/v1", + "launch": { + "kind": "plugin-action", + "plugin_id": replay_plugin_id, + "action_id": replay_action_id, + "session_id": command.session_id, + "speed": command.speed, + "loop": command.loop, + "dispatched": dispatched, + }, + "runtime_state": runtime_state, + } + + @router.get("/api/v1/observation-sessions/{session_id}/recording-preparation") + async def get_recording_preparation( + session_id: str, + if_match: Annotated[str | None, Header(alias="If-Match")] = None, + ) -> Any: + if recording_preparation_manager is None: + raise HTTPException( + status_code=409, + detail="Сервис фоновой подготовки записей не настроен.", + ) + snapshot = recording_preparation_manager.status(session_id) + if snapshot is None: + if if_match is None: + raise HTTPException( + status_code=428, + detail="Укажите ETag подготовки в заголовке If-Match.", + ) + # A polling URL is generation-bound. It must never silently create + # a replacement and bind an old browser poller to the new job. + raise HTTPException( + status_code=412, + detail="Подготовка была заменена более новой операцией.", + ) + _require_matching_preparation(snapshot, if_match) + if snapshot.state == "ready" and snapshot.recording is not None: + reserved = recording_preparation_manager.reserve_ready( + session_id, + preparation_id=snapshot.preparation_id, + ) + if reserved is None or reserved.recording is None: + raise HTTPException( + status_code=412, + detail="Подготовка была заменена более новой операцией.", + ) + snapshot = reserved + return JSONResponse( + content=_recording_launch_document( + reserved.recording, + snapshot.command, + snapshot.recorded_media or (), + ), + headers={ + "Cache-Control": "no-store", + "ETag": _preparation_etag(snapshot.preparation_id), + "X-Content-Type-Options": "nosniff", + }, + ) + return _preparation_response(snapshot) + + @router.delete( + "/api/v1/observation-sessions/{session_id}/recording-preparation", + status_code=204, + ) + def delete_recording_preparation( + session_id: str, + if_match: Annotated[str | None, Header(alias="If-Match")] = None, + ) -> Response: + if recording_preparation_manager is None: + raise HTTPException( + status_code=409, + detail="Сервис фоновой подготовки записей не настроен.", + ) + snapshot = recording_preparation_manager.status(session_id) + if snapshot is None: + raise HTTPException(status_code=404, detail="Подготовка записи не найдена.") + _require_matching_preparation(snapshot, if_match) + recording_preparation_manager.cancel( + session_id, + preparation_id=snapshot.preparation_id, + ) + return Response(status_code=204) + + @router.get("/api/v1/observation-sessions/{session_id}/recording.rrd") + async def get_observation_session_recording( + session_id: str, + generation: Annotated[str | None, Query(max_length=64)] = None, + if_match: Annotated[str | None, Header(alias="If-Match")] = None, + if_none_match: Annotated[str | None, Header()] = None, + ) -> Response: + if generation is not None and SAFE_SHA256.fullmatch(generation) is None: + raise HTTPException( + status_code=412, + detail="Поколение подготовленной записи не совпадает.", + ) + if recording_materializer is None: + raise HTTPException( + status_code=409, + detail="Сервис seekable-записей не настроен.", + ) + release_recording: Callable[[], None] | None + if recording_preparation_manager is not None: + if if_match is None and generation is None: + raise HTTPException( + status_code=428, + detail="Укажите digest записи через generation или If-Match.", + ) + snapshot = recording_preparation_manager.status(session_id) + if snapshot is None: + try: + command = await run_in_threadpool( + _prepare_replay, + store, + catalog_refresher, + session_id, + 1.0, + False, + False, + ) + snapshot = recording_preparation_manager.enqueue(command) + except SessionNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except (SessionNotReplayableError, SessionIntegrityError) as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except RecordingPreparationQueueFull as exc: + raise HTTPException( + status_code=503, + detail="Очередь подготовки записей переполнена.", + ) from exc + except ValueError as exc: + raise HTTPException( + status_code=422, + detail="Некорректный идентификатор сессии.", + ) from exc + if snapshot.state in { + "queued", + "validating", + "exporting", + "finalizing", + }: + return _preparation_response(snapshot) + if snapshot.state != "ready" or snapshot.recording is None: + return _preparation_response(snapshot) + recording = snapshot.recording + etag = f'"sha256:{recording.sha256}"' + if generation is not None: + _require_matching_recording_generation(recording.sha256, generation) + if if_match is not None: + _require_matching_digest(etag, if_match) + pinned = recording_preparation_manager.pin_ready( + session_id, + preparation_id=snapshot.preparation_id, + ) + if pinned is None: + raise HTTPException( + status_code=412, + detail="Подготовленная запись была заменена.", + ) + snapshot, pinned_release = pinned + release_recording = pinned_release + pinned_recording = snapshot.recording + if pinned_recording is None: + release_recording() + raise HTTPException( + status_code=500, + detail="Подготовленная запись сессии недоступна.", + ) + recording = pinned_recording + elif allow_synchronous_recording_fallback: + try: + command = await run_in_threadpool( + _prepare_replay, + store, + catalog_refresher, + session_id, + 1.0, + False, + False, + ) + except SessionNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except (SessionNotReplayableError, SessionIntegrityError) as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException( + status_code=422, + detail="Некорректный идентификатор сессии.", + ) from exc + recording, release_recording = await _materialize_pinned( + recording_materializer, + command, + ) + try: + if generation is not None: + _require_matching_recording_generation(recording.sha256, generation) + if if_match is not None: + _require_matching_digest(f'"sha256:{recording.sha256}"', if_match) + except HTTPException: + if release_recording is not None: + release_recording() + raise + else: + raise HTTPException( + status_code=503, + detail="Сервис фоновой подготовки записей не настроен.", + ) + etag = f'"sha256:{recording.sha256}"' + cache_control = ( + IMMUTABLE_RECORDING_CACHE_CONTROL + if generation is not None + else REVALIDATED_RECORDING_CACHE_CONTROL + ) + if _if_none_match_matches(if_none_match, etag): + if release_recording is not None: + release_recording() + return Response( + status_code=304, + headers={ + "Cache-Control": cache_control, + "ETag": etag, + "X-Content-Type-Options": "nosniff", + }, + ) + try: + stat_result = recording.path.stat() + except OSError as exc: + if release_recording is not None: + release_recording() + raise HTTPException( + status_code=500, + detail="Подготовленная запись сессии недоступна.", + ) from exc + response_type = _ReleasingFileResponse if release_recording is not None else FileResponse + response_kwargs: dict[str, Any] = {} + if release_recording is not None: + response_kwargs["release"] = release_recording + return response_type( + recording.path, + media_type=recording.media_type, + filename=f"{recording.session_id}.rrd", + content_disposition_type="inline", + stat_result=stat_result, + headers={ + "Cache-Control": cache_control, + "ETag": etag, + "Content-Length": str(recording.byte_length), + "X-Content-Type-Options": "nosniff", + }, + **response_kwargs, + ) + + @router.post("/api/v1/observation-sessions/{session_id}/blueprint.rrd") + async def get_observation_session_blueprint( + session_id: str, + request: RecordedBlueprintRequest, + ) -> Response: + try: + await run_in_threadpool( + _prepare_replay, + store, + catalog_refresher, + session_id, + 1.0, + False, + False, + ) + payload = await run_in_threadpool( + recorded_blueprint_rrd, + RerunSceneSettings( + point_size=request.point_size, + palette=request.palette, + custom_color=request.custom_color, + accumulation_seconds=request.accumulation_seconds, + show_points=request.show_points, + show_trajectory=request.show_trajectory, + show_grid=request.show_grid, + ), + application_id=RECORDED_APPLICATION_ID, + recording_id=request.recording_id, + ) + except SessionNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except (SessionNotReplayableError, SessionIntegrityError) as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except RrdExportError as exc: + raise HTTPException( + status_code=500, + detail="Не удалось подготовить настройки визуализатора.", + ) from exc + except ValueError as exc: + raise HTTPException( + status_code=422, + detail="Некорректный идентификатор сессии.", + ) from exc + return Response( + content=payload, + media_type="application/vnd.rerun.rrd", + headers={ + "Cache-Control": "no-store", + "X-Content-Type-Options": "nosniff", + "Content-Disposition": 'inline; filename="blueprint.rrd"', + }, + ) + + @router.get("/api/v1/observation-sessions/{session_id}/media/{artifact_id}/manifest") + def get_recorded_media_manifest( + session_id: str, + artifact_id: str, + if_match: Annotated[str | None, Header(alias="If-Match")] = None, + ) -> JSONResponse: + manifest = _resolve_recorded_media_manifest( + store, + recorded_media_inspector, + recording_preparation_manager, + allow_synchronous_recording_fallback, + catalog_refresher, + session_id, + artifact_id, + ) + manifest_etag = f'"sha256:{manifest.generation_sha256}"' + _require_matching_digest(manifest_etag, if_match) + return JSONResponse( + content=_recorded_media_manifest_document(manifest), + headers={ + "Cache-Control": "private, no-cache", + "ETag": manifest_etag, + "X-Content-Type-Options": "nosniff", + }, + ) + + @router.get( + "/api/v1/observation-sessions/{session_id}/media/{artifact_id}/" + "epochs/{epoch_ordinal}/init.mp4" + ) + def get_recorded_media_init( + session_id: str, + artifact_id: str, + epoch_ordinal: int, + if_match: Annotated[str | None, Header(alias="If-Match")] = None, + range_header: Annotated[str | None, Header(alias="Range")] = None, + ) -> Response: + manifest = _resolve_recorded_media_manifest( + store, + recorded_media_inspector, + recording_preparation_manager, + allow_synchronous_recording_fallback, + catalog_refresher, + session_id, + artifact_id, + ) + expected_sha256 = _recorded_media_init_sha256(manifest, epoch_ordinal) + _require_matching_digest(f'"sha256:{expected_sha256}"', if_match) + try: + media_file = recorded_media_inspector.get_init(manifest, epoch_ordinal) + except SessionIntegrityError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + return _recorded_media_file_response(media_file, range_header) + + @router.get( + "/api/v1/observation-sessions/{session_id}/media/{artifact_id}/" + "epochs/{epoch_ordinal}/segments/{segment_sequence}.m4s" + ) + def get_recorded_media_segment( + session_id: str, + artifact_id: str, + epoch_ordinal: int, + segment_sequence: int, + if_match: Annotated[str | None, Header(alias="If-Match")] = None, + range_header: Annotated[str | None, Header(alias="Range")] = None, + ) -> Response: + manifest = _resolve_recorded_media_manifest( + store, + recorded_media_inspector, + recording_preparation_manager, + allow_synchronous_recording_fallback, + catalog_refresher, + session_id, + artifact_id, + ) + expected_sha256 = _recorded_media_segment_sha256( + manifest, + epoch_ordinal, + segment_sequence, + ) + _require_matching_digest(f'"sha256:{expected_sha256}"', if_match) + try: + media_file = recorded_media_inspector.get_segment( + manifest, + epoch_ordinal, + segment_sequence, + ) + except SessionIntegrityError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + return _recorded_media_file_response(media_file, range_header) + + @router.get("/api/v1/workspace-layouts/{workspace_id}") + def get_workspace_layout(workspace_id: str, response: Response) -> dict[str, Any]: + try: + stored = store.get_layout(workspace_id) + document = _layout_document(stored.workspace_id, stored.revision, stored.layout) + response.headers["ETag"] = f'"{stored.revision}"' + return document + except SessionNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=422, detail="Некорректный workspace id.") from exc + + @router.put("/api/v1/workspace-layouts/{workspace_id}") + def put_workspace_layout( + workspace_id: str, + request: LayoutPutRequest, + response: Response, + if_match: Annotated[str, Header(alias="If-Match")], + ) -> dict[str, Any]: + header_revision = _parse_if_match(if_match) + if workspace_id != request.workspace_id: + raise HTTPException(status_code=422, detail="Workspace id не совпадает с документом.") + if header_revision != request.revision: + raise HTTPException(status_code=412, detail="Ревизия документа устарела.") + payload = request.model_dump( + mode="json", + exclude={"version", "revision", "workspace_id"}, + ) + try: + stored = store.save_layout( + workspace_id, + schema_version=request.version, + expected_revision=request.revision, + name="Пространственная сцена", + layout=payload, + ) + response.headers["ETag"] = f'"{stored.revision}"' + return _layout_document(stored.workspace_id, stored.revision, stored.layout) + except LayoutConflictError as exc: + raise HTTPException(status_code=412, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=422, detail="Некорректная компоновка.") from exc + + return router + + +def _refresh_catalog(refresher: CatalogRefresher | None) -> None: + if refresher is None: + return + try: + result = refresher() + if inspect.isawaitable(result): + if inspect.iscoroutine(result): + result.close() + raise TypeError("catalog refresher must be synchronous") + except Exception as exc: + raise HTTPException( + status_code=503, + detail="Не удалось обновить каталог сохранённых сессий.", + ) from exc + + +def _prepare_replay( + store: SessionStore, + catalog_refresher: CatalogRefresher | None, + session_id: str, + speed: float, + loop: bool, + refresh_first: bool, +) -> ReplayCommand: + if refresh_first: + _refresh_catalog(catalog_refresher) + try: + return store.prepare_replay(session_id, speed=speed, loop=loop) + except SessionNotFoundError: + if refresh_first or catalog_refresher is None: + raise + _refresh_catalog(catalog_refresher) + return store.prepare_replay(session_id, speed=speed, loop=loop) + + +def _validate_source_id(source_id: str) -> None: + if SAFE_SOURCE_ID.fullmatch(source_id) is None: + raise ValueError("source identifier has an invalid shape") + + +def _parse_if_match(value: str) -> int: + normalized = value.strip() + if len(normalized) < 3 or normalized[0] != '"' or normalized[-1] != '"': + raise HTTPException(status_code=422, detail="If-Match должен содержать quoted revision.") + revision = normalized[1:-1] + if not revision.isascii() or not revision.isdecimal(): + raise HTTPException(status_code=422, detail="If-Match содержит некорректную ревизию.") + return int(revision) + + +def _layout_document( + workspace_id: str, + revision: int, + layout: dict[str, Any], +) -> dict[str, Any]: + try: + document = LayoutPutRequest.model_validate( + { + "version": 1, + "revision": revision, + "workspace_id": workspace_id, + **layout, + } + ) + except ValueError as exc: + raise SessionIntegrityError("stored workspace layout violates schema version 1") from exc + return document.model_dump(mode="json") + + +ReplayLauncherCallable = Callable[ + [ReplayCommand], + Mapping[str, Any] | Awaitable[Mapping[str, Any]], +] + + +async def _materialize( + materializer: RecordingMaterializer, + command: ReplayCommand, +) -> MaterializedRecording: + try: + recording = await run_in_threadpool(materializer, command) + except RecordingMaterializationError as exc: + raise HTTPException( + status_code=500, + detail="Не удалось подготовить запись сессии.", + ) from exc + if not isinstance(recording, MaterializedRecording): + raise HTTPException( + status_code=500, + detail="Сервис записи вернул некорректный результат.", + ) + return recording + + +async def _materialize_pinned( + materializer: RecordingMaterializer, + command: ReplayCommand, +) -> tuple[MaterializedRecording, Callable[[], None] | None]: + pin = getattr(materializer, "materialize_pinned", None) + if not callable(pin): + return await _materialize(materializer, command), None + try: + result = await run_in_threadpool(pin, command) + except RecordingMaterializationError as exc: + raise HTTPException( + status_code=500, + detail="Не удалось подготовить запись сессии.", + ) from exc + if ( + not isinstance(result, tuple) + or len(result) != 2 + or not isinstance(result[0], MaterializedRecording) + or not callable(result[1]) + ): + raise HTTPException( + status_code=500, + detail="Сервис записи вернул некорректный pinned-результат.", + ) + return result[0], result[1] + + +def _preparation_response(snapshot: RecordingPreparationSnapshot) -> JSONResponse: + encoded_session_id = quote(snapshot.session_id, safe="") + status_url = f"/api/v1/observation-sessions/{encoded_session_id}/recording-preparation" + preparation: dict[str, Any] = { + "preparation_id": snapshot.preparation_id, + "session_id": snapshot.session_id, + "state": snapshot.state, + "progress": snapshot.progress, + "updated_at_utc": snapshot.updated_at_utc, + "status_url": status_url, + "cancellable": snapshot.cancellable, + } + terminal_failure = snapshot.state in {"failed", "cancelled"} + if terminal_failure: + preparation["retryable"] = snapshot.retryable + preparation["error"] = snapshot.error or "Подготовка записи отменена." + headers = { + "Location": status_url, + "Cache-Control": "no-store", + "ETag": _preparation_etag(snapshot.preparation_id), + "X-Content-Type-Options": "nosniff", + } + if not terminal_failure: + headers["Retry-After"] = "1" + return JSONResponse( + status_code=409 if terminal_failure else 202, + content={ + "schema_version": "missioncore.observation-session-preparation/v1", + "preparation": preparation, + }, + headers=headers, + ) + + +def _preparation_etag(preparation_id: str) -> str: + return f'"{preparation_id}"' + + +def _require_matching_preparation( + snapshot: RecordingPreparationSnapshot, + if_match: str | None, +) -> None: + if if_match is None: + raise HTTPException( + status_code=428, + detail="Укажите ETag подготовки в заголовке If-Match.", + ) + if if_match.strip() != _preparation_etag(snapshot.preparation_id): + raise HTTPException( + status_code=412, + detail="Подготовка была заменена более новой операцией.", + ) + + +def _require_matching_digest(expected_etag: str, if_match: str | None) -> None: + if if_match is None: + raise HTTPException( + status_code=428, + detail="Укажите digest поколения в заголовке If-Match.", + ) + if if_match.strip() != expected_etag: + raise HTTPException( + status_code=412, + detail="Запрошенное поколение данных устарело.", + ) + + +def _require_matching_recording_generation( + expected_sha256: str, + generation: str, +) -> None: + if SAFE_SHA256.fullmatch(generation) is None or generation != expected_sha256: + raise HTTPException( + status_code=412, + detail="Поколение подготовленной записи не совпадает.", + ) + + +def _catalog_preparation_document( + store: SessionStore, + manager: SessionRecordingPreparationManager | None, + session_id: str, + replayable: bool, +) -> dict[str, Any] | None: + if manager is None or not replayable: + return None + snapshot = manager.status(session_id) + if snapshot is None: + try: + snapshot = manager.enqueue(store.prepare_replay(session_id)) + except ( + RecordingPreparationQueueFull, + SessionNotFoundError, + SessionNotReplayableError, + SessionIntegrityError, + ValueError, + ): + return None + document: dict[str, Any] = { + "preparation_id": snapshot.preparation_id, + "state": snapshot.state, + "progress": snapshot.progress, + "updated_at_utc": snapshot.updated_at_utc, + "cancellable": snapshot.cancellable, + "retryable": snapshot.retryable, + } + if snapshot.error is not None: + document["error"] = snapshot.error + return document + + +def _recording_launch_document( + recording: MaterializedRecording, + command: ReplayCommand, + recorded_media: tuple[RecordedMediaManifest, ...] = (), +) -> dict[str, Any]: + encoded_session_id = quote(recording.session_id, safe="") + source_url = f"/api/v1/observation-sessions/{encoded_session_id}/recording.rrd" + return { + "schema_version": "missioncore.observation-session-replay/v2", + "launch": { + "kind": "rerun-recording", + "session_id": recording.session_id, + "source_url": source_url, + "viewer_source_url": f"{source_url}?generation={recording.sha256}", + "media_type": recording.media_type, + "timeline": recording.timeline, + "timeline_start_seconds": recording.timeline_start_ns / 1_000_000_000, + "timeline_end_seconds": recording.timeline_end_ns / 1_000_000_000, + "seekable": True, + "byte_length": recording.byte_length, + "sha256": recording.sha256, + "playback": { + "speed": command.speed, + "loop": command.loop, + }, + "media_sources": [ + _recorded_media_launch_source( + manifest, + index=index, + ) + for index, manifest in enumerate(recorded_media, start=1) + ], + }, + } + + +def _inspect_recorded_media( + store: SessionStore, + inspector: RecordedMediaInspector, + command: ReplayCommand, + recording: MaterializedRecording, +) -> tuple[RecordedMediaManifest, ...]: + manifests = tuple( + inspector.inspect(artifact, command) + for artifact in store.list_recorded_media(command.session_id) + ) + validate_recorded_media_timeline( + manifests, + recording_start_seconds=recording.timeline_start_ns / 1_000_000_000, + recording_end_seconds=recording.timeline_end_ns / 1_000_000_000, + ) + return manifests + + +def _recorded_media_launch_source( + manifest: RecordedMediaManifest, + *, + index: int, +) -> dict[str, Any]: + encoded_session_id = quote(manifest.session_id, safe="") + encoded_artifact_id = quote(manifest.artifact_id, safe="") + return { + "id": manifest.public_source_id, + "label": f"Записанная камера {index}", + "modality": "video", + "manifest_url": ( + f"/api/v1/observation-sessions/{encoded_session_id}/media/" + f"{encoded_artifact_id}/manifest" + ), + "media_type": "video/mp4", + "manifest_generation_sha256": manifest.generation_sha256, + "byte_length": manifest.byte_length, + "timeline_start_seconds": manifest.timeline_start_seconds, + "timeline_end_seconds": manifest.timeline_end_seconds, + "seekable": True, + "synchronization": manifest.synchronization, + } + + +def _recorded_media_manifest_document( + manifest: RecordedMediaManifest, +) -> dict[str, Any]: + encoded_session_id = quote(manifest.session_id, safe="") + encoded_artifact_id = quote(manifest.artifact_id, safe="") + base = f"/api/v1/observation-sessions/{encoded_session_id}/media/{encoded_artifact_id}" + return { + "schema_version": RECORDED_MEDIA_MANIFEST_SCHEMA, + "source_id": manifest.public_source_id, + "generation_sha256": manifest.generation_sha256, + "byte_length": manifest.byte_length, + "timeline_start_seconds": manifest.timeline_start_seconds, + "timeline_end_seconds": manifest.timeline_end_seconds, + "synchronization": manifest.synchronization, + "epochs": [ + { + "ordinal": epoch.ordinal, + "timeline_start_seconds": epoch.timeline_start_seconds, + "timeline_end_seconds": epoch.timeline_end_seconds, + "media_type": epoch.media_type, + "init_url": f"{base}/epochs/{epoch.ordinal}/init.mp4", + "init_byte_length": epoch.init_byte_length, + "init_sha256": epoch.init_sha256, + "segment_count": len(epoch.segments), + "segment_url_prefix": f"{base}/epochs/{epoch.ordinal}/segments/", + "segments": [ + { + "sequence": segment.sequence, + "url": ( + f"{base}/epochs/{epoch.ordinal}/segments/" + f"{segment.sequence}.m4s" + ), + "byte_length": segment.byte_length, + "sha256": segment.sha256, + } + for segment in epoch.segments + ], + } + for epoch in manifest.epochs + ], + } + + +def _resolve_recorded_media_manifest( + store: SessionStore, + inspector: RecordedMediaInspector, + manager: SessionRecordingPreparationManager | None, + allow_synchronous_fallback: bool, + catalog_refresher: CatalogRefresher | None, + session_id: str, + artifact_id: str, +) -> RecordedMediaManifest: + if manager is not None: + try: + _validate_source_id(artifact_id) + except ValueError as exc: + raise HTTPException( + status_code=422, + detail="Некорректный идентификатор записанного медиаканала.", + ) from exc + snapshot = manager.status(session_id) + if ( + snapshot is None + or snapshot.state != "ready" + or snapshot.recorded_media is None + ): + raise HTTPException( + status_code=409, + detail="Медиаканал ещё не подготовлен.", + ) + matches = tuple( + manifest + for manifest in snapshot.recorded_media + if manifest.artifact_id == artifact_id + ) + if len(matches) != 1: + raise HTTPException(status_code=404, detail="Записанный медиаканал не найден.") + return matches[0] + if not allow_synchronous_fallback: + raise HTTPException( + status_code=503, + detail="Сервис фоновой подготовки медиаканалов не настроен.", + ) + try: + command = _prepare_replay( + store, + catalog_refresher, + session_id, + 1.0, + False, + False, + ) + artifact = store.get_recorded_media(session_id, artifact_id) + return inspector.inspect(artifact, command) + except SessionNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except (SessionNotReplayableError, SessionIntegrityError) as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException( + status_code=422, + detail="Некорректный идентификатор записанного медиаканала.", + ) from exc + + +def _recorded_media_init_sha256( + manifest: RecordedMediaManifest, + epoch_ordinal: int, +) -> str: + matches = tuple(epoch for epoch in manifest.epochs if epoch.ordinal == epoch_ordinal) + if len(matches) != 1: + raise HTTPException(status_code=404, detail="Эпоха записанного медиаканала не найдена.") + return matches[0].init_sha256 + + +def _recorded_media_segment_sha256( + manifest: RecordedMediaManifest, + epoch_ordinal: int, + segment_sequence: int, +) -> str: + matches = tuple(epoch for epoch in manifest.epochs if epoch.ordinal == epoch_ordinal) + if len(matches) != 1: + raise HTTPException(status_code=404, detail="Эпоха записанного медиаканала не найдена.") + segments = tuple( + segment for segment in matches[0].segments if segment.sequence == segment_sequence + ) + if len(segments) != 1: + raise HTTPException(status_code=404, detail="Сегмент записанного медиаканала не найден.") + return segments[0].sha256 + + +def _if_none_match_matches(value: str | None, etag: str) -> bool: + if value is None: + return False + # RFC 9110 requires weak comparison for If-None-Match. The recording ETag + # is content-addressed, so accepting its weak spelling is safe and lets a + # browser reuse an already-downloaded multi-hundred-megabyte RRD. + expected = etag.removeprefix("W/") + for candidate in value.split(","): + normalized = candidate.strip() + if normalized == "*" or normalized.removeprefix("W/") == expected: + return True + return False + + +def _recorded_media_file_response( + media_file: RecordedMediaFile, + range_header: str | None, +) -> Response: + payload = media_file.payload + status_code = 200 + content_range: str | None = None + if range_header is not None: + start, end = _parse_byte_range(range_header, len(payload)) + payload = payload[start : end + 1] + status_code = 206 + content_range = f"bytes {start}-{end}/{media_file.byte_length}" + headers = { + "Accept-Ranges": "bytes", + "Cache-Control": "private, max-age=31536000, immutable, no-transform", + "ETag": f'"sha256:{media_file.sha256}"', + "Content-Length": str(len(payload)), + "X-Content-Type-Options": "nosniff", + "Content-Disposition": f'inline; filename="{media_file.filename}"', + } + if content_range is not None: + headers["Content-Range"] = content_range + return Response( + content=payload, + status_code=status_code, + media_type=media_file.media_type, + headers=headers, + ) + + +def _parse_byte_range(value: str, byte_length: int) -> tuple[int, int]: + match = re.fullmatch(r"bytes=(\d*)-(\d*)", value.strip()) + if match is None or byte_length < 1: + raise HTTPException( + status_code=416, + detail="Некорректный Range для записанного медиасегмента.", + headers={"Content-Range": f"bytes */{byte_length}"}, + ) + raw_start, raw_end = match.groups() + if not raw_start and not raw_end: + raise HTTPException( + status_code=416, + detail="Некорректный Range для записанного медиасегмента.", + headers={"Content-Range": f"bytes */{byte_length}"}, + ) + if not raw_start: + suffix = int(raw_end) + if suffix < 1: + raise HTTPException( + status_code=416, + detail="Некорректный Range для записанного медиасегмента.", + headers={"Content-Range": f"bytes */{byte_length}"}, + ) + start = max(0, byte_length - suffix) + return start, byte_length - 1 + start = int(raw_start) + end = byte_length - 1 if not raw_end else int(raw_end) + if start >= byte_length or end < start: + raise HTTPException( + status_code=416, + detail="Range выходит за границы записанного медиасегмента.", + headers={"Content-Range": f"bytes */{byte_length}"}, + ) + return start, min(end, byte_length - 1) diff --git a/tests/test_active_session_lease.py b/tests/test_active_session_lease.py new file mode 100644 index 0000000..f2bb55f --- /dev/null +++ b/tests/test_active_session_lease.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from pathlib import Path + +from k1link.sessions import ActiveSessionLease, recover_stale_active_session_marker +from k1link.sessions.legacy import discover_legacy_viewer_sessions + + +def test_active_session_lease_hides_live_evidence_until_release(tmp_path: Path) -> None: + sessions = tmp_path / "sessions" + session = sessions / "20260717T011050Z_viewer_live" + + lease = ActiveSessionLease.acquire(sessions, session) + try: + session.mkdir() + capture = session / "captures" / "mqtt_live" + capture.mkdir(parents=True) + (capture / "mqtt.raw.k1mqtt").write_bytes(b"K1MQTT\x00unfinished") + candidates = discover_legacy_viewer_sessions(sessions) + assert candidates == () + assert recover_stale_active_session_marker(sessions) is False + finally: + lease.release() + + assert not (sessions / ".current_session").exists() + candidates = discover_legacy_viewer_sessions(sessions) + assert len(candidates) == 1 + assert candidates[0].session_id == session.name + assert candidates[0].replayable is False + + +def test_startup_recovery_removes_only_an_unlocked_stale_marker(tmp_path: Path) -> None: + sessions = tmp_path / "sessions" + sessions.mkdir() + marker = sessions / ".current_session" + marker.write_text("20260717T011050Z_viewer_live\n", encoding="utf-8") + + assert recover_stale_active_session_marker(sessions) is True + assert not marker.exists() + assert recover_stale_active_session_marker(sessions) is False diff --git a/tests/test_camera_archive.py b/tests/test_camera_archive.py new file mode 100644 index 0000000..5e1b444 --- /dev/null +++ b/tests/test_camera_archive.py @@ -0,0 +1,338 @@ +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from k1link.sessions.legacy import discover_legacy_viewer_sessions +from k1link.web.camera_archive import ( + CAMERA_ARCHIVE_SCHEMA, + CAMERA_COMMIT_POLICY, + CameraArchiveError, + CameraArchiveWriter, + recover_incomplete_camera_archives, +) + + +def test_camera_archive_writes_canonical_segments_and_seals_summary(tmp_path: Path) -> None: + session = tmp_path / "session" + session.mkdir() + writer = CameraArchiveWriter( + session, + "sensor.camera.left", + 7, + commit_interval_seconds=60, + commit_bytes=1024, + ) + + init = b"init-segment" + first = b"first-fragment" + second = b"second-fragment" + writer.append("init", init, host_epoch_ns=100, host_monotonic_ns=200) + writer.append("media", first, host_epoch_ns=110, host_monotonic_ns=210) + writer.append("media", second, host_epoch_ns=120, host_monotonic_ns=220) + + # The tuning values cannot weaken the segment RPO: every append has already + # published its fragment, index record, and interrupted crash checkpoint. + checkpoint = json.loads(writer.summary_path.read_text(encoding="utf-8")) + assert checkpoint["status"] == "interrupted" + assert checkpoint["segment_count"] == 2 + assert checkpoint["commit_policy"] == CAMERA_COMMIT_POLICY + + summary = writer.close() + entries = [json.loads(line) for line in writer.index_path.read_text().splitlines()] + assert writer.archive_dir.name == "epoch-7" + assert writer.init_path.read_bytes() == init + assert [path.name for path in sorted(writer.segments_dir.iterdir())] == [ + "1.m4s", + "2.m4s", + ] + assert [entry["kind"] for entry in entries] == ["media", "media"] + assert [entry["sequence"] for entry in entries] == [1, 2] + assert [entry["path"] for entry in entries] == [ + "segments/1.m4s", + "segments/2.m4s", + ] + for entry, expected in zip(entries, (first, second), strict=True): + payload = (writer.archive_dir / entry["path"]).read_bytes() + assert payload == expected + assert entry["length"] == len(expected) + assert entry["sha256"] == hashlib.sha256(expected).hexdigest() + assert summary["schema_version"] == CAMERA_ARCHIVE_SCHEMA + assert summary["status"] == "complete" + assert summary["segment_count"] == 2 + assert summary["entry_count"] == 2 + assert summary["media_segment_count"] == 2 + assert summary["valid_bytes"] == len(init) + len(first) + len(second) + assert summary["stream_sha256"] == hashlib.sha256(init + first + second).hexdigest() + assert summary["synchronization"] == "host-arrival-best-effort" + assert summary["artifacts"] == { + "init": "init.mp4", + "segments": "segments", + "index": "index.jsonl", + } + assert "192.168" not in json.dumps(summary) + assert writer.close() == summary + + +def test_camera_archive_rejects_media_without_init_and_existing_epoch(tmp_path: Path) -> None: + session = tmp_path / "session" + session.mkdir() + writer = CameraArchiveWriter(session, "sensor.camera.right", 1) + with pytest.raises(CameraArchiveError, match="before"): + writer.append("media", b"fragment") + writer.append("init", b"init") + writer.close(status="interrupted", failure_code="source-ended") + + with pytest.raises(FileExistsError): + CameraArchiveWriter(session, "sensor.camera.right", 1) + with pytest.raises(ValueError, match="safe storage"): + CameraArchiveWriter(session, "../camera", 2) + + +@pytest.mark.parametrize("symlink_component", ["media", "source"]) +def test_camera_writer_never_follows_precreated_storage_symlinks( + tmp_path: Path, + symlink_component: str, +) -> None: + session = tmp_path / "session" + session.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + if symlink_component == "media": + (session / "media").symlink_to(outside, target_is_directory=True) + else: + media = session / "media" + media.mkdir() + (media / "sensor.camera.left").symlink_to(outside, target_is_directory=True) + + with pytest.raises(CameraArchiveError, match="no-follow"): + CameraArchiveWriter(session, "sensor.camera.left", 1) + + assert list(outside.iterdir()) == [] + + +def test_camera_writer_keeps_directory_fds_across_path_swap(tmp_path: Path) -> None: + session = tmp_path / "session" + session.mkdir() + writer = CameraArchiveWriter(session, "sensor.camera.left", 1) + writer.append("init", b"init") + + original_epoch = writer.archive_dir + moved_epoch = original_epoch.with_name("epoch-1-moved") + original_epoch.rename(moved_epoch) + outside_epoch = tmp_path / "outside-epoch" + outside_epoch.mkdir() + original_epoch.symlink_to(outside_epoch, target_is_directory=True) + + original_segments = moved_epoch / "segments" + moved_segments = moved_epoch / "segments-held-by-fd" + original_segments.rename(moved_segments) + outside_segments = tmp_path / "outside-segments" + outside_segments.mkdir() + original_segments.symlink_to(outside_segments, target_is_directory=True) + + writer.append("media", b"frame-after-path-swap") + summary = writer.close() + + assert (moved_segments / "1.m4s").read_bytes() == b"frame-after-path-swap" + assert json.loads((moved_epoch / "summary.json").read_text(encoding="utf-8")) == summary + assert list(outside_epoch.iterdir()) == [] + assert list(outside_segments.iterdir()) == [] + + +def test_recovery_seals_unindexed_durable_tail_and_preserves_orphans(tmp_path: Path) -> None: + sessions_root = tmp_path / "sessions" + session = sessions_root / "20260717T011050Z_viewer_live" + session.mkdir(parents=True) + writer = CameraArchiveWriter(session, "sensor.camera.left", 3) + writer.append("init", b"init") + writer.append("media", b"frame-1", host_epoch_ns=100, host_monotonic_ns=200) + + # Catalog refresh must not hash or rewrite a writer that is still active in + # this server process. + assert recover_incomplete_camera_archives(sessions_root) == () + writer.close(status="interrupted", failure_code="synthetic-process-crash") + + epoch = writer.archive_dir + writer.summary_path.unlink() + # Simulate a process dying after atomically publishing the next fragment but + # before its JSONL entry, plus a non-contiguous fragment that cannot be put on + # the trusted timeline. Recovery salvages #2 and quarantines (never deletes) #4. + (writer.segments_dir / "2.m4s").write_bytes(b"frame-2") + (writer.segments_dir / "4.m4s").write_bytes(b"orphan-frame") + + recovered = recover_incomplete_camera_archives(sessions_root) + + assert len(recovered) == 1 + summary = recovered[0] + assert summary["status"] == "interrupted" + assert summary["failure_code"] == "server-process-interrupted" + assert summary["segment_count"] == 2 + entries = [ + json.loads(line) + for line in writer.index_path.read_text(encoding="utf-8").splitlines() + ] + assert [entry["sequence"] for entry in entries] == [1, 2] + assert entries[0]["host_epoch_ns"] == 100 + assert entries[1]["recovered"] is True + assert (writer.segments_dir / "2.m4s").read_bytes() == b"frame-2" + assert not (writer.segments_dir / "4.m4s").exists() + assert any( + path.read_bytes() == b"orphan-frame" + for path in (epoch / "recovery-orphans").iterdir() + if path.is_file() + ) + + # Recovery output is exactly the layout consumed by legacy media discovery, + # and a second scan is an idempotent no-op. + assert recover_incomplete_camera_archives(sessions_root) == () + candidates = discover_legacy_viewer_sessions(sessions_root) + assert len(candidates) == 1 + assert [source.source_id for source in candidates[0].media_sources] == [ + "sensor.camera.left" + ] + + +def test_recovery_replaces_index_and_summary_symlinks_without_following_targets( + tmp_path: Path, +) -> None: + sessions_root = tmp_path / "sessions" + session = sessions_root / "20260717T011051Z_viewer_live" + session.mkdir(parents=True) + writer = CameraArchiveWriter(session, "sensor.camera.right", 1) + writer.append("init", b"init") + writer.append("media", b"frame") + writer.close(status="interrupted") + + outside_index = tmp_path / "outside-index" + outside_summary = tmp_path / "outside-summary" + outside_index.write_bytes(b"do-not-read-or-overwrite-index") + outside_summary.write_bytes(b"do-not-read-or-overwrite-summary") + writer.index_path.unlink() + writer.summary_path.unlink() + writer.index_path.symlink_to(outside_index) + writer.summary_path.symlink_to(outside_summary) + + recovered = recover_incomplete_camera_archives(sessions_root) + + assert len(recovered) == 1 + assert recovered[0]["segment_count"] == 1 + assert outside_index.read_bytes() == b"do-not-read-or-overwrite-index" + assert outside_summary.read_bytes() == b"do-not-read-or-overwrite-summary" + assert writer.index_path.is_symlink() is False + assert writer.summary_path.is_symlink() is False + assert json.loads(writer.index_path.read_text(encoding="utf-8"))["recovered"] is True + + +def test_recovery_fails_closed_on_symlinked_quarantine_directory(tmp_path: Path) -> None: + sessions_root = tmp_path / "sessions" + session = sessions_root / "20260717T011052Z_viewer_live" + session.mkdir(parents=True) + writer = CameraArchiveWriter(session, "sensor.camera.left", 1) + writer.append("init", b"init") + writer.append("media", b"frame-1") + writer.close(status="interrupted") + writer.summary_path.unlink() + orphan = writer.segments_dir / "3.m4s" + orphan.write_bytes(b"orphan") + outside = tmp_path / "outside-quarantine" + outside.mkdir() + (writer.archive_dir / "recovery-orphans").symlink_to(outside, target_is_directory=True) + + with pytest.raises(CameraArchiveError, match="real directory"): + recover_incomplete_camera_archives(sessions_root) + + assert list(outside.iterdir()) == [] + assert orphan.read_bytes() == b"orphan" + + +def test_recovery_quarantines_segment_symlink_without_reading_target(tmp_path: Path) -> None: + sessions_root = tmp_path / "sessions" + session = sessions_root / "20260717T011053Z_viewer_live" + session.mkdir(parents=True) + writer = CameraArchiveWriter(session, "sensor.camera.left", 1) + writer.append("init", b"init") + writer.append("media", b"frame-1") + writer.close(status="interrupted") + writer.summary_path.unlink() + outside = tmp_path / "outside-segment" + outside.write_bytes(b"external-evidence-must-not-be-read-or-moved") + (writer.segments_dir / "2.m4s").symlink_to(outside) + + recovered = recover_incomplete_camera_archives(sessions_root) + + assert len(recovered) == 1 + assert recovered[0]["segment_count"] == 1 + assert outside.read_bytes() == b"external-evidence-must-not-be-read-or-moved" + quarantined = list((writer.archive_dir / "recovery-orphans").glob("2.m4s*")) + assert len(quarantined) == 1 + assert quarantined[0].is_symlink() + assert quarantined[0].resolve() == outside.resolve() + + +def test_recovery_lock_symlink_is_rejected_without_touching_target(tmp_path: Path) -> None: + sessions_root = tmp_path / "sessions" + sessions_root.mkdir() + outside = tmp_path / "outside-lock" + outside.write_bytes(b"external-lock-target") + (sessions_root / ".camera-recovery.lock").symlink_to(outside) + + with pytest.raises(CameraArchiveError, match="lock failed no-follow"): + recover_incomplete_camera_archives(sessions_root) + + assert outside.read_bytes() == b"external-lock-target" + + +@pytest.mark.skipif(os.name != "posix", reason="uses POSIX flock to assert worker serialization") +def test_recovery_startup_lease_serializes_another_process(tmp_path: Path) -> None: + import fcntl + + sessions_root = tmp_path / "sessions" + sessions_root.mkdir() + lock_path = sessions_root / ".camera-recovery.lock" + lock_path.touch(mode=0o600) + descriptor = os.open(lock_path, os.O_RDWR) + process: subprocess.Popen[str] | None = None + locked = False + try: + fcntl.flock(descriptor, fcntl.LOCK_EX) + locked = True + process = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "from pathlib import Path; " + "from k1link.web.camera_archive import " + "recover_incomplete_camera_archives; " + "recover_incomplete_camera_archives(Path(__import__('sys').argv[1])); " + "print('recovered')" + ), + str(sessions_root), + ], + cwd=Path(__file__).parents[1], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + time.sleep(0.15) + assert process.poll() is None + fcntl.flock(descriptor, fcntl.LOCK_UN) + locked = False + stdout, stderr = process.communicate(timeout=5) + assert process.returncode == 0, stderr + assert stdout.strip() == "recovered" + finally: + if locked: + fcntl.flock(descriptor, fcntl.LOCK_UN) + os.close(descriptor) + if process is not None and process.poll() is None: + process.kill() + process.wait(timeout=5) diff --git a/tests/test_mqtt_capture.py b/tests/test_mqtt_capture.py index 4a121e4..b4c36d5 100644 --- a/tests/test_mqtt_capture.py +++ b/tests/test_mqtt_capture.py @@ -10,6 +10,7 @@ import pytest from paho.mqtt.packettypes import PacketTypes from paho.mqtt.reasoncodes import ReasonCode +import k1link.mqtt.capture as capture_module from k1link.mqtt.capture import ( FRAME_HEADER, RAW_MAGIC, @@ -256,3 +257,85 @@ def test_capture_reader_rejects_invalid_or_unbounded_frames( with pytest.raises(CaptureFormatError, match=message): list(iter_capture_frames(path, max_payload_bytes=4)) + + +def _mqtt_message(topic: str, payload: bytes) -> mqtt.MQTTMessage: + message = mqtt.MQTTMessage(topic=topic.encode("utf-8")) + message.payload = payload + message.qos = 0 + message.retain = False + message.dup = False + return message + + +def test_group_commit_fsyncs_raw_before_publishing_metadata( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + writer = capture_module._CaptureWriter(tmp_path / "capture", 1024) # noqa: SLF001 + writer.open() + assert writer._raw is not None # noqa: SLF001 + assert writer._metadata is not None # noqa: SLF001 + raw_fd = writer._raw.fileno() # noqa: SLF001 + metadata_fd = writer._metadata.fileno() # noqa: SLF001 + fsync_calls: list[int] = [] + real_fsync = capture_module.os.fsync + + def observe_fsync(descriptor: int) -> None: + fsync_calls.append(descriptor) + real_fsync(descriptor) + + monkeypatch.setattr(capture_module.os, "fsync", observe_fsync) + monkeypatch.setattr(capture_module, "GROUP_COMMIT_MAX_MESSAGES", 2) + + writer.record(_mqtt_message("RealtimePath", b"one")) + assert writer.metadata_path.read_bytes() == b"" + writer.record(_mqtt_message("RealtimePath", b"two")) + + assert fsync_calls[:2] == [raw_fd, metadata_fd] + assert len(writer.metadata_path.read_text(encoding="utf-8").splitlines()) == 2 + writer.close() + + +def test_group_commit_timer_bounds_quiet_stream_rpo( + tmp_path: Path, +) -> None: + writer = capture_module._CaptureWriter(tmp_path / "capture", 1024) # noqa: SLF001 + writer.open() + writer.record(_mqtt_message("RealtimePath", b"one")) + assert writer.metadata_path.read_bytes() == b"" + + writer.maybe_commit( + writer._last_commit_monotonic # noqa: SLF001 + + capture_module.GROUP_COMMIT_INTERVAL_SECONDS + ) + + assert len(writer.metadata_path.read_text(encoding="utf-8").splitlines()) == 1 + writer.close() + + +def test_raw_fsync_failure_never_publishes_metadata_ahead_of_raw( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + writer = capture_module._CaptureWriter(tmp_path / "capture", 1024) # noqa: SLF001 + writer.open() + writer.record(_mqtt_message("RealtimePath", b"one")) + assert writer._raw is not None # noqa: SLF001 + raw_fd = writer._raw.fileno() # noqa: SLF001 + real_fsync = capture_module.os.fsync + + def fail_raw_fsync(descriptor: int) -> None: + if descriptor == raw_fd: + raise OSError("synthetic raw fsync failure") + real_fsync(descriptor) + + monkeypatch.setattr(capture_module.os, "fsync", fail_raw_fsync) + + with pytest.raises(OSError, match="synthetic raw fsync failure"): + writer._commit_pending() # noqa: SLF001 + + assert writer.metadata_path.read_bytes() == b"" + # Restore durability primitive so the fixture can close normally. + monkeypatch.setattr(capture_module.os, "fsync", real_fsync) + writer.close() diff --git a/tests/test_rrd_export.py b/tests/test_rrd_export.py new file mode 100644 index 0000000..5c51a68 --- /dev/null +++ b/tests/test_rrd_export.py @@ -0,0 +1,324 @@ +from __future__ import annotations + +import hashlib +import json +import struct +import subprocess +import sys +import threading +from pathlib import Path + +import pytest + +import k1link.viewer.rrd_export as export_module +from k1link.mqtt.capture import FRAME_HEADER, RAW_MAGIC +from k1link.viewer.rerun_bridge import RerunSceneSettings +from k1link.viewer.rrd_export import ( + RECORDED_POINTS_VISUALIZER_ID, + RECORDED_ROOT_CONTAINER_ID, + RECORDED_SPATIAL_VIEW_ID, + SESSION_TIMELINE, + RrdExportCancelled, + RrdExportError, + _recorded_blueprint, + export_k1mqtt_to_rrd, + recorded_blueprint_rrd, +) + + +def _point_payload(x: float) -> bytes: + return struct.pack(" bytes: + return struct.pack(" Path: + capture = directory / "mqtt.raw.k1mqtt" + raw = bytearray(RAW_MAGIC) + metadata: list[dict[str, object]] = [] + epoch_origin_ns = 1_784_124_315_000_000_000 + monotonic_origin_ns = 9_000_000_000 + for sequence, (topic, payload, session_time_ns) in enumerate(frames, start=1): + topic_raw = topic.encode("utf-8") + raw.extend(FRAME_HEADER.pack(len(topic_raw), len(payload))) + raw.extend(topic_raw) + raw.extend(payload) + metadata.append( + { + "record_type": "message", + "sequence": sequence, + "received_at_epoch_ns": epoch_origin_ns + session_time_ns, + "received_monotonic_ns": monotonic_origin_ns + session_time_ns, + } + ) + capture.write_bytes(raw) + (directory / "mqtt.metadata.jsonl").write_text( + "".join(json.dumps(item) + "\n" for item in metadata), + encoding="utf-8", + ) + return capture + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _print_rrd_entity(path: Path, entity: str) -> str: + completed = subprocess.run( + [ + sys.executable, + "-m", + "rerun_cli", + "rrd", + "print", + "-vvv", + "--entity", + entity, + str(path), + ], + check=True, + capture_output=True, + text=True, + ) + return completed.stdout + + +def test_recorded_blueprint_accumulates_point_frames_on_session_timeline() -> None: + blueprint = _recorded_blueprint( + RerunSceneSettings( + point_size=7.5, + palette="custom", + custom_color="#112233", + accumulation_seconds=18.5, + show_points=False, + show_trajectory=True, + show_grid=False, + ) + ) + spatial_view = blueprint.root_container.contents[0] + visible_ranges = spatial_view.properties["VisibleTimeRanges"] + line_grid = spatial_view.properties["LineGrid3D"] + + assert visible_ranges.ranges.as_arrow_array().to_pylist() == [ + { + "timeline": SESSION_TIMELINE, + "range": { + "start": -18_500_000_000, + "end": 0, + }, + } + ] + assert line_grid.visible.as_arrow_array().to_pylist() == [False] + point_behavior, point_visualizer = spatial_view.visualizer_overrides["/world/points"] + trajectory_behavior = spatial_view.visualizer_overrides["/world/trajectory"] + assert point_behavior.visible.as_arrow_array().to_pylist() == [False] + assert trajectory_behavior.visible.as_arrow_array().to_pylist() == [True] + point_overrides = { + str(batch.component_descriptor()): batch.as_arrow_array().to_pylist() + for batch in point_visualizer.overrides + } + assert point_overrides == { + "Points3D:colors": [0x112233FF], + # Negative values are Rerun's encoding for screen-space UI points. + "Points3D:radii": [-7.5], + } + + +def test_zero_accumulation_uses_latest_frame_instead_of_empty_time_range() -> None: + blueprint = _recorded_blueprint( + RerunSceneSettings(accumulation_seconds=0.0, show_grid=True) + ) + spatial_view = blueprint.root_container.contents[0] + + assert "VisibleTimeRanges" not in spatial_view.properties + point_behavior, point_visualizer = spatial_view.visualizer_overrides["/world/points"] + trajectory_behavior = spatial_view.visualizer_overrides["/world/trajectory"] + assert point_behavior.visible.as_arrow_array().to_pylist() == [True] + assert trajectory_behavior.visible.as_arrow_array().to_pylist() == [True] + point_components = { + str(batch.component_descriptor()) for batch in point_visualizer.overrides + } + assert point_components == {"Points3D:radii"} + + +def test_dynamic_blueprint_reuses_scene_ids_without_playback_mutation() -> None: + first = _recorded_blueprint( + RerunSceneSettings(show_points=False, show_trajectory=True), + include_initial_playback_state=False, + ) + second = _recorded_blueprint( + RerunSceneSettings(show_points=True, show_trajectory=False), + include_initial_playback_state=False, + ) + + assert not hasattr(first, "time_panel") + assert not hasattr(second, "time_panel") + assert first.root_container.id == second.root_container.id == RECORDED_ROOT_CONTAINER_ID + first_view = first.root_container.contents[0] + second_view = second.root_container.contents[0] + assert first_view.id == second_view.id == RECORDED_SPATIAL_VIEW_ID + + first_point_behavior, first_point_visualizer = first_view.visualizer_overrides[ + "/world/points" + ] + second_point_behavior, second_point_visualizer = second_view.visualizer_overrides[ + "/world/points" + ] + first_trajectory = first_view.visualizer_overrides["/world/trajectory"] + second_trajectory = second_view.visualizer_overrides["/world/trajectory"] + assert first_point_visualizer.id == RECORDED_POINTS_VISUALIZER_ID + assert second_point_visualizer.id == RECORDED_POINTS_VISUALIZER_ID + assert first_point_behavior.visible.as_arrow_array().to_pylist() == [False] + assert second_point_behavior.visible.as_arrow_array().to_pylist() == [True] + assert first_trajectory.visible.as_arrow_array().to_pylist() == [True] + assert second_trajectory.visible.as_arrow_array().to_pylist() == [False] + + payload = recorded_blueprint_rrd( + RerunSceneSettings(show_points=False, show_trajectory=False), + recording_id="stable-recording", + ) + assert b"PlayState" not in payload + assert b"play_state" not in payload + assert str(RECORDED_ROOT_CONTAINER_ID).encode() in payload + assert str(RECORDED_SPATIAL_VIEW_ID).encode() in payload + assert str(RECORDED_POINTS_VISUALIZER_ID).encode() in payload + + +def test_export_preserves_every_decodable_frame_and_source_timeline(tmp_path: Path) -> None: + capture = _write_capture( + tmp_path, + [ + ("lixel/application/report/heartbeat", b"opaque", 0), + ("RealtimePointcloud", _point_payload(1.0), 100_000_000), + ("RealtimePath", _pose_payload(1.0), 600_000_000), + ("RealtimePointcloud", _point_payload(2.0), 900_000_000), + ("RealtimePath", _pose_payload(1.2), 1_200_000_000), + ("lixel/application/report/heartbeat", b"opaque-tail", 9_500_000_000), + ], + ) + output = tmp_path / "session.rrd" + + summary = export_k1mqtt_to_rrd(capture, output) + + assert summary["source_messages"] == 6 + assert summary["decoded_messages"] == 4 + assert summary["point_frames"] == 2 + assert summary["pose_frames"] == 2 + assert summary["ignored_messages"] == 2 + assert summary["points"] == 2 + assert summary["trajectory_poses"] == 2 + assert summary["trajectory_updates"] == 2 + assert summary["session_origin_monotonic_ns"] == 9_000_000_000 + assert summary["timeline"] == "session_time" + assert summary["timeline_start_ns"] == 0 + assert summary["timeline_end_ns"] == 1_200_000_000 + assert summary["timeline_span_ns"] == 1_200_000_000 + assert summary["first_decoded_time_ns"] == 100_000_000 + assert summary["last_decoded_time_ns"] == 1_200_000_000 + assert summary["source_sha256"] == _sha256(capture) + assert summary["rrd_bytes"] == output.stat().st_size + assert summary["rrd_sha256"] == _sha256(output) + assert output.stat().st_size > 0 + assert list(tmp_path.glob(".session.rrd.*.tmp")) == [] + + origin_table = _print_rrd_entity(output, "/__mission_core/session_origin") + assert "/__mission_core/session_origin" in origin_table + assert "session_time" in origin_table + assert "P0D" in origin_table + assert "[true]" in origin_table + + +def test_atomic_rename_failure_preserves_existing_recording( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + capture = _write_capture( + tmp_path, + [("RealtimePointcloud", _point_payload(1.0), 0)], + ) + output = tmp_path / "session.rrd" + trusted = b"existing-good-recording" + output.write_bytes(trusted) + + def fail_replace(_source: Path, _destination: Path) -> None: + raise OSError("synthetic rename failure") + + monkeypatch.setattr(export_module.os, "replace", fail_replace) + + with pytest.raises(RrdExportError, match="synthetic rename failure"): + export_k1mqtt_to_rrd(capture, output) + + assert output.read_bytes() == trusted + assert list(tmp_path.glob(".session.rrd.*.tmp")) == [] + + +def test_missing_monotonic_metadata_never_replaces_existing_recording(tmp_path: Path) -> None: + capture = _write_capture( + tmp_path, + [("RealtimePointcloud", _point_payload(1.0), 0)], + ) + (tmp_path / "mqtt.metadata.jsonl").unlink() + output = tmp_path / "session.rrd" + output.write_bytes(b"existing-good-recording") + + with pytest.raises(RrdExportError, match="received_monotonic_ns"): + export_k1mqtt_to_rrd(capture, output) + + assert output.read_bytes() == b"existing-good-recording" + assert list(tmp_path.glob(".session.rrd.*.tmp")) == [] + + +def test_export_preserves_valid_prefix_before_crash_metadata_tail(tmp_path: Path) -> None: + capture = _write_capture( + tmp_path, + [ + ("RealtimePointcloud", _point_payload(1.0), 100_000_000), + ("RealtimePointcloud", _point_payload(2.0), 200_000_000), + ], + ) + metadata_path = tmp_path / "mqtt.metadata.jsonl" + first_line = metadata_path.read_text(encoding="utf-8").splitlines(keepends=True)[0] + metadata_path.write_text( + first_line + '{"record_type":"message"', + encoding="utf-8", + ) + output = tmp_path / "session.rrd" + + summary = export_k1mqtt_to_rrd(capture, output) + + assert summary["source_messages"] == 1 + assert summary["decoded_messages"] == 1 + assert summary["point_frames"] == 1 + assert summary["timeline_end_ns"] == 0 + assert output.stat().st_size > 0 + + +def test_export_honors_cooperative_cancellation_without_publishing(tmp_path: Path) -> None: + capture = _write_capture( + tmp_path, + [("RealtimePointcloud", _point_payload(1.0), 0)], + ) + output = tmp_path / "session.rrd" + output.write_bytes(b"trusted-existing-recording") + cancelled = threading.Event() + cancelled.set() + + with pytest.raises(RrdExportCancelled): + export_k1mqtt_to_rrd(capture, output, cancel_event=cancelled) + + assert output.read_bytes() == b"trusted-existing-recording" + assert list(tmp_path.glob(".session.rrd.*.tmp")) == [] diff --git a/tests/test_session_api.py b/tests/test_session_api.py new file mode 100644 index 0000000..855b0f2 --- /dev/null +++ b/tests/test_session_api.py @@ -0,0 +1,1659 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +import stat +import threading +import time +from collections.abc import Callable +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest +from fastapi import APIRouter, HTTPException, Response +from fastapi.routing import APIRoute + +import k1link.sessions.media as recorded_media_module +from k1link.mqtt.capture import FRAME_HEADER, RAW_MAGIC +from k1link.sessions import ( + MaterializedRecording, + RecordedMediaInspector, + ReplayCommand, + SessionIntegrityError, + SessionRecordingMaterializer, + SessionRecordingPreparationManager, + SessionStore, +) +from k1link.sessions.media import _manifest_generation_sha256 +from k1link.viewer.rerun_bridge import RerunSceneSettings +from k1link.viewer.rrd_export import recorded_blueprint_rrd +from k1link.web.camera_archive import CameraArchiveWriter +from k1link.web.session_api import ( + LayoutPutRequest, + RecordedBlueprintRequest, + ReplayRequest, + build_session_router, +) + + +def make_legacy_session(sessions_root: Path, session_id: str) -> Path: + session = sessions_root / session_id + capture = session / "captures" / "mqtt_live" + capture.mkdir(parents=True) + topic_counts = { + "lixel/application/report/lio_pcl": 1, + "lixel/application/report/lio_pose": 1, + } + raw = bytearray(RAW_MAGIC) + metadata: list[dict[str, object]] = [] + for sequence, topic in enumerate(topic_counts, start=1): + topic_bytes = topic.encode() + payload = b"fixture" + frame_offset = len(raw) + raw.extend(FRAME_HEADER.pack(len(topic_bytes), len(payload))) + raw.extend(topic_bytes) + raw.extend(payload) + metadata.append( + { + "record_type": "message", + "sequence": sequence, + "received_at_epoch_ns": 1_000_000_000 + sequence - 1, + "received_monotonic_ns": 2_000_000_000 + sequence - 1, + "topic": topic, + "payload_bytes": len(payload), + "raw_frame_offset": frame_offset, + "raw_payload_offset": frame_offset + FRAME_HEADER.size + len(topic_bytes), + "raw_frame_bytes": FRAME_HEADER.size + len(topic_bytes) + len(payload), + } + ) + raw_path = capture / "mqtt.raw.k1mqtt" + raw_path.write_bytes(raw) + metadata_path = capture / "mqtt.metadata.jsonl" + metadata_path.write_text( + "".join(json.dumps(record) + "\n" for record in metadata), + encoding="utf-8", + ) + (capture / "mqtt.summary.json").write_text( + json.dumps( + { + "created_at_utc": "2026-07-16T20:56:32.699Z", + "completed_at_utc": "2026-07-16T21:20:43.018Z", + "capture_elapsed_seconds": 1440.1, + "stop_reason": "external_stop", + "error": None, + "message_count": 2, + "raw_bytes": len(raw), + "topic_counts": topic_counts, + "artifact_hashes": { + "raw_sha256": hashlib.sha256(raw).hexdigest(), + "metadata_jsonl_sha256": hashlib.sha256(metadata_path.read_bytes()).hexdigest(), + }, + } + ), + encoding="utf-8", + ) + (session / "manifest.redacted.json").write_text("{}", encoding="utf-8") + return session + + +def make_recorded_h264_fixture( + *, + timescale: int = 1_000, + sample_duration: int = 500, +) -> tuple[bytes, bytes]: + def box(box_type: bytes, payload: bytes = b"") -> bytes: + return (8 + len(payload)).to_bytes(4, "big") + box_type + payload + + def full_box( + box_type: bytes, + payload: bytes = b"", + *, + flags: int = 0, + ) -> bytes: + return box(box_type, bytes([0]) + flags.to_bytes(3, "big") + payload) + + track_id = 1 + tkhd = full_box( + b"tkhd", + b"\x00" * 8 + track_id.to_bytes(4, "big") + b"\x00" * 4, + ) + mdhd = full_box( + b"mdhd", + b"\x00" * 8 + timescale.to_bytes(4, "big") + b"\x00" * 4, + ) + hdlr = full_box(b"hdlr", b"\x00" * 4 + b"vide") + trak = box(b"trak", tkhd + box(b"mdia", mdhd + hdlr)) + trex = full_box( + b"trex", + track_id.to_bytes(4, "big") + + (1).to_bytes(4, "big") + + sample_duration.to_bytes(4, "big") + + b"\x00" * 8, + ) + avcc = box(b"avcC", b"\x01\x64\x00\x28") + init = box(b"ftyp", b"isom") + box(b"moov", trak + box(b"mvex", trex) + avcc) + + tfhd = full_box(b"tfhd", track_id.to_bytes(4, "big"), flags=0x020000) + trun = full_box(b"trun", (1).to_bytes(4, "big")) + fragment = box(b"moof", box(b"traf", tfhd + trun)) + box(b"mdat", b"frame") + return init, fragment + + +def endpoint(router: APIRouter, path: str, method: str) -> Callable[..., Any]: + routes = router.routes + return next( + route.endpoint + for route in routes + if isinstance(route, APIRoute) and route.path == path and method in route.methods + ) + + +def assert_no_local_paths(value: object, root: Path) -> None: + serialized = json.dumps(value, ensure_ascii=False, default=str) + assert str(root) not in serialized + assert "mqtt.raw.k1mqtt" not in serialized + + +async def render_file_response( + response: Response, + *, + range_header: bytes | None, +) -> tuple[dict[str, Any], bytes]: + sent: list[dict[str, Any]] = [] + + async def receive() -> dict[str, Any]: + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(message: dict[str, Any]) -> None: + sent.append(message) + + scope: dict[str, Any] = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": "/recording.rrd", + "raw_path": b"/recording.rrd", + "query_string": b"", + "headers": [] if range_header is None else [(b"range", range_header)], + "client": ("127.0.0.1", 1), + "server": ("127.0.0.1", 80), + } + await response(scope, receive, send) + start = next(message for message in sent if message["type"] == "http.response.start") + body = b"".join( + message.get("body", b"") for message in sent if message["type"] == "http.response.body" + ) + return start, body + + +def test_session_router_lists_details_and_dispatches_opaque_replay(tmp_path: Path) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + observed: list[ReplayCommand] = [] + + async def launch(command: ReplayCommand) -> dict[str, Any]: + observed.append(command) + return {"source_mode": "replay", "phase": "replay"} + + router = build_session_router(store, replay_launcher=launch) + list_route = endpoint(router, "/api/v1/observation-sessions", "GET") + detail_route = endpoint(router, "/api/v1/observation-sessions/{session_id}", "GET") + replay_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/replay", + "POST", + ) + + listing = list_route(limit=20, cursor=None) + detail = detail_route(session_id=session.name) + replay = asyncio.run(replay_route(session_id=session.name, request=None)) + + assert set(listing) == {"items"} + assert listing["items"][0] == { + "id": session.name, + "label": session.name, + "started_at_utc": "2026-07-16T20:56:32.699Z", + "completed_at_utc": "2026-07-16T21:20:43.018Z", + "status": "ready", + "modalities": ["point-cloud", "trajectory"], + "duration_seconds": 1440.1, + "replayable": True, + } + assert detail["timeline"]["seekable"] is True + assert detail["modalities"] == ["point-cloud", "trajectory"] + assert replay["launch"] == { + "kind": "plugin-action", + "plugin_id": "nodedc.device.xgrids-lixelkity-k1", + "action_id": "stream.start-replay", + "session_id": session.name, + "speed": 1.0, + "loop": False, + "dispatched": True, + } + assert replay["runtime_state"]["source_mode"] == "replay" + assert observed[0].source_path.name == "mqtt.raw.k1mqtt" + for value in (listing, detail, replay): + assert_no_local_paths(value, repository) + + +def test_session_router_returns_seekable_recording_and_serves_byte_ranges( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + payload = b"RRD-seekable-recording" + export_calls = 0 + + def export_recording(source: Path, destination: Path) -> dict[str, object]: + nonlocal export_calls + export_calls += 1 + destination.write_bytes(payload) + return { + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "rrd_sha256": hashlib.sha256(payload).hexdigest(), + "rrd_bytes": len(payload), + "timeline": "session_time", + "timeline_start_ns": 0, + "timeline_end_ns": 2_500_000_000, + } + + materializer = SessionRecordingMaterializer( + store.data_dir, + exporter=export_recording, + ) + router = build_session_router( + store, + recording_materializer=materializer, + allow_synchronous_recording_fallback=True, + ) + replay_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/replay", + "POST", + ) + recording_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/recording.rrd", + "GET", + ) + + replay = asyncio.run( + replay_route( + session_id=session.name, + request=ReplayRequest(speed=2.0, loop=True), + ) + ) + + assert replay == { + "schema_version": "missioncore.observation-session-replay/v2", + "launch": { + "kind": "rerun-recording", + "session_id": session.name, + "source_url": (f"/api/v1/observation-sessions/{session.name}/recording.rrd"), + "viewer_source_url": ( + f"/api/v1/observation-sessions/{session.name}/recording.rrd" + f"?generation={hashlib.sha256(payload).hexdigest()}" + ), + "media_type": "application/vnd.rerun.rrd", + "timeline": "session_time", + "timeline_start_seconds": 0.0, + "timeline_end_seconds": 2.5, + "seekable": True, + "byte_length": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + "playback": {"speed": 2.0, "loop": True}, + "media_sources": [], + }, + } + assert_no_local_paths(replay, repository) + + file_response = asyncio.run(recording_route(session_id=session.name)) + response_start, response_body = asyncio.run( + render_file_response(file_response, range_header=b"bytes=4-11") + ) + + headers = { + key.decode("latin-1"): value.decode("latin-1") for key, value in response_start["headers"] + } + assert response_start["status"] == 206 + assert response_body == payload[4:12] + assert headers["accept-ranges"] == "bytes" + assert headers["content-range"] == f"bytes 4-11/{len(payload)}" + assert headers["content-type"] == "application/vnd.rerun.rrd" + assert headers["cache-control"] == "private, no-cache, no-transform" + assert str(repository) not in json.dumps(headers) + + not_modified = asyncio.run( + recording_route( + session_id=session.name, + if_none_match=f'W/{headers["etag"]}, "unrelated"', + ) + ) + assert not_modified.status_code == 304 + assert not_modified.body == b"" + assert not_modified.headers["etag"] == headers["etag"] + assert not_modified.headers["cache-control"] == "private, no-cache, no-transform" + assert export_calls == 1 + + +def test_cold_replay_returns_quick_202_then_status_returns_ready_launch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + started = threading.Event() + release = threading.Event() + calls = 0 + payload = b"background-recording" + + def export_recording(source: Path, destination: Path) -> dict[str, object]: + nonlocal calls + calls += 1 + started.set() + assert release.wait(timeout=2) + destination.write_bytes(payload) + return { + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "rrd_sha256": hashlib.sha256(payload).hexdigest(), + "rrd_bytes": len(payload), + "timeline": "session_time", + "timeline_start_ns": 0, + "timeline_end_ns": 2_500_000_000, + } + + materializer = SessionRecordingMaterializer( + store.data_dir, + exporter=export_recording, + ) + manager = SessionRecordingPreparationManager(materializer) + + def forbidden_cold_cache_resolution(*_args: object, **_kwargs: object) -> None: + raise AssertionError("HTTP must not perform cold cache validation") + + monkeypatch.setattr(manager, "resolve_cached", forbidden_cold_cache_resolution) + monkeypatch.setattr(manager, "resolve_cached_pinned", forbidden_cold_cache_resolution) + router = build_session_router( + store, + recording_materializer=materializer, + recording_preparation_manager=manager, + ) + replay_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/replay", + "POST", + ) + status_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/recording-preparation", + "GET", + ) + list_route = endpoint(router, "/api/v1/observation-sessions", "GET") + recording_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/recording.rrd", + "GET", + ) + try: + with pytest.raises(HTTPException) as missing_generation: + asyncio.run(recording_route(session_id=session.name, if_none_match=None)) + assert missing_generation.value.status_code == 428 + + cold_file_before = time.monotonic() + cold_file = asyncio.run( + recording_route( + session_id=session.name, + if_match='"sha256:previous-generation"', + if_none_match=None, + ) + ) + assert time.monotonic() - cold_file_before < 0.1 + assert cold_file.status_code == 202 + cold_preparation_id = json.loads(cold_file.body)["preparation"]["preparation_id"] + + before = time.monotonic() + pending = asyncio.run( + replay_route( + session_id=session.name, + request=ReplayRequest(speed=2.0, loop=True), + ) + ) + + assert time.monotonic() - before < 0.1 + assert pending.status_code == 202 + document = json.loads(pending.body) + preparation = document["preparation"] + assert document["schema_version"] == "missioncore.observation-session-preparation/v1" + assert preparation["preparation_id"] == cold_preparation_id + assert preparation["state"] in {"queued", "validating", "exporting"} + assert preparation["status_url"].endswith("/recording-preparation") + assert pending.headers["location"] == preparation["status_url"] + assert pending.headers["retry-after"] == "1" + preparation_etag = pending.headers["etag"] + assert preparation_etag == f'"{preparation["preparation_id"]}"' + assert started.wait(timeout=1) + catalog_preparation = list_route(limit=20, cursor=None)["items"][0]["preparation"] + assert catalog_preparation["preparation_id"] == preparation["preparation_id"] + assert catalog_preparation["state"] in {"validating", "exporting"} + + duplicate = asyncio.run( + replay_route( + session_id=session.name, + request=ReplayRequest(speed=2.0, loop=True), + ) + ) + assert ( + json.loads(duplicate.body)["preparation"]["preparation_id"] + == preparation["preparation_id"] + ) + release.set() + deadline = time.monotonic() + 2 + ready: dict[str, Any] | None = None + while time.monotonic() < deadline: + candidate = asyncio.run( + status_route( + session_id=session.name, + if_match=preparation_etag, + ) + ) + if candidate.status_code == 200: + assert candidate.headers["etag"] == preparation_etag + ready = json.loads(candidate.body) + break + time.sleep(0.005) + + assert ready is not None + assert ready["launch"]["source_url"].endswith("/recording.rrd") + assert ready["launch"]["viewer_source_url"] == ( + f"{ready['launch']['source_url']}?generation={ready['launch']['sha256']}" + ) + assert ready["launch"]["playback"] == {"speed": 1.0, "loop": False} + recording_etag = f'"sha256:{ready["launch"]["sha256"]}"' + with pytest.raises(HTTPException) as stale_generation: + asyncio.run( + recording_route( + session_id=session.name, + if_match='"sha256:stale"', + if_none_match=None, + ) + ) + assert stale_generation.value.status_code == 412 + with pytest.raises(HTTPException) as stale_query_generation: + asyncio.run( + recording_route( + session_id=session.name, + generation="0" * 64, + if_none_match=None, + ) + ) + assert stale_query_generation.value.status_code == 412 + with pytest.raises(HTTPException) as malformed_query_generation: + asyncio.run( + recording_route( + session_id=session.name, + generation="NOT-A-CANONICAL-DIGEST", + if_none_match=None, + ) + ) + assert malformed_query_generation.value.status_code == 412 + + recording_response = asyncio.run( + recording_route( + session_id=session.name, + if_match=recording_etag, + if_none_match=None, + ) + ) + recording_start, recording_body = asyncio.run( + render_file_response(recording_response, range_header=None) + ) + recording_headers = { + key.decode("latin-1"): value.decode("latin-1") + for key, value in recording_start["headers"] + } + assert recording_start["status"] == 200 + assert recording_body == payload + assert recording_headers["etag"] == recording_etag + assert recording_headers["content-length"] == str(len(payload)) + assert recording_headers["content-type"] == "application/vnd.rerun.rrd" + assert recording_headers["cache-control"] == "private, no-cache, no-transform" + + viewer_response = asyncio.run( + recording_route( + session_id=session.name, + generation=ready["launch"]["sha256"], + if_none_match=None, + ) + ) + viewer_start, viewer_body = asyncio.run( + render_file_response(viewer_response, range_header=None) + ) + viewer_headers = { + key.decode("latin-1"): value.decode("latin-1") + for key, value in viewer_start["headers"] + } + assert viewer_start["status"] == 200 + assert viewer_body == payload + assert viewer_headers["etag"] == recording_etag + assert viewer_headers["content-length"] == str(len(payload)) + assert viewer_headers["content-type"] == "application/vnd.rerun.rrd" + assert viewer_headers["cache-control"] == ( + "private, max-age=31536000, immutable, no-transform" + ) + + viewer_not_modified = asyncio.run( + recording_route( + session_id=session.name, + generation=ready["launch"]["sha256"], + if_none_match=recording_etag, + ) + ) + assert viewer_not_modified.status_code == 304 + assert viewer_not_modified.headers["etag"] == recording_etag + assert viewer_not_modified.headers["cache-control"] == ( + "private, max-age=31536000, immutable, no-transform" + ) + assert calls == 1 + finally: + release.set() + manager.close() + + +def test_recording_response_releases_pin_once_when_asgi_send_fails( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + recording_path = tmp_path / "pinned.rrd" + payload = b"RRD-pinned-response" + recording_path.write_bytes(payload) + recording = MaterializedRecording( + session_id=session.name, + path=recording_path, + media_type="application/vnd.rerun.rrd", + byte_length=len(payload), + sha256=hashlib.sha256(payload).hexdigest(), + source_sha256="source-digest", + timeline="session_time", + timeline_start_ns=0, + timeline_end_ns=1_000_000_000, + ) + + class PinnedMaterializer: + def __init__(self) -> None: + self.release_calls = 0 + + def __call__(self, _command: ReplayCommand) -> MaterializedRecording: + return recording + + def materialize_pinned( + self, + _command: ReplayCommand, + ) -> tuple[MaterializedRecording, Callable[[], None]]: + return recording, self.release + + def release(self) -> None: + self.release_calls += 1 + + materializer = PinnedMaterializer() + router = build_session_router( + store, + recording_materializer=materializer, + allow_synchronous_recording_fallback=True, + ) + recording_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/recording.rrd", + "GET", + ) + response = asyncio.run(recording_route(session_id=session.name)) + + async def send_failure() -> None: + async def receive() -> dict[str, Any]: + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(message: dict[str, Any]) -> None: + if message["type"] == "http.response.body": + raise RuntimeError("synthetic ASGI send failure") + + scope: dict[str, Any] = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": "/recording.rrd", + "raw_path": b"/recording.rrd", + "query_string": b"", + "headers": [], + "client": ("127.0.0.1", 1), + "server": ("127.0.0.1", 80), + } + await response(scope, receive, send) + + with pytest.raises(RuntimeError, match="synthetic ASGI send failure"): + asyncio.run(send_failure()) + assert materializer.release_calls == 1 + + # A response must never double-release the same cache lease, even if an + # ASGI host accidentally invokes the response object again after failure. + with pytest.raises(RuntimeError, match="synthetic ASGI send failure"): + asyncio.run(send_failure()) + assert materializer.release_calls == 1 + + +def test_preparation_status_and_cancel_are_bound_to_exact_job_etag(tmp_path: Path) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + started = threading.Event() + release = threading.Event() + + def exporter(source: Path, destination: Path) -> dict[str, object]: + started.set() + assert release.wait(timeout=2) + payload = b"recording" + destination.write_bytes(payload) + return { + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "rrd_sha256": hashlib.sha256(payload).hexdigest(), + "rrd_bytes": len(payload), + "timeline": "session_time", + "timeline_start_ns": 0, + "timeline_end_ns": 1, + } + + materializer = SessionRecordingMaterializer(store.data_dir, exporter=exporter) + manager = SessionRecordingPreparationManager(materializer) + router = build_session_router( + store, + recording_materializer=materializer, + recording_preparation_manager=manager, + ) + replay_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/replay", + "POST", + ) + status_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/recording-preparation", + "GET", + ) + cancel_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/recording-preparation", + "DELETE", + ) + try: + pending = asyncio.run(replay_route(session_id=session.name, request=None)) + assert pending.status_code == 202 + assert started.wait(timeout=1) + etag = pending.headers["etag"] + + with pytest.raises(HTTPException) as stale_status: + asyncio.run(status_route(session_id=session.name, if_match='"stale-job"')) + assert stale_status.value.status_code == 412 + + with pytest.raises(HTTPException) as stale_cancel: + cancel_route(session_id=session.name, if_match='"stale-job"') + assert stale_cancel.value.status_code == 412 + current = manager.status(session.name) + assert current is not None + assert f'"{current.preparation_id}"' == etag + assert current.state in {"validating", "exporting"} + + assert cancel_route(session_id=session.name, if_match=etag).status_code == 204 + release.set() + deadline = time.monotonic() + 2 + while time.monotonic() < deadline: + current = manager.status(session.name) + if current is not None and current.state == "cancelled": + break + time.sleep(0.005) + assert current is not None and current.state == "cancelled" + finally: + release.set() + manager.close() + + +def test_stale_status_poll_does_not_create_a_replacement_job(tmp_path: Path) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + calls = 0 + + def exporter(_source: Path, _destination: Path) -> dict[str, object]: + nonlocal calls + calls += 1 + raise AssertionError("a stale status poll must not enqueue work") + + materializer = SessionRecordingMaterializer(store.data_dir, exporter=exporter) + manager = SessionRecordingPreparationManager(materializer) + router = build_session_router( + store, + recording_materializer=materializer, + recording_preparation_manager=manager, + ) + status_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/recording-preparation", + "GET", + ) + try: + with pytest.raises(HTTPException) as stale: + asyncio.run( + status_route( + session_id=session.name, + if_match='"no-longer-current"', + ) + ) + assert stale.value.status_code == 412 + assert manager.status(session.name) is None + assert calls == 0 + finally: + manager.close() + + +def test_ready_post_uses_each_callers_playback_without_mutating_shared_job( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + calls = 0 + + def exporter(source: Path, destination: Path) -> dict[str, object]: + nonlocal calls + calls += 1 + payload = b"recording" + destination.write_bytes(payload) + return { + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "rrd_sha256": hashlib.sha256(payload).hexdigest(), + "rrd_bytes": len(payload), + "timeline": "session_time", + "timeline_start_ns": 0, + "timeline_end_ns": 1, + } + + materializer = SessionRecordingMaterializer(store.data_dir, exporter=exporter) + manager = SessionRecordingPreparationManager(materializer) + command = store.prepare_replay(session.name) + manager.enqueue(command) + deadline = time.monotonic() + 2 + while time.monotonic() < deadline: + ready = manager.status(session.name) + if ready is not None and ready.state == "ready": + break + time.sleep(0.005) + router = build_session_router( + store, + recording_materializer=materializer, + recording_preparation_manager=manager, + ) + replay_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/replay", + "POST", + ) + try: + first = asyncio.run( + replay_route( + session_id=session.name, + request=ReplayRequest(speed=2.0, loop=True), + ) + ) + second = asyncio.run( + replay_route( + session_id=session.name, + request=ReplayRequest(speed=7.0, loop=False), + ) + ) + + assert first["launch"]["playback"] == {"speed": 2.0, "loop": True} + assert second["launch"]["playback"] == {"speed": 7.0, "loop": False} + shared = manager.status(session.name) + assert shared is not None + assert shared.command.speed == 1.0 + assert shared.command.loop is False + assert calls == 1 + finally: + manager.close() + + +def test_production_router_never_materializes_recording_inline(tmp_path: Path) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + calls = 0 + + def exporter(_source: Path, _destination: Path) -> dict[str, object]: + nonlocal calls + calls += 1 + raise AssertionError("public route must not export inline") + + materializer = SessionRecordingMaterializer(store.data_dir, exporter=exporter) + router = build_session_router(store, recording_materializer=materializer) + replay_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/replay", + "POST", + ) + recording_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/recording.rrd", + "GET", + ) + + with pytest.raises(HTTPException) as replay_error: + asyncio.run(replay_route(session_id=session.name, request=None)) + with pytest.raises(HTTPException) as recording_error: + asyncio.run(recording_route(session_id=session.name, if_none_match=None)) + + assert replay_error.value.status_code == 503 + assert recording_error.value.status_code == 503 + assert calls == 0 + +def test_recorded_blueprint_endpoint_is_small_strict_and_session_scoped( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + observed_settings: list[RerunSceneSettings] = [] + + def capture_settings(settings: RerunSceneSettings, **kwargs: Any) -> bytes: + observed_settings.append(settings) + return recorded_blueprint_rrd(settings, **kwargs) + + monkeypatch.setattr( + "k1link.web.session_api.recorded_blueprint_rrd", + capture_settings, + ) + router = build_session_router(store) + blueprint_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/blueprint.rrd", + "POST", + ) + + response = asyncio.run( + blueprint_route( + session_id=session.name, + request=RecordedBlueprintRequest( + application_id="nodedc_mission_core_recorded", + recording_id="recording-001", + accumulation_seconds=18.5, + show_points=False, + show_trajectory=True, + show_grid=False, + point_size=6.25, + palette="custom", + custom_color="#112233", + ), + ) + ) + + assert response.media_type == "application/vnd.rerun.rrd" + assert response.headers["cache-control"] == "no-store" + assert response.headers["x-content-type-options"] == "nosniff" + assert response.body.startswith(b"RRF2") + assert len(response.body) < 100_000 + assert_no_local_paths(response.headers, repository) + assert len(observed_settings) == 1 + assert observed_settings[0].show_points is False + assert observed_settings[0].show_trajectory is True + + with pytest.raises(ValueError): + RecordedBlueprintRequest.model_validate( + { + "application_id": "nodedc_mission_core_recorded", + "recording_id": "recording-001", + "accumulation_seconds": 12, + "show_grid": True, + "source_url": "https://outside.invalid/recording.rrd", + } + ) + with pytest.raises(HTTPException) as missing: + asyncio.run( + blueprint_route( + session_id="missing-session", + request=RecordedBlueprintRequest( + application_id="nodedc_mission_core_recorded", + recording_id="recording-001", + accumulation_seconds=12, + show_points=True, + show_trajectory=True, + show_grid=True, + ), + ) + ) + assert missing.value.status_code == 404 + + +def test_session_router_exposes_opaque_recorded_media_manifest_and_ranges( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + writer = CameraArchiveWriter(session, "sensor.camera.private-left", 9) + init, segment = make_recorded_h264_fixture() + writer.append("init", init, host_epoch_ns=1_100_000_000, host_monotonic_ns=2_100_000_000) + writer.append( + "media", + segment, + host_epoch_ns=1_250_000_000, + host_monotonic_ns=2_250_000_000, + ) + writer.close() + + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + payload = b"recording" + + def export_recording(source: Path, destination: Path) -> dict[str, object]: + destination.write_bytes(payload) + return { + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "rrd_sha256": hashlib.sha256(payload).hexdigest(), + "rrd_bytes": len(payload), + "timeline": "session_time", + "timeline_start_ns": 0, + "timeline_end_ns": 5_000_000_000, + } + + router = build_session_router( + store, + recording_materializer=SessionRecordingMaterializer( + store.data_dir, + exporter=export_recording, + ), + allow_synchronous_recording_fallback=True, + ) + replay_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/replay", + "POST", + ) + manifest_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/media/{artifact_id}/manifest", + "GET", + ) + init_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/media/{artifact_id}/" + "epochs/{epoch_ordinal}/init.mp4", + "GET", + ) + segment_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/media/{artifact_id}/" + "epochs/{epoch_ordinal}/segments/{segment_sequence}.m4s", + "GET", + ) + + replay = asyncio.run(replay_route(session_id=session.name, request=None)) + source = replay["launch"]["media_sources"][0] + assert source == { + "id": source["id"], + "label": "Записанная камера 1", + "modality": "video", + "manifest_url": source["manifest_url"], + "manifest_generation_sha256": source["manifest_generation_sha256"], + "byte_length": len(init) + len(segment), + "media_type": "video/mp4", + "timeline_start_seconds": 0.0, + "timeline_end_seconds": 0.5, + "seekable": True, + "synchronization": "host-arrival-best-effort", + } + assert source["id"].startswith("recorded.camera.") + assert "sensor.camera.private-left" not in json.dumps(replay) + artifact_id = source["manifest_url"].split("/")[-2] + + with pytest.raises(HTTPException) as missing_manifest_generation: + manifest_route(session_id=session.name, artifact_id=artifact_id) + assert missing_manifest_generation.value.status_code == 428 + with pytest.raises(HTTPException) as stale_manifest_generation: + manifest_route( + session_id=session.name, + artifact_id=artifact_id, + if_match='"sha256:stale"', + ) + assert stale_manifest_generation.value.status_code == 412 + + manifest_response = manifest_route( + session_id=session.name, + artifact_id=artifact_id, + if_match=f'"sha256:{source["manifest_generation_sha256"]}"', + ) + manifest = json.loads(manifest_response.body) + assert manifest["schema_version"] == "missioncore.observation-recorded-media/v2" + assert manifest["source_id"] == source["id"] + assert manifest["generation_sha256"] == source["manifest_generation_sha256"] + assert manifest["byte_length"] == source["byte_length"] + assert manifest["timeline_start_seconds"] == source["timeline_start_seconds"] + assert manifest["timeline_end_seconds"] == source["timeline_end_seconds"] + assert manifest_response.headers["etag"] == ( + f'"sha256:{manifest["generation_sha256"]}"' + ) + init_sha256 = hashlib.sha256(init).hexdigest() + segment_sha256 = hashlib.sha256(segment).hexdigest() + assert manifest["epochs"] == [ + { + "ordinal": 1, + "timeline_start_seconds": 0.0, + "timeline_end_seconds": 0.5, + "media_type": 'video/mp4; codecs="avc1.640028"', + "init_url": manifest["epochs"][0]["init_url"], + "init_byte_length": len(init), + "init_sha256": init_sha256, + "segment_count": 1, + "segment_url_prefix": manifest["epochs"][0]["segment_url_prefix"], + "segments": [ + { + "sequence": 1, + "url": manifest["epochs"][0]["segments"][0]["url"], + "byte_length": len(segment), + "sha256": segment_sha256, + } + ], + } + ] + assert "sensor.camera.private-left" not in json.dumps(manifest) + assert_no_local_paths(manifest, repository) + + with pytest.raises(HTTPException) as missing_init_generation: + init_route( + session_id=session.name, + artifact_id=artifact_id, + epoch_ordinal=1, + range_header="bytes=4-7", + ) + assert missing_init_generation.value.status_code == 428 + + init_response = init_route( + session_id=session.name, + artifact_id=artifact_id, + epoch_ordinal=1, + if_match=f'"sha256:{init_sha256}"', + range_header="bytes=4-7", + ) + init_start, init_body = asyncio.run( + render_file_response(init_response, range_header=b"bytes=4-7") + ) + assert init_start["status"] == 206 + assert init_body == init[4:8] + + with pytest.raises(HTTPException) as stale_segment_generation: + segment_route( + session_id=session.name, + artifact_id=artifact_id, + epoch_ordinal=1, + segment_sequence=1, + if_match='"sha256:stale"', + range_header=None, + ) + assert stale_segment_generation.value.status_code == 412 + + segment_response = segment_route( + session_id=session.name, + artifact_id=artifact_id, + epoch_ordinal=1, + segment_sequence=1, + if_match=f'"sha256:{segment_sha256}"', + range_header="bytes=0-5", + ) + segment_start, segment_body = asyncio.run( + render_file_response(segment_response, range_header=b"bytes=0-5") + ) + headers = { + key.decode("latin-1"): value.decode("latin-1") for key, value in segment_start["headers"] + } + assert segment_start["status"] == 206 + assert segment_body == segment[:6] + assert headers["cache-control"] == ( + "private, max-age=31536000, immutable, no-transform" + ) + assert headers["etag"] == f'"sha256:{segment_sha256}"' + assert headers["content-length"] == "6" + assert headers["x-content-type-options"] == "nosniff" + + original_segment = writer.segments_dir / "1.m4s" + outside = tmp_path / "outside.m4s" + outside.write_bytes(segment) + original_segment.unlink() + original_segment.symlink_to(outside) + with pytest.raises(HTTPException) as confined: + segment_route( + session_id=session.name, + artifact_id=artifact_id, + epoch_ordinal=1, + segment_sequence=1, + if_match=f'"sha256:{segment_sha256}"', + range_header=None, + ) + assert confined.value.status_code == 409 + + +def test_ready_launch_uses_background_prepared_camera_manifest_without_rescan( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + writer = CameraArchiveWriter(session, "sensor.camera.private-left", 1) + init, segment = make_recorded_h264_fixture() + writer.append( + "init", + init, + host_epoch_ns=1_100_000_000, + host_monotonic_ns=2_100_000_000, + ) + writer.append( + "media", + segment, + host_epoch_ns=1_250_000_000, + host_monotonic_ns=2_250_000_000, + ) + writer.close() + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + + def exporter(source: Path, destination: Path) -> dict[str, object]: + payload = b"prepared-recording" + destination.write_bytes(payload) + return { + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "rrd_sha256": hashlib.sha256(payload).hexdigest(), + "rrd_bytes": len(payload), + "timeline": "session_time", + "timeline_start_ns": 0, + "timeline_end_ns": 5_000_000_000, + } + + materializer = SessionRecordingMaterializer(store.data_dir, exporter=exporter) + inspector = RecordedMediaInspector() + + def prepare_media( + command: ReplayCommand, + _recording: MaterializedRecording, + ) -> tuple[Any, ...]: + return tuple( + inspector.inspect(artifact, command) + for artifact in store.list_recorded_media(command.session_id) + ) + + manager = SessionRecordingPreparationManager( + materializer, + ready_preparer=prepare_media, + ) + try: + manager.enqueue(store.prepare_replay(session.name)) + deadline = time.monotonic() + 2 + snapshot = manager.status(session.name) + while ( + snapshot is None or snapshot.state != "ready" + ) and time.monotonic() < deadline: + time.sleep(0.005) + snapshot = manager.status(session.name) + assert snapshot is not None and snapshot.state == "ready" + assert snapshot.recorded_media is not None and len(snapshot.recorded_media) == 1 + + def forbidden_rescan(*_args: object, **_kwargs: object) -> None: + raise AssertionError("ready HTTP path must not rescan RRD or camera indexes") + + monkeypatch.setattr(inspector, "inspect", forbidden_rescan) + monkeypatch.setattr(manager, "resolve_cached", forbidden_rescan) + monkeypatch.setattr(manager, "resolve_cached_pinned", forbidden_rescan) + router = build_session_router( + store, + recording_materializer=materializer, + recording_preparation_manager=manager, + media_inspector=inspector, + ) + replay_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/replay", + "POST", + ) + manifest_route = endpoint( + router, + "/api/v1/observation-sessions/{session_id}/media/{artifact_id}/manifest", + "GET", + ) + + replay = asyncio.run(replay_route(session_id=session.name, request=None)) + source = replay["launch"]["media_sources"][0] + artifact_id = source["manifest_url"].split("/")[-2] + manifest_response = manifest_route( + session_id=session.name, + artifact_id=artifact_id, + if_match=f'"sha256:{source["manifest_generation_sha256"]}"', + ) + manifest = json.loads(manifest_response.body) + assert manifest["generation_sha256"] == source["manifest_generation_sha256"] + finally: + manager.close() + + +def test_recorded_media_epoch_ends_are_monotonic_generation_bound_and_not_rrd_padded( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + init, segment = make_recorded_h264_fixture(sample_duration=250) + first = CameraArchiveWriter(session, "sensor.camera.private-left", 1) + first.append("init", init, host_epoch_ns=1_100_000_000, host_monotonic_ns=2_100_000_000) + first.append( + "media", + segment, + host_epoch_ns=1_250_000_000, + host_monotonic_ns=2_250_000_000, + ) + first.close() + second = CameraArchiveWriter(session, "sensor.camera.private-left", 2) + second.append("init", init, host_epoch_ns=1_800_000_000, host_monotonic_ns=2_800_000_000) + second.append( + "media", + segment, + host_epoch_ns=1_800_000_000, + host_monotonic_ns=2_800_000_000, + ) + second.append( + "media", + segment, + host_epoch_ns=2_000_000_000, + host_monotonic_ns=3_000_000_000, + ) + second.close() + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + command = store.prepare_replay(session.name) + artifact = store.list_recorded_media(session.name)[0] + manifest = RecordedMediaInspector().inspect(artifact, command) + + assert [epoch.timeline_start_seconds for epoch in manifest.epochs] == [0.0, 0.55] + assert [epoch.timeline_end_seconds for epoch in manifest.epochs] == [0.25, 1.05] + assert manifest.timeline_end_seconds == 1.05 + assert manifest.byte_length == 2 * len(init) + 3 * len(segment) + + original_generation = manifest.generation_sha256 + changed_epochs = ( + *manifest.epochs[:-1], + replace( + manifest.epochs[-1], + timeline_end_seconds=manifest.epochs[-1].timeline_end_seconds + 0.1, + ), + ) + changed_generation = _manifest_generation_sha256( + public_source_id=manifest.public_source_id, + artifact_id=manifest.artifact_id, + synchronization=manifest.synchronization, + epochs=changed_epochs, + ) + assert changed_generation != original_generation + + +def test_recorded_media_rejects_non_monotonic_segments_and_overlapping_epochs( + tmp_path: Path, +) -> None: + init, segment = make_recorded_h264_fixture(sample_duration=500) + + non_monotonic_root = tmp_path / "non-monotonic" + non_monotonic_session = make_legacy_session( + non_monotonic_root / "sessions", + "20260716T205632Z_viewer_live", + ) + writer = CameraArchiveWriter(non_monotonic_session, "sensor.camera.private-left", 1) + writer.append("init", init) + writer.append( + "media", + segment, + host_epoch_ns=1_600_000_000, + host_monotonic_ns=2_600_000_000, + ) + writer.append( + "media", + segment, + host_epoch_ns=1_500_000_000, + host_monotonic_ns=2_500_000_000, + ) + writer.close() + store = SessionStore(non_monotonic_root, data_dir=tmp_path / "non-monotonic-data") + store.import_legacy_viewer_live(non_monotonic_root / "sessions") + with pytest.raises(SessionIntegrityError, match="strictly monotonic"): + RecordedMediaInspector().inspect( + store.list_recorded_media(non_monotonic_session.name)[0], + store.prepare_replay(non_monotonic_session.name), + ) + + overlap_root = tmp_path / "overlap" + overlap_session = make_legacy_session( + overlap_root / "sessions", + "20260716T205633Z_viewer_live", + ) + first = CameraArchiveWriter(overlap_session, "sensor.camera.private-left", 1) + first.append("init", init) + first.append( + "media", + segment, + host_epoch_ns=1_250_000_000, + host_monotonic_ns=2_250_000_000, + ) + first.close() + second = CameraArchiveWriter(overlap_session, "sensor.camera.private-left", 2) + second.append("init", init) + second.append( + "media", + segment, + host_epoch_ns=1_500_000_000, + host_monotonic_ns=2_500_000_000, + ) + second.close() + store = SessionStore(overlap_root, data_dir=tmp_path / "overlap-data") + store.import_legacy_viewer_live(overlap_root / "sessions") + with pytest.raises(SessionIntegrityError, match="overlap"): + RecordedMediaInspector().inspect( + store.list_recorded_media(overlap_session.name)[0], + store.prepare_replay(overlap_session.name), + ) + + +def test_background_preparation_fails_closed_on_unparseable_recorded_media( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + writer = CameraArchiveWriter(session, "sensor.camera.private-left", 1) + writer.append("init", b"not-an-iso-bmff-init") + writer.append("media", b"not-an-iso-bmff-fragment") + writer.close() + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + + def exporter(source: Path, destination: Path) -> dict[str, object]: + return { + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "rrd_sha256": hashlib.sha256(b"recording").hexdigest(), + "rrd_bytes": destination.write_bytes(b"recording"), + "timeline": "session_time", + "timeline_start_ns": 0, + "timeline_end_ns": 5_000_000_000, + } + + inspector = RecordedMediaInspector() + + def prepare_media( + command: ReplayCommand, + _recording: MaterializedRecording, + ) -> tuple[Any, ...]: + return tuple( + inspector.inspect(artifact, command) + for artifact in store.list_recorded_media(command.session_id) + ) + + manager = SessionRecordingPreparationManager( + SessionRecordingMaterializer(store.data_dir, exporter=exporter), + ready_preparer=prepare_media, + ) + try: + manager.enqueue(store.prepare_replay(session.name)) + deadline = time.monotonic() + 2 + snapshot = manager.status(session.name) + while snapshot is not None and snapshot.state not in {"failed", "ready"}: + assert time.monotonic() < deadline + time.sleep(0.005) + snapshot = manager.status(session.name) + assert snapshot is not None + assert snapshot.state == "failed" + assert snapshot.recording is None + assert snapshot.recorded_media is None + finally: + manager.close() + + +def test_background_preparation_rejects_camera_outside_rrd_timeline( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + init, segment = make_recorded_h264_fixture(sample_duration=500) + writer = CameraArchiveWriter(session, "sensor.camera.private-left", 1) + writer.append("init", init) + writer.append( + "media", + segment, + host_epoch_ns=1_500_000_000, + host_monotonic_ns=2_500_000_000, + ) + writer.close() + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + + def exporter(source: Path, destination: Path) -> dict[str, object]: + payload = b"short-spatial-recording" + destination.write_bytes(payload) + return { + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "rrd_sha256": hashlib.sha256(payload).hexdigest(), + "rrd_bytes": len(payload), + "timeline": "session_time", + "timeline_start_ns": 0, + "timeline_end_ns": 400_000_000, + } + + inspector = RecordedMediaInspector(tmp_path / "prepared-media") + + def prepare_media( + command: ReplayCommand, + _recording: MaterializedRecording, + ) -> tuple[Any, ...]: + return tuple( + inspector.inspect(artifact, command) + for artifact in store.list_recorded_media(command.session_id) + ) + + manager = SessionRecordingPreparationManager( + SessionRecordingMaterializer(store.data_dir, exporter=exporter), + ready_preparer=prepare_media, + ) + try: + manager.enqueue(store.prepare_replay(session.name)) + deadline = time.monotonic() + 2 + snapshot = manager.status(session.name) + while snapshot is not None and snapshot.state not in {"failed", "ready"}: + assert time.monotonic() < deadline + time.sleep(0.005) + snapshot = manager.status(session.name) + assert snapshot is not None + assert snapshot.state == "failed" + assert snapshot.recording is None + assert snapshot.recorded_media is None + finally: + manager.close() + + +def test_recorded_media_preparation_sidecar_reuses_and_rebuilds_generation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + init, segment = make_recorded_h264_fixture() + writer = CameraArchiveWriter(session, "sensor.camera.private-left", 1) + writer.append("init", init) + writer.append( + "media", + segment, + host_epoch_ns=1_500_000_000, + host_monotonic_ns=2_500_000_000, + ) + writer.close() + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + command = store.prepare_replay(session.name) + artifact = store.list_recorded_media(session.name)[0] + cache_root = tmp_path / "prepared-media" + initial = RecordedMediaInspector(cache_root).inspect(artifact, command) + sidecars = list(cache_root.glob("*.json")) + assert len(sidecars) == 1 + sidecar = sidecars[0] + assert stat.S_IMODE(sidecar.stat().st_mode) == 0o600 + assert str(session) not in sidecar.read_text(encoding="utf-8") + + def forbidden_reparse(*_args: object, **_kwargs: object) -> None: + raise AssertionError("restart must reuse the prepared media sidecar") + + with monkeypatch.context() as context: + context.setattr(recorded_media_module, "_read_manifest", forbidden_reparse) + context.setattr(recorded_media_module, "_mp4_video_timing", forbidden_reparse) + restarted = RecordedMediaInspector(cache_root).inspect(artifact, command) + assert restarted.generation_sha256 == initial.generation_sha256 + assert restarted.timeline_end_seconds == initial.timeline_end_seconds + + original_read_manifest = recorded_media_module._read_manifest + reparses = 0 + + def counted_reparse(*args: Any, **kwargs: Any) -> Any: + nonlocal reparses + reparses += 1 + return original_read_manifest(*args, **kwargs) + + sidecar.write_bytes(b"{corrupt") + with monkeypatch.context() as context: + context.setattr(recorded_media_module, "_read_manifest", counted_reparse) + repaired = RecordedMediaInspector(cache_root).inspect(artifact, command) + assert repaired.generation_sha256 == initial.generation_sha256 + assert reparses == 1 + assert json.loads(sidecar.read_text(encoding="utf-8"))["checksum_sha256"] + + segment_path = writer.segments_dir / "1.m4s" + replacement = writer.segments_dir / "replacement.m4s" + replacement.write_bytes(segment_path.read_bytes()) + replacement.replace(segment_path) + reparses = 0 + with monkeypatch.context() as context: + context.setattr(recorded_media_module, "_read_manifest", counted_reparse) + replaced = RecordedMediaInspector(cache_root).inspect(artifact, command) + assert replaced.generation_sha256 == initial.generation_sha256 + assert reparses == 1 + + +def test_recorded_media_preparation_scavenges_crash_temp_files(tmp_path: Path) -> None: + cache_root = tmp_path / "prepared-media" + cache_root.mkdir() + crash_temp = cache_root / f".tmp-{'a' * 32}" + crash_temp.write_bytes(b"partial") + + RecordedMediaInspector(cache_root) + + assert crash_temp.exists() is False + + +def test_layout_routes_round_trip_versioned_document(tmp_path: Path) -> None: + store = SessionStore(tmp_path / "repo", data_dir=tmp_path / "data") + router = build_session_router(store) + put_route = endpoint(router, "/api/v1/workspace-layouts/{workspace_id}", "PUT") + get_route = endpoint(router, "/api/v1/workspace-layouts/{workspace_id}", "GET") + request = LayoutPutRequest( + version=1, + revision=0, + workspace_id="observation.spatial", + scene_settings={ + "projection": "3d", + "point_size": 3.0, + "color_mode": "intensity", + "palette": "turbo", + "custom_color": "#35d7c1", + "accumulation_seconds": 12, + "show_points": True, + "show_trajectory": True, + "show_grid": True, + "show_labels": False, + "show_camera_frustums": True, + }, + tool_windows={ + "sources_open": True, + "display_open": False, + "layers_open": False, + "order": ["sources", "display", "layers"], + }, + visible_source_ids=["sensor.lidar.primary"], + active_floating_source_id=None, + window_rects={"sensor.camera.left": {"x": 0.65, "y": 0.65, "width": 0.3, "height": 0.3}}, + viewport_size={"width": 1440, "height": 900}, + ) + put_response = Response() + + saved = put_route( + workspace_id="observation.spatial", + request=request, + response=put_response, + if_match='"0"', + ) + get_response = Response() + loaded = get_route(workspace_id="observation.spatial", response=get_response) + + assert saved == loaded + assert set(loaded) == { + "version", + "revision", + "workspace_id", + "scene_settings", + "tool_windows", + "visible_source_ids", + "active_floating_source_id", + "window_rects", + "viewport_size", + } + assert loaded["version"] == 1 + assert loaded["workspace_id"] == "observation.spatial" + assert loaded["revision"] == 1 + assert put_response.headers["etag"] == '"1"' + assert get_response.headers["etag"] == '"1"' + + stale = request.model_copy(update={"revision": 0}) + with pytest.raises(HTTPException) as conflict: + put_route( + workspace_id="observation.spatial", + request=stale, + response=Response(), + if_match='"0"', + ) + assert conflict.value.status_code == 412 diff --git a/tests/test_session_preparation.py b/tests/test_session_preparation.py new file mode 100644 index 0000000..1e05ab1 --- /dev/null +++ b/tests/test_session_preparation.py @@ -0,0 +1,377 @@ +from __future__ import annotations + +import hashlib +import threading +import time +from collections.abc import Callable +from dataclasses import replace +from pathlib import Path + +import pytest + +from k1link.sessions.models import ReplayCommand +from k1link.sessions.preparation import ( + RecordingPreparationSnapshot, + SessionRecordingPreparationManager, +) +from k1link.sessions.recording import RecordingMaterializationError, SessionRecordingMaterializer + + +def _command(root: Path, session_id: str = "20260717T100000Z_viewer_live") -> ReplayCommand: + root.mkdir(parents=True) + source = root / "mqtt.raw.k1mqtt" + metadata = root / "mqtt.metadata.jsonl" + source.write_bytes(b"native-session-source") + metadata.write_text('{"record_type":"message","sequence":1}\n', encoding="utf-8") + return ReplayCommand( + session_id=session_id, + source_path=source, + allowed_root=root, + session_root=root, + replay_byte_length=source.stat().st_size, + metadata_byte_length=metadata.stat().st_size, + expected_source_sha256=None, + speed=1.0, + loop=False, + ) + + +def _summary(source: Path, destination: Path, payload: bytes) -> dict[str, object]: + destination.write_bytes(payload) + return { + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "rrd_sha256": hashlib.sha256(payload).hexdigest(), + "rrd_bytes": len(payload), + "timeline": "session_time", + "timeline_start_ns": 0, + "timeline_end_ns": 1_000_000_000, + } + + +def _wait_for_state( + manager: SessionRecordingPreparationManager, + session_id: str, + states: set[str], + timeout: float = 2.0, +) -> RecordingPreparationSnapshot: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + snapshot = manager.status(session_id) + if snapshot is not None and snapshot.state in states: + return snapshot + time.sleep(0.005) + raise AssertionError(f"preparation did not reach {states}") + + +def test_manager_returns_quick_job_deduplicates_and_reports_monotonic_progress( + tmp_path: Path, +) -> None: + command = _command(tmp_path / "session") + started = threading.Event() + release = threading.Event() + calls = 0 + + def exporter( + source: Path, + destination: Path, + *, + cancel_event: threading.Event | None = None, + activity_callback: Callable[[], None] | None = None, + ) -> dict[str, object]: + nonlocal calls + calls += 1 + started.set() + deadline = time.monotonic() + 2 + while not release.wait(timeout=0.005): + assert cancel_event is None or not cancel_event.is_set() + if activity_callback is not None: + activity_callback() + assert time.monotonic() < deadline + return _summary(source, destination, b"prepared-recording") + + manager = SessionRecordingPreparationManager( + SessionRecordingMaterializer(tmp_path / "private", exporter=exporter), + heartbeat_interval_seconds=0.01, + ) + try: + before = time.monotonic() + first = manager.enqueue(command) + elapsed = time.monotonic() - before + duplicate = manager.enqueue(command) + + assert elapsed < 0.1 + assert duplicate.preparation_id == first.preparation_id + assert started.wait(timeout=1) + exporting = _wait_for_state(manager, command.session_id, {"exporting"}) + time.sleep(0.03) + heartbeat = manager.status(command.session_id) + assert heartbeat is not None + assert heartbeat.updated_at_utc > exporting.updated_at_utc + assert heartbeat.progress >= exporting.progress + assert heartbeat.cancellable is True + release.set() + ready = _wait_for_state(manager, command.session_id, {"ready"}) + + assert calls == 1 + assert exporting.progress <= ready.progress == 1.0 + assert ready.recording is not None + assert ready.updated_at_utc.endswith("Z") + finally: + release.set() + manager.close() + + +def test_manager_failure_is_retryable_and_retry_publishes_new_job(tmp_path: Path) -> None: + command = _command(tmp_path / "session") + calls = 0 + + def exporter(source: Path, destination: Path) -> dict[str, object]: + nonlocal calls + calls += 1 + if calls == 1: + raise OSError("simulated converter failure") + return _summary(source, destination, b"retry-recording") + + manager = SessionRecordingPreparationManager( + SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) + ) + try: + first = manager.enqueue(command) + failed = _wait_for_state(manager, command.session_id, {"failed"}) + retry = manager.enqueue(command, retry_failed=True) + ready = _wait_for_state(manager, command.session_id, {"ready"}) + + assert failed.retryable is True + assert failed.error == "Не удалось подготовить запись сессии." + assert retry.preparation_id != first.preparation_id + assert ready.preparation_id == retry.preparation_id + assert calls == 2 + finally: + manager.close() + + +def test_manager_cancels_queued_job_without_exporting_it(tmp_path: Path) -> None: + first = _command(tmp_path / "first") + second = replace( + _command(tmp_path / "second"), + session_id="20260717T100001Z_viewer_live", + ) + started = threading.Event() + release = threading.Event() + calls = 0 + + def exporter(source: Path, destination: Path) -> dict[str, object]: + nonlocal calls + calls += 1 + started.set() + assert release.wait(timeout=2) + return _summary(source, destination, b"recording") + + manager = SessionRecordingPreparationManager( + SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) + ) + try: + manager.enqueue(first) + assert started.wait(timeout=1) + manager.enqueue(second) + assert manager.cancel(second.session_id) is True + cancelled = _wait_for_state(manager, second.session_id, {"cancelled"}) + release.set() + _wait_for_state(manager, first.session_id, {"ready"}) + time.sleep(0.02) + + assert cancelled.cancellable is False + assert calls == 1 + finally: + release.set() + manager.close() + + +def test_manager_can_restart_across_repeated_application_lifespans(tmp_path: Path) -> None: + command = _command(tmp_path / "session") + calls = 0 + + def exporter(source: Path, destination: Path) -> dict[str, object]: + nonlocal calls + calls += 1 + return _summary(source, destination, b"restartable-recording") + + manager = SessionRecordingPreparationManager( + SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) + ) + try: + manager.enqueue(command) + _wait_for_state(manager, command.session_id, {"ready"}) + manager.close() + manager.start() + + resumed = manager.enqueue(command) + + assert resumed.state == "ready" + assert calls == 1 + finally: + manager.close() + + +def test_noncooperative_exporter_has_no_fake_heartbeat_and_restart_waits_for_old_writer( + tmp_path: Path, +) -> None: + first = _command(tmp_path / "first") + second = replace( + _command(tmp_path / "second"), + session_id="20260717T100002Z_viewer_live", + ) + first_started = threading.Event() + release_first = threading.Event() + active = 0 + max_active = 0 + calls = 0 + guard = threading.Lock() + + def exporter(source: Path, destination: Path) -> dict[str, object]: + nonlocal active, calls, max_active + with guard: + calls += 1 + active += 1 + max_active = max(max_active, active) + call = calls + try: + if call == 1: + first_started.set() + assert release_first.wait(timeout=2) + return _summary(source, destination, f"recording-{call}".encode()) + finally: + with guard: + active -= 1 + + manager = SessionRecordingPreparationManager( + SessionRecordingMaterializer(tmp_path / "private", exporter=exporter), + heartbeat_interval_seconds=0.01, + ) + try: + manager.enqueue(first) + assert first_started.wait(timeout=1) + blocked = _wait_for_state(manager, first.session_id, {"exporting"}) + time.sleep(0.03) + unchanged = manager.status(first.session_id) + assert unchanged is not None + assert unchanged.updated_at_utc == blocked.updated_at_utc + assert unchanged.cancellable is False + + manager.close(timeout=0.001) + manager.start() + queued = manager.enqueue(second) + assert queued.state == "queued" + time.sleep(0.03) + assert calls == 1 + + release_first.set() + ready = _wait_for_state(manager, second.session_id, {"ready"}) + assert ready.recording is not None + assert calls == 2 + assert max_active == 1 + finally: + release_first.set() + manager.close() + + +def test_reconciler_retry_does_not_replace_operator_cancel_across_restart( + tmp_path: Path, +) -> None: + command = _command(tmp_path / "session") + started = threading.Event() + release = threading.Event() + calls = 0 + + def exporter(source: Path, destination: Path) -> dict[str, object]: + nonlocal calls + calls += 1 + started.set() + assert release.wait(timeout=2) + return _summary(source, destination, b"operator-cancelled") + + manager = SessionRecordingPreparationManager( + SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) + ) + try: + original = manager.enqueue(command) + assert started.wait(timeout=1) + assert manager.cancel( + command.session_id, + preparation_id=original.preparation_id, + ) + manager.close(timeout=0.001) + manager.start() + release.set() + cancelled = _wait_for_state(manager, command.session_id, {"cancelled"}) + + reconciled = manager.enqueue(command, retry_interrupted=True) + + assert reconciled.preparation_id == cancelled.preparation_id + assert reconciled.state == "cancelled" + assert calls == 1 + finally: + release.set() + manager.close() + + +def test_shared_preparation_command_ignores_per_request_playback_policy(tmp_path: Path) -> None: + command = _command(tmp_path / "session") + started = threading.Event() + release = threading.Event() + + def exporter(source: Path, destination: Path) -> dict[str, object]: + started.set() + assert release.wait(timeout=2) + return _summary(source, destination, b"recording") + + manager = SessionRecordingPreparationManager( + SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) + ) + try: + first = manager.enqueue(replace(command, speed=2.0, loop=True)) + assert started.wait(timeout=1) + duplicate = manager.enqueue(replace(command, speed=7.0, loop=False)) + + assert duplicate.preparation_id == first.preparation_id + assert duplicate.command.speed == 1.0 + assert duplicate.command.loop is False + finally: + release.set() + manager.close() + + +def test_launch_reservation_blocks_eviction_until_lease_expires(tmp_path: Path) -> None: + first = _command(tmp_path / "first") + second = replace( + _command(tmp_path / "second"), + session_id="20260717T100003Z_viewer_live", + ) + + def exporter(source: Path, destination: Path) -> dict[str, object]: + return _summary(source, destination, b"R" * (40 * 1024)) + + materializer = SessionRecordingMaterializer( + tmp_path / "private", + exporter=exporter, + cache_max_bytes=50 * 1024, + free_space_reserve_bytes=0, + ) + manager = SessionRecordingPreparationManager(materializer) + try: + first_recording = materializer.materialize(first) + resolved = manager.resolve_cached(first) + assert resolved is not None and resolved.state == "ready" + reserved = manager.reserve_cached(first, lease_seconds=0.05) + assert reserved is not None and reserved.recording is not None + + with pytest.raises(RecordingMaterializationError, match="cache quota"): + materializer.materialize(second) + assert first_recording.path.exists() + + time.sleep(0.08) + second_recording = materializer.materialize(second) + assert second_recording.path.exists() + assert not first_recording.path.exists() + finally: + manager.close() diff --git a/tests/test_session_recording.py b/tests/test_session_recording.py new file mode 100644 index 0000000..26fcfe5 --- /dev/null +++ b/tests/test_session_recording.py @@ -0,0 +1,522 @@ +from __future__ import annotations + +import hashlib +import json +import os +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pytest + +import k1link.sessions.recording as recording_module +from k1link.sessions.models import ReplayCommand +from k1link.sessions.recording import ( + CACHE_SCHEMA, + RERUN_RECORDING_MEDIA_TYPE, + RecordingMaterializationError, + SessionRecordingMaterializer, +) + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +class FakeExporter: + def __init__(self, *, delay_seconds: float = 0.0) -> None: + self.calls = 0 + self.delay_seconds = delay_seconds + self._lock = threading.Lock() + + def __call__(self, source: Path, destination: Path) -> dict[str, object]: + with self._lock: + self.calls += 1 + sequence = self.calls + if self.delay_seconds: + time.sleep(self.delay_seconds) + destination.write_bytes(b"RRD" + sequence.to_bytes(2, "big") + source.read_bytes()) + return { + "source_sha256": _sha256(source), + "rrd_sha256": _sha256(destination), + "rrd_bytes": destination.stat().st_size, + "timeline": "session_time", + "timeline_start_ns": 0, + "timeline_end_ns": 2_500_000_000, + } + + +def _command(tmp_path: Path) -> ReplayCommand: + tmp_path.mkdir(parents=True, exist_ok=True) + source = tmp_path / "mqtt.raw.k1mqtt" + source.write_bytes(b"native-source-recording") + metadata = tmp_path / "mqtt.metadata.jsonl" + metadata.write_text('{"record_type":"message","sequence":1}\n', encoding="utf-8") + return ReplayCommand( + session_id="20260716T205632Z_viewer_live", + source_path=source, + allowed_root=tmp_path, + session_root=tmp_path, + replay_byte_length=source.stat().st_size, + metadata_byte_length=metadata.stat().st_size, + expected_source_sha256=None, + speed=1.0, + loop=False, + ) + + +def test_materializer_reuses_only_a_digest_validated_private_cache(tmp_path: Path) -> None: + command = _command(tmp_path) + exporter = FakeExporter() + materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) + + first = materializer.materialize(command) + second = materializer.materialize(command) + + assert first == second + assert exporter.calls == 1 + assert first.media_type == RERUN_RECORDING_MEDIA_TYPE + assert first.timeline == "session_time" + assert first.timeline_start_ns == 0 + assert first.timeline_end_ns == 2_500_000_000 + assert first.path.is_relative_to(materializer.recordings_root) + assert first.path.stat().st_mode & 0o777 == 0o600 + sidecar = first.path.with_name("scene.rrd.cache.json") + assert sidecar.stat().st_mode & 0o777 == 0o600 + document = json.loads(sidecar.read_text(encoding="utf-8")) + assert document["schema_version"] == CACHE_SCHEMA + assert "source_path" not in document + assert "recording_path" not in document + assert str(tmp_path) not in sidecar.read_text(encoding="utf-8") + + after_restart = SessionRecordingMaterializer( + tmp_path / "private", + exporter=exporter, + ).materialize(command) + assert after_restart == first + assert exporter.calls == 1 + + +@pytest.mark.parametrize( + "obsolete_schema", + [ + "missioncore.derived-rerun-recording-cache/v1", + "missioncore.derived-rerun-recording-cache/v2", + "missioncore.derived-rerun-recording-cache/v4", + "missioncore.derived-rerun-recording-cache/v5", + ], +) +def test_materializer_rebuilds_incompatible_recording_cache_schema( + tmp_path: Path, + obsolete_schema: str, +) -> None: + command = _command(tmp_path) + exporter = FakeExporter() + private_root = tmp_path / "private" + first = SessionRecordingMaterializer(private_root, exporter=exporter).materialize(command) + sidecar = first.path.with_name("scene.rrd.cache.json") + document = json.loads(sidecar.read_text(encoding="utf-8")) + document["schema_version"] = obsolete_schema + sidecar.write_text(json.dumps(document), encoding="utf-8") + + rebuilt = SessionRecordingMaterializer(private_root, exporter=exporter).materialize(command) + + assert exporter.calls == 2 + assert rebuilt.path == first.path + assert json.loads(sidecar.read_text(encoding="utf-8"))["schema_version"] == CACHE_SCHEMA + + +def test_failed_rebuild_preserves_previously_published_cache(tmp_path: Path) -> None: + command = _command(tmp_path) + exporter = FakeExporter() + materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) + first = materializer.materialize(command) + published_bytes = first.path.read_bytes() + sidecar = first.path.with_name("scene.rrd.cache.json") + published_sidecar = sidecar.read_bytes() + command.source_path.write_bytes(b"new-source-that-requires-rebuild") + changed = replace(command, replay_byte_length=command.source_path.stat().st_size) + + def fail_export(_source: Path, destination: Path) -> dict[str, object]: + destination.write_bytes(b"incomplete-candidate") + raise OSError("simulated export failure") + + failing = SessionRecordingMaterializer(tmp_path / "private", exporter=fail_export) + with pytest.raises(RecordingMaterializationError): + failing.materialize(changed) + + assert first.path.read_bytes() == published_bytes + assert sidecar.read_bytes() == published_sidecar + assert not tuple(first.path.parent.glob(".scene.*.candidate.rrd")) + + +def test_materializer_rebuilds_when_source_or_recording_changes(tmp_path: Path) -> None: + command = _command(tmp_path) + exporter = FakeExporter() + materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) + + first = materializer.materialize(command) + original_recording_stat = first.path.stat() + corrupted = bytearray(first.path.read_bytes()) + corrupted[-1] ^= 0x01 + first.path.write_bytes(corrupted) + os.utime( + first.path, + ns=(original_recording_stat.st_atime_ns, original_recording_stat.st_mtime_ns), + ) + + repaired = materializer.materialize(command) + assert exporter.calls == 2 + assert repaired.sha256 == _sha256(repaired.path) + assert repaired.sha256 != hashlib.sha256(corrupted).hexdigest() + + command.source_path.write_bytes(b"new-native-source") + command = replace(command, replay_byte_length=command.source_path.stat().st_size) + updated = materializer.materialize(command) + assert exporter.calls == 3 + assert updated.source_sha256 == _sha256(command.source_path) + + +def test_concurrent_materialization_exports_one_recording(tmp_path: Path) -> None: + command = _command(tmp_path) + exporter = FakeExporter(delay_seconds=0.05) + materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) + + with ThreadPoolExecutor(max_workers=8) as executor: + recordings = tuple(executor.map(materializer.materialize, [command] * 8)) + + assert exporter.calls == 1 + assert len({recording.sha256 for recording in recordings}) == 1 + assert len({recording.path for recording in recordings}) == 1 + + +def test_materializer_never_follows_cache_symlinks_outside_private_root( + tmp_path: Path, +) -> None: + command = _command(tmp_path) + exporter = FakeExporter() + materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) + outside = tmp_path / "outside" + outside.mkdir() + session_cache = materializer.recordings_root / command.session_id + session_cache.symlink_to(outside, target_is_directory=True) + + with pytest.raises(RecordingMaterializationError, match="must not be a symlink"): + materializer.materialize(command) + + assert list(outside.iterdir()) == [] + session_cache.unlink() + session_cache.mkdir() + outside_recording = outside / "scene.rrd" + outside_recording.write_bytes(b"do-not-touch") + (session_cache / "scene.rrd").symlink_to(outside_recording) + + recording = materializer.materialize(command) + + assert recording.path.read_bytes().startswith(b"RRD") + assert outside_recording.read_bytes() == b"do-not-touch" + + +def test_materializer_revalidates_prepared_source_without_following_replaced_symlink( + tmp_path: Path, +) -> None: + command = _command(tmp_path / "session") + exporter = FakeExporter() + materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) + outside = tmp_path / "outside.k1mqtt" + outside.write_bytes(command.source_path.read_bytes()) + command.source_path.unlink() + command.source_path.symlink_to(outside) + + with pytest.raises(RecordingMaterializationError, match="missing or unsafe"): + materializer.materialize(command) + + assert exporter.calls == 0 + assert outside.read_bytes() == b"native-source-recording" + + +def test_materializer_exports_only_validated_prefix_before_crash_tail( + tmp_path: Path, +) -> None: + command = _command(tmp_path / "session") + committed_raw_bytes = command.replay_byte_length + committed_metadata_bytes = command.metadata_byte_length + with command.source_path.open("ab") as stream: + stream.write(b"uncommitted-raw-tail") + metadata = command.source_path.with_name("mqtt.metadata.jsonl") + with metadata.open("ab") as stream: + stream.write(b'{"record_type":"message"') + observed: list[tuple[int, int]] = [] + + class PrefixExporter(FakeExporter): + def __call__(self, source: Path, destination: Path) -> dict[str, object]: + observed.append( + (source.stat().st_size, source.with_name("mqtt.metadata.jsonl").stat().st_size) + ) + return super().__call__(source, destination) + + exporter = PrefixExporter() + materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) + + recording = materializer.materialize(command) + + assert observed == [(committed_raw_bytes, committed_metadata_bytes)] + assert recording.source_sha256 == hashlib.sha256(b"native-source-recording").hexdigest() + assert command.source_path.read_bytes().endswith(b"uncommitted-raw-tail") + + +def test_global_singleflight_bounds_exports_across_different_sessions(tmp_path: Path) -> None: + first = _command(tmp_path / "first") + second = replace( + _command(tmp_path / "second"), + session_id="20260716T205633Z_viewer_live", + ) + + class ConcurrencyExporter(FakeExporter): + def __init__(self) -> None: + super().__init__() + self.active = 0 + self.max_active = 0 + + def __call__(self, source: Path, destination: Path) -> dict[str, object]: + with self._lock: + self.active += 1 + self.max_active = max(self.max_active, self.active) + try: + time.sleep(0.03) + return super().__call__(source, destination) + finally: + with self._lock: + self.active -= 1 + + exporter = ConcurrencyExporter() + materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) + + with ThreadPoolExecutor(max_workers=2) as executor: + tuple(executor.map(materializer.materialize, (first, second))) + + assert exporter.calls == 2 + assert exporter.max_active == 1 + + +def test_cross_process_file_lock_serializes_independent_materializer_instances( + tmp_path: Path, +) -> None: + first = _command(tmp_path / "first") + second = replace( + _command(tmp_path / "second"), + session_id="20260716T205634Z_viewer_live", + ) + + class ConcurrencyExporter(FakeExporter): + def __init__(self) -> None: + super().__init__() + self.active = 0 + self.max_active = 0 + + def __call__(self, source: Path, destination: Path) -> dict[str, object]: + with self._lock: + self.active += 1 + self.max_active = max(self.max_active, self.active) + try: + time.sleep(0.03) + return super().__call__(source, destination) + finally: + with self._lock: + self.active -= 1 + + exporter = ConcurrencyExporter() + private_root = tmp_path / "private" + first_materializer = SessionRecordingMaterializer(private_root, exporter=exporter) + second_materializer = SessionRecordingMaterializer(private_root, exporter=exporter) + + with ThreadPoolExecutor(max_workers=2) as executor: + first_future = executor.submit(first_materializer.materialize, first) + second_future = executor.submit(second_materializer.materialize, second) + assert first_future.result(timeout=2).path.exists() + assert second_future.result(timeout=2).path.exists() + + assert exporter.calls == 2 + assert exporter.max_active == 1 + + +def test_export_scavenges_confined_crash_candidates_before_quota_check( + tmp_path: Path, +) -> None: + command = _command(tmp_path / "session") + private_root = tmp_path / "private" + materializer = SessionRecordingMaterializer( + private_root, + exporter=FakeExporter(), + cache_max_bytes=32 * 1024, + free_space_reserve_bytes=0, + ) + session_cache = materializer.recordings_root / command.session_id + session_cache.mkdir() + stale_candidate = session_cache / ".scene.deadbeef.candidate.rrd" + stale_candidate.write_bytes(b"x" * (24 * 1024)) + stale_export_temp = session_cache / "..scene.deadbeef.candidate.rrd.uuid.tmp" + stale_export_temp.write_bytes(b"x" * 1024) + stale_stage = session_cache / ".source.deadbeef.tmp" + stale_stage.mkdir() + (stale_stage / "mqtt.raw.k1mqtt").write_bytes(b"x" * 1024) + + recording = materializer.materialize(command) + + assert recording.path.exists() + assert not stale_candidate.exists() + assert not stale_export_temp.exists() + assert not stale_stage.exists() + + +def test_cached_recording_and_response_lease_do_not_wait_for_other_export( + tmp_path: Path, +) -> None: + first = _command(tmp_path / "first") + second = replace( + _command(tmp_path / "second"), + session_id="20260716T205633Z_viewer_live", + ) + + class BlockingSecondExporter(FakeExporter): + def __init__(self) -> None: + super().__init__() + self.second_started = threading.Event() + self.release_second = threading.Event() + + def __call__(self, source: Path, destination: Path) -> dict[str, object]: + with self._lock: + next_call = self.calls + 1 + if next_call == 2: + self.second_started.set() + if not self.release_second.wait(timeout=2): + raise AssertionError("test did not release the blocked export") + return super().__call__(source, destination) + + exporter = BlockingSecondExporter() + materializer = SessionRecordingMaterializer(tmp_path / "private", exporter=exporter) + first_recording = materializer.materialize(first) + + with ThreadPoolExecutor(max_workers=3) as executor: + exporting = executor.submit(materializer.materialize, second) + assert exporter.second_started.wait(timeout=1) + + cached = executor.submit(materializer.materialize, first).result(timeout=0.2) + pinned, release = executor.submit( + materializer.materialize_pinned, + first, + ).result(timeout=0.2) + + assert cached == first_recording + assert pinned == first_recording + release() + exporter.release_second.set() + assert exporting.result(timeout=1).session_id == second.session_id + + +def test_cache_quota_evicts_lru_derived_recording_without_deleting_raw( + tmp_path: Path, +) -> None: + first = _command(tmp_path / "first") + second = replace( + _command(tmp_path / "second"), + session_id="20260716T205633Z_viewer_live", + ) + + class SizedExporter(FakeExporter): + def __call__(self, source: Path, destination: Path) -> dict[str, object]: + with self._lock: + self.calls += 1 + destination.write_bytes(b"R" * (40 * 1024)) + return { + "source_sha256": _sha256(source), + "rrd_sha256": _sha256(destination), + "rrd_bytes": destination.stat().st_size, + "timeline": "session_time", + "timeline_start_ns": 0, + "timeline_end_ns": 1, + } + + materializer = SessionRecordingMaterializer( + tmp_path / "private", + exporter=SizedExporter(), + cache_max_bytes=50 * 1024, + free_space_reserve_bytes=0, + ) + first_recording = materializer.materialize(first) + first_source_bytes = first.source_path.read_bytes() + + second_recording = materializer.materialize(second) + + assert not first_recording.path.exists() + assert second_recording.path.exists() + assert first.source_path.read_bytes() == first_source_bytes + assert second.source_path.exists() + + +def test_pinned_cache_entry_survives_eviction_until_release(tmp_path: Path) -> None: + first = _command(tmp_path / "first") + second = replace( + _command(tmp_path / "second"), + session_id="20260716T205633Z_viewer_live", + ) + + class SizedExporter(FakeExporter): + def __call__(self, source: Path, destination: Path) -> dict[str, object]: + with self._lock: + self.calls += 1 + destination.write_bytes(b"R" * (40 * 1024)) + return { + "source_sha256": _sha256(source), + "rrd_sha256": _sha256(destination), + "rrd_bytes": destination.stat().st_size, + "timeline": "session_time", + "timeline_start_ns": 0, + "timeline_end_ns": 1, + } + + materializer = SessionRecordingMaterializer( + tmp_path / "private", + exporter=SizedExporter(), + cache_max_bytes=50 * 1024, + free_space_reserve_bytes=0, + ) + first_recording, release = materializer.materialize_pinned(first) + + with pytest.raises(RecordingMaterializationError, match="cache quota"): + materializer.materialize(second) + + assert first_recording.path.exists() + + release() + second_recording = materializer.materialize(second) + + assert not first_recording.path.exists() + assert second_recording.path.exists() + + +def test_cache_free_space_reserve_refuses_export_without_touching_raw( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + command = _command(tmp_path / "session") + source_bytes = command.source_path.read_bytes() + materializer = SessionRecordingMaterializer( + tmp_path / "private", + exporter=FakeExporter(), + cache_max_bytes=1024 * 1024, + free_space_reserve_bytes=1024, + ) + monkeypatch.setattr( + recording_module.shutil, + "disk_usage", + lambda _path: SimpleNamespace(free=1023), + ) + + with pytest.raises(RecordingMaterializationError, match="free-space reserve"): + materializer.materialize(command) + + assert command.source_path.read_bytes() == source_bytes diff --git a/tests/test_session_store.py b/tests/test_session_store.py new file mode 100644 index 0000000..b9f7252 --- /dev/null +++ b/tests/test_session_store.py @@ -0,0 +1,573 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +import sqlite3 +from pathlib import Path + +import pytest + +from k1link.mqtt.capture import FRAME_HEADER, RAW_MAGIC, iter_capture_frames +from k1link.sessions import ( + LayoutConflictError, + SessionIntegrityError, + SessionNotFoundError, + SessionStore, + resolve_missioncore_evidence_dir, +) + + +def make_legacy_session( + sessions_root: Path, + session_id: str, + *, + created_at: str = "2026-07-16T20:56:32.699Z", +) -> Path: + session = sessions_root / session_id + capture = session / "captures" / "mqtt_live" + capture.mkdir(parents=True) + frames = [ + ("lixel/application/report/lio_pcl", b"point-frame"), + ("lixel/application/report/lio_pose", b"pose-frame"), + ] + raw = bytearray(RAW_MAGIC) + metadata: list[dict[str, object]] = [] + for sequence, (topic, payload) in enumerate(frames, start=1): + topic_bytes = topic.encode() + frame_offset = len(raw) + raw.extend(FRAME_HEADER.pack(len(topic_bytes), len(payload))) + raw.extend(topic_bytes) + raw.extend(payload) + metadata.append( + { + "schema_version": 1, + "record_type": "message", + "sequence": sequence, + "received_at_utc": f"2026-07-16T20:56:{31 + sequence:02d}.699Z", + "received_at_epoch_ns": 1_784_235_391_699_000_000 + sequence * 1_000_000_000, + "received_monotonic_ns": 9_000_000_000 + sequence * 1_000_000_000, + "topic": topic, + "payload_bytes": len(payload), + "raw_frame_offset": frame_offset, + "raw_payload_offset": frame_offset + FRAME_HEADER.size + len(topic_bytes), + "raw_frame_bytes": FRAME_HEADER.size + len(topic_bytes) + len(payload), + } + ) + raw_path = capture / "mqtt.raw.k1mqtt" + raw_path.write_bytes(raw) + raw_hash = hashlib.sha256(raw).hexdigest() + metadata_path = capture / "mqtt.metadata.jsonl" + metadata_path.write_text( + "".join(json.dumps(record, separators=(",", ":")) + "\n" for record in metadata), + encoding="utf-8", + ) + metadata_hash = hashlib.sha256(metadata_path.read_bytes()).hexdigest() + (capture / "mqtt.summary.json").write_text( + json.dumps( + { + "created_at_utc": created_at, + "completed_at_utc": "2026-07-16T21:20:43.018Z", + "capture_elapsed_seconds": 1440.1, + "stop_reason": "external_stop", + "error": None, + "message_count": 2, + "raw_bytes": len(raw), + "payload_bytes": sum(len(payload) for _, payload in frames), + "topic_counts": { + "lixel/application/report/lio_pcl": 1, + "lixel/application/report/lio_pose": 1, + }, + "artifact_hashes": { + "raw_sha256": raw_hash, + "metadata_jsonl_sha256": metadata_hash, + }, + # This is deliberately sensitive and must not enter API DTOs. + "target_ipv4": "192.168.99.77", + } + ), + encoding="utf-8", + ) + (session / "manifest.redacted.json").write_text( + json.dumps({"started_at_utc": created_at, "target": "redacted"}), + encoding="utf-8", + ) + return session + + +def test_evidence_root_is_private_and_configurable( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository = tmp_path / "repo" + assert resolve_missioncore_evidence_dir(repository) == ( + repository / ".runtime" / "mission-core" / "evidence" / "sessions" + ).resolve() + + configured = tmp_path / "external-evidence" + monkeypatch.setenv("MISSIONCORE_EVIDENCE_DIR", str(configured)) + assert resolve_missioncore_evidence_dir(repository) == configured.resolve() + + +def test_catalog_reconciles_sessions_removed_from_one_evidence_root(tmp_path: Path) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + store = SessionStore(repository, data_dir=tmp_path / "data") + + assert store.import_legacy_viewer_live(sessions) == (session.name,) + shutil.rmtree(session) + + assert store.import_legacy_viewer_live(sessions) == () + assert store.list_recent().items == () + + +def make_recorded_camera_source( + session: Path, + source_id: str = "sensor.camera.left", + *, + complete: bool = True, +) -> Path: + epoch = session / "media" / source_id / "epoch-1" + segments = epoch / "segments" + segments.mkdir(parents=True) + (epoch / "init.mp4").write_bytes(b"ftyp-mission-core") + (segments / "1.m4s").write_bytes(b"moof-camera-frame") + (epoch / "index.jsonl").write_text( + json.dumps({"sequence": 1, "started_at_seconds": 0.0}) + "\n", + encoding="utf-8", + ) + if complete: + (epoch / "summary.json").write_text( + json.dumps( + { + "schema_version": "missioncore.camera-recording/v1", + "source_id": source_id, + "segment_count": 1, + } + ), + encoding="utf-8", + ) + return epoch + + +def replace_summary_with_recovery_metadata( + session: Path, + *, + corrupt_trailing_line: bool = False, +) -> None: + capture = session / "captures" / "mqtt_live" + raw_path = capture / "mqtt.raw.k1mqtt" + timestamps = ( + ("2026-07-16T20:56:32.699Z", 1_784_235_392_699_000_000, 10_000_000_000), + ("2026-07-16T20:56:35.199Z", 1_784_235_395_199_000_000, 12_500_000_000), + ) + records = [] + for frame, (timestamp, epoch_ns, monotonic_ns) in zip( + iter_capture_frames(raw_path), + timestamps, + strict=True, + ): + records.append( + { + "schema_version": 1, + "record_type": "message", + "sequence": frame.sequence, + "received_at_utc": timestamp, + "received_at_epoch_ns": epoch_ns, + "received_monotonic_ns": monotonic_ns, + "topic": frame.topic, + "payload_bytes": frame.raw_frame_bytes + - FRAME_HEADER.size + - len(frame.topic.encode("utf-8")), + "raw_frame_offset": frame.raw_frame_offset, + "raw_payload_offset": frame.raw_payload_offset, + "raw_frame_bytes": frame.raw_frame_bytes, + } + ) + metadata = b"".join( + (json.dumps(record, separators=(",", ":")) + "\n").encode("utf-8") + for record in records + ) + if corrupt_trailing_line: + metadata += b'{"record_type":"message"' + (capture / "mqtt.metadata.jsonl").write_bytes(metadata) + (capture / "mqtt.summary.json").write_text("{corrupt", encoding="utf-8") + + +def serialized(value: object) -> str: + return json.dumps(value, ensure_ascii=False, default=str) + + +def test_store_uses_private_wal_database_and_idempotently_imports_legacy_session( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + make_legacy_session(sessions, "20260716T205632Z_viewer_live") + store = SessionStore(repository, data_dir=tmp_path / "data") + + first = store.import_legacy_viewer_live(sessions) + second = store.import_legacy_viewer_live(sessions) + + assert first == second == ("20260716T205632Z_viewer_live",) + page = store.list_recent() + assert len(page.items) == 1 + summary = page.items[0] + assert summary.status == "ready" + assert summary.modalities == ("point-cloud", "trajectory") + assert summary.replayable is True + assert summary.source_count == 2 + assert "192.168.99.77" not in serialized(page.as_dict()) + assert str(repository) not in serialized(page.as_dict()) + + detail = store.get_session(summary.session_id) + assert [source.source_id for source in detail.sources] == [ + "sensor.lidar.primary", + "spatial.trajectory", + ] + assert all(source.seekable for source in detail.sources) + assert str(repository) not in serialized(detail.as_dict()) + command = store.prepare_replay(summary.session_id, speed=2.0, loop=True) + assert command.source_path.name == "mqtt.raw.k1mqtt" + assert command.speed == 2.0 + assert command.loop is True + + with sqlite3.connect(store.database_path) as connection: + assert connection.execute("PRAGMA journal_mode").fetchone()[0] == "wal" + assert store.database_path.stat().st_mode & 0o777 == 0o600 + + +def test_recent_sessions_are_sorted_and_cursor_paginated(tmp_path: Path) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + make_legacy_session( + sessions, + "20260716T191025Z_viewer_live", + created_at="2026-07-16T19:10:25.352Z", + ) + make_legacy_session( + sessions, + "20260716T205632Z_viewer_live", + created_at="2026-07-16T20:56:32.699Z", + ) + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + + first = store.list_recent(limit=1) + assert first.items[0].session_id == "20260716T205632Z_viewer_live" + assert first.next_cursor == "20260716T205632Z_viewer_live" + second = store.list_recent(limit=1, cursor=first.next_cursor) + assert second.items[0].session_id == "20260716T191025Z_viewer_live" + assert second.next_cursor is None + + +def test_legacy_import_adds_video_only_for_validated_recording_tree(tmp_path: Path) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + complete = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + incomplete = make_legacy_session(sessions, "20260716T205632Z_viewer_live_2") + make_recorded_camera_source(complete) + make_recorded_camera_source(incomplete, complete=False) + store = SessionStore(repository, data_dir=tmp_path / "data") + + store.import_legacy_viewer_live(sessions) + + complete_detail = store.get_session(complete.name) + assert complete_detail.summary.modalities == ("point-cloud", "trajectory", "video") + assert complete_detail.summary.source_count == 3 + camera = next( + source for source in complete_detail.sources if source.source_id == "sensor.camera.left" + ) + assert camera.modality == "video" + assert camera.semantic_channel_id == "camera.video.recorded" + assert camera.seekable is True + video_artifact = next( + artifact + for artifact in complete_detail.artifacts + if artifact.artifact_id == camera.artifact_id + ) + assert video_artifact.kind == "recorded-video" + assert video_artifact.integrity_status == "validated-structure" + assert str(repository) not in serialized(complete_detail.as_dict()) + + incomplete_detail = store.get_session(incomplete.name) + assert incomplete_detail.summary.modalities == ("point-cloud", "trajectory") + assert all(source.modality != "video" for source in incomplete_detail.sources) + + +def test_interrupted_capture_is_recovered_from_aligned_raw_and_metadata_prefix( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + replace_summary_with_recovery_metadata(session, corrupt_trailing_line=True) + store = SessionStore(repository, data_dir=tmp_path / "data") + + store.import_legacy_viewer_live(sessions) + + detail = store.get_session(session.name) + assert detail.summary.status == "interrupted" + assert detail.summary.replayable is True + assert detail.summary.modalities == ("point-cloud", "trajectory") + assert detail.summary.started_at_utc == "2026-07-16T20:56:32.699Z" + assert detail.summary.completed_at_utc == "2026-07-16T20:56:35.199Z" + assert detail.summary.duration_seconds == 2.5 + assert detail.artifacts[0].integrity_status == "validated-prefix" + assert store.prepare_replay(session.name).source_path.name == "mqtt.raw.k1mqtt" + + +def test_catalog_upsert_promotes_recovered_session_after_summary_is_completed( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + summary_path = session / "captures" / "mqtt_live" / "mqtt.summary.json" + metadata_path = session / "captures" / "mqtt_live" / "mqtt.metadata.jsonl" + completed_summary = summary_path.read_text(encoding="utf-8") + completed_metadata = metadata_path.read_text(encoding="utf-8") + replace_summary_with_recovery_metadata(session) + store = SessionStore(repository, data_dir=tmp_path / "data") + + store.import_legacy_viewer_live(sessions) + assert store.get_session(session.name).summary.status == "interrupted" + + summary_path.write_text(completed_summary, encoding="utf-8") + metadata_path.write_text(completed_metadata, encoding="utf-8") + store.import_legacy_viewer_live(sessions) + + promoted = store.get_session(session.name).summary + assert promoted.status == "ready" + assert promoted.duration_seconds == 1440.1 + assert promoted.replayable is True + + +def test_interrupted_capture_does_not_trust_metadata_outside_session(tmp_path: Path) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + replace_summary_with_recovery_metadata(session) + metadata = session / "captures" / "mqtt_live" / "mqtt.metadata.jsonl" + outside = tmp_path / "outside.metadata.jsonl" + outside.write_bytes(metadata.read_bytes()) + metadata.unlink() + metadata.symlink_to(outside) + store = SessionStore(repository, data_dir=tmp_path / "data") + + store.import_legacy_viewer_live(sessions) + + detail = store.get_session(session.name) + assert detail.summary.status == "failed" + assert detail.summary.replayable is False + assert detail.summary.modalities == () + + +def test_interrupted_capture_rejects_newline_terminated_metadata_corruption( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + replace_summary_with_recovery_metadata(session) + metadata = session / "captures" / "mqtt_live" / "mqtt.metadata.jsonl" + with metadata.open("ab") as stream: + stream.write(b"{corrupt\n") + store = SessionStore(repository, data_dir=tmp_path / "data") + + store.import_legacy_viewer_live(sessions) + + detail = store.get_session(session.name) + assert detail.summary.status == "failed" + assert detail.summary.replayable is False + + +def test_replay_resolution_rejects_artifact_symlink_escape(tmp_path: Path) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + store = SessionStore(repository, data_dir=tmp_path / "data") + store.import_legacy_viewer_live(sessions) + raw = session / "captures" / "mqtt_live" / "mqtt.raw.k1mqtt" + outside = tmp_path / "outside.k1mqtt" + outside.write_bytes(raw.read_bytes()) + raw.unlink() + raw.symlink_to(outside) + + with pytest.raises(SessionIntegrityError, match="escapes"): + store.prepare_replay(session.name) + + +def test_completed_capture_is_not_replayable_when_declared_integrity_fails( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + raw = session / "captures" / "mqtt_live" / "mqtt.raw.k1mqtt" + corrupted = bytearray(raw.read_bytes()) + corrupted[-1] ^= 0x01 + raw.write_bytes(corrupted) + store = SessionStore(repository, data_dir=tmp_path / "data") + + store.import_legacy_viewer_live(sessions) + + detail = store.get_session(session.name) + assert detail.summary.status == "failed" + assert detail.summary.replayable is False + assert detail.summary.modalities == () + + +def test_completed_capture_is_not_replayable_when_declared_count_is_wrong( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + summary_path = session / "captures" / "mqtt_live" / "mqtt.summary.json" + summary = json.loads(summary_path.read_text(encoding="utf-8")) + summary["message_count"] = 3 + summary_path.write_text(json.dumps(summary), encoding="utf-8") + store = SessionStore(repository, data_dir=tmp_path / "data") + + store.import_legacy_viewer_live(sessions) + + assert store.get_session(session.name).summary.replayable is False + + +def test_current_session_marker_keeps_active_capture_out_of_replay(tmp_path: Path) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + (sessions / ".current_session").write_text( + f"sessions/{session.name}\n", + encoding="utf-8", + ) + store = SessionStore(repository, data_dir=tmp_path / "data") + + store.import_legacy_viewer_live(sessions) + + with pytest.raises(SessionNotFoundError): + store.get_session(session.name) + + (sessions / ".current_session").unlink() + store.import_legacy_viewer_live(sessions) + assert store.get_session(session.name).summary.replayable is True + + +def test_interrupted_capture_replays_only_committed_prefix_before_partial_raw_tail( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + replace_summary_with_recovery_metadata(session) + raw = session / "captures" / "mqtt_live" / "mqtt.raw.k1mqtt" + committed_bytes = raw.stat().st_size + with raw.open("ab") as stream: + stream.write(FRAME_HEADER.pack(12, 100)[:7]) + store = SessionStore(repository, data_dir=tmp_path / "data") + + store.import_legacy_viewer_live(sessions) + + detail = store.get_session(session.name) + command = store.prepare_replay(session.name) + assert detail.summary.status == "interrupted" + assert detail.summary.replayable is True + assert command.replay_byte_length == committed_bytes + assert command.replay_byte_length < command.source_path.stat().st_size + + +def test_interrupted_capture_rejects_non_frame_raw_tail(tmp_path: Path) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + replace_summary_with_recovery_metadata(session) + raw = session / "captures" / "mqtt_live" / "mqtt.raw.k1mqtt" + with raw.open("ab") as stream: + stream.write(FRAME_HEADER.pack(0, 0)) + store = SessionStore(repository, data_dir=tmp_path / "data") + + store.import_legacy_viewer_live(sessions) + + assert store.get_session(session.name).summary.replayable is False + + +def test_interrupted_capture_tolerates_bounded_group_commit_raw_tail( + tmp_path: Path, +) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + replace_summary_with_recovery_metadata(session) + raw = session / "captures" / "mqtt_live" / "mqtt.raw.k1mqtt" + committed_bytes = raw.stat().st_size + with raw.open("ab") as stream: + for payload in (b"pending-one", b"pending-two"): + topic = b"RealtimePath" + stream.write(FRAME_HEADER.pack(len(topic), len(payload))) + stream.write(topic) + stream.write(payload) + store = SessionStore(repository, data_dir=tmp_path / "data") + + store.import_legacy_viewer_live(sessions) + + command = store.prepare_replay(session.name) + assert store.get_session(session.name).summary.status == "interrupted" + assert command.replay_byte_length == committed_bytes + + +def test_failed_final_summary_falls_back_to_last_committed_prefix(tmp_path: Path) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + capture = session / "captures" / "mqtt_live" + metadata = capture / "mqtt.metadata.jsonl" + first_record = metadata.read_text(encoding="utf-8").splitlines(keepends=True)[0] + metadata.write_text(first_record, encoding="utf-8") + summary_path = capture / "mqtt.summary.json" + summary = json.loads(summary_path.read_text(encoding="utf-8")) + summary["error"] = "synthetic metadata fsync failure" + summary_path.write_text(json.dumps(summary), encoding="utf-8") + store = SessionStore(repository, data_dir=tmp_path / "data") + + store.import_legacy_viewer_live(sessions) + + detail = store.get_session(session.name) + assert detail.summary.status == "interrupted" + assert detail.summary.replayable is True + assert detail.summary.modalities == ("point-cloud",) + + +def test_layout_save_is_atomic_and_revision_checked(tmp_path: Path) -> None: + store = SessionStore(tmp_path / "repo", data_dir=tmp_path / "data") + + first = store.save_layout( + "observation.spatial", + schema_version=1, + expected_revision=0, + name="Операторская сцена", + layout={"visible_source_ids": ["sensor.lidar.primary"], "windows": []}, + ) + assert first.revision == 1 + assert store.get_layout("observation.spatial") == first + + with pytest.raises(LayoutConflictError, match="revision changed"): + store.save_layout( + "observation.spatial", + schema_version=1, + expected_revision=0, + name="Устаревшая запись", + layout={}, + ) + + second = store.save_layout( + "observation.spatial", + schema_version=1, + expected_revision=1, + name="Операторская сцена", + layout={"visible_source_ids": [], "windows": []}, + ) + assert second.revision == 2 + assert store.get_layout("observation.spatial").layout["visible_source_ids"] == [] diff --git a/tests/test_viewer_replay.py b/tests/test_viewer_replay.py index 02a3b0d..a93bc07 100644 --- a/tests/test_viewer_replay.py +++ b/tests/test_viewer_replay.py @@ -16,6 +16,14 @@ def _write_native(path: Path, topic: str, payload: bytes) -> None: ) +def _append_native(path: Path, topic: str, payload: bytes) -> None: + topic_raw = topic.encode() + with path.open("ab") as stream: + stream.write(FRAME_HEADER.pack(len(topic_raw), len(payload))) + stream.write(topic_raw) + stream.write(payload) + + def test_native_replay_uses_aligned_metadata_timing(tmp_path: Path) -> None: capture = tmp_path / "mqtt.raw.k1mqtt" _write_native(capture, "lixel/application/report/lio_pose", b"pose") @@ -39,6 +47,35 @@ def test_native_replay_uses_aligned_metadata_timing(tmp_path: Path) -> None: assert message.received_monotonic_ns == 456 +def test_native_replay_stops_at_crash_truncated_metadata_tail(tmp_path: Path) -> None: + capture = tmp_path / "mqtt.raw.k1mqtt" + _write_native(capture, "RealtimePointcloud", b"first") + _append_native(capture, "RealtimePointcloud", b"uncommitted-tail") + first_record = { + "record_type": "message", + "sequence": 1, + "received_at_epoch_ns": 123_000_000_000, + "received_monotonic_ns": 456, + } + (tmp_path / "mqtt.metadata.jsonl").write_text( + json.dumps(first_record) + "\n" + '{"record_type":"message"', + encoding="utf-8", + ) + + messages = list(iter_replay_messages(capture)) + + assert [message.payload for message in messages] == [b"first"] + + +def test_native_replay_rejects_newline_terminated_metadata_corruption(tmp_path: Path) -> None: + capture = tmp_path / "mqtt.raw.k1mqtt" + _write_native(capture, "RealtimePointcloud", b"frame") + (tmp_path / "mqtt.metadata.jsonl").write_text("{corrupt\n", encoding="utf-8") + + with pytest.raises(ReplayFormatError, match="not valid JSON"): + list(iter_replay_messages(capture)) + + def test_legacy_tsv_replay_validates_and_decodes_payload(tmp_path: Path) -> None: capture = tmp_path / "payloads.tsv" capture.write_bytes(b"1784124315.186225000\tRealtimePath\t4\t0001aaff\n")