feat(plugins): isolate device integrations
This commit is contained in:
DCCONSTRUCTIONS
@@ -14,12 +14,22 @@ from .media import (
|
||||
)
|
||||
from .models import (
|
||||
LayoutConflictError,
|
||||
ObservationArtifactCandidate,
|
||||
ObservationSessionCandidate,
|
||||
RecordedMediaArtifact,
|
||||
ReplayArtifact,
|
||||
ReplayCommand,
|
||||
SessionIntegrityError,
|
||||
SessionNotFoundError,
|
||||
SessionNotReplayableError,
|
||||
)
|
||||
from .plugin_contract import (
|
||||
ObservationArchiveSource,
|
||||
ObservationRuntimeContribution,
|
||||
PluginRecordingExportCancelled,
|
||||
PluginRecordingExportError,
|
||||
RecordingExporter,
|
||||
)
|
||||
from .preparation import (
|
||||
RecordingPreparationQueueFull,
|
||||
RecordingPreparationSnapshot,
|
||||
@@ -42,6 +52,12 @@ __all__ = [
|
||||
"ActiveSessionLease",
|
||||
"ActiveSessionLeaseError",
|
||||
"MaterializedRecording",
|
||||
"ObservationArchiveSource",
|
||||
"ObservationArtifactCandidate",
|
||||
"ObservationRuntimeContribution",
|
||||
"ObservationSessionCandidate",
|
||||
"PluginRecordingExportCancelled",
|
||||
"PluginRecordingExportError",
|
||||
"RecordingMaterializationCancelled",
|
||||
"RecordedMediaArtifact",
|
||||
"RECORDED_MEDIA_MANIFEST_SCHEMA",
|
||||
@@ -49,6 +65,8 @@ __all__ = [
|
||||
"RecordedMediaInspector",
|
||||
"RecordedMediaManifest",
|
||||
"ReplayCommand",
|
||||
"ReplayArtifact",
|
||||
"RecordingExporter",
|
||||
"RecordingMaterializationError",
|
||||
"RecordingPreparationQueueFull",
|
||||
"RecordingPreparationSnapshot",
|
||||
|
||||
@@ -1,818 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import IO, Any
|
||||
|
||||
from k1link.mqtt.capture import (
|
||||
FRAME_HEADER,
|
||||
GROUP_COMMIT_MAX_BYTES,
|
||||
GROUP_COMMIT_MAX_MESSAGES,
|
||||
MAX_CONFIGURABLE_MESSAGE_BYTES,
|
||||
MAX_TOPIC_BYTES,
|
||||
RAW_MAGIC,
|
||||
)
|
||||
|
||||
from .models import (
|
||||
LegacyMediaSourceCandidate,
|
||||
LegacySessionCandidate,
|
||||
SessionModality,
|
||||
SessionStatus,
|
||||
)
|
||||
|
||||
MAX_LEGACY_JSON_BYTES = 2 * 1024 * 1024
|
||||
LEGACY_SESSION_PATTERN = re.compile(r"^[A-Za-z0-9._-]+_viewer_live(?:_[0-9]+)?$")
|
||||
SHA256_PATTERN = re.compile(r"^[a-f0-9]{64}$")
|
||||
MEDIA_SOURCE_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
MEDIA_EPOCH_PATTERN = re.compile(r"^epoch-(?!0+$)[0-9]+$")
|
||||
MEDIA_SEGMENT_PATTERN = re.compile(r"^[0-9]+\.m4s$")
|
||||
MAX_MEDIA_INDEX_BYTES = 32 * 1024 * 1024
|
||||
MAX_RECOVERY_METADATA_BYTES = 64 * 1024 * 1024
|
||||
MAX_RECOVERY_METADATA_LINE_BYTES = 64 * 1024
|
||||
MAX_RECOVERY_MESSAGES = 500_000
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _RecoveredCapture:
|
||||
message_count: int
|
||||
topic_counts: dict[str, int]
|
||||
started_at_utc: str
|
||||
completed_at_utc: str
|
||||
duration_seconds: float
|
||||
raw_committed_bytes: int
|
||||
metadata_committed_bytes: int
|
||||
|
||||
|
||||
def discover_legacy_viewer_sessions(root: Path) -> tuple[LegacySessionCandidate, ...]:
|
||||
"""Describe legacy live sessions without copying or decoding their payloads."""
|
||||
|
||||
allowed_root = root.expanduser().resolve()
|
||||
if not allowed_root.is_dir():
|
||||
return ()
|
||||
active_session_root = _active_session_root(allowed_root)
|
||||
candidates: list[LegacySessionCandidate] = []
|
||||
for entry in sorted(allowed_root.iterdir()):
|
||||
if not entry.is_dir() or not LEGACY_SESSION_PATTERN.fullmatch(entry.name):
|
||||
continue
|
||||
resolved = entry.resolve()
|
||||
if not resolved.is_relative_to(allowed_root):
|
||||
continue
|
||||
# The saved-session catalog deliberately omits the writer-owned run.
|
||||
# Besides avoiding a misleading interrupted/error row, this prevents
|
||||
# the two-second reconciler from repeatedly scanning and hashing a
|
||||
# capture that is still growing. Marker release makes it discoverable
|
||||
# on the next reconciliation pass.
|
||||
if active_session_root == resolved:
|
||||
continue
|
||||
candidates.append(
|
||||
_describe_session(
|
||||
allowed_root,
|
||||
resolved,
|
||||
active=False,
|
||||
)
|
||||
)
|
||||
return tuple(candidates)
|
||||
|
||||
|
||||
def _describe_session(
|
||||
allowed_root: Path,
|
||||
session_root: Path,
|
||||
*,
|
||||
active: bool,
|
||||
) -> LegacySessionCandidate:
|
||||
capture_root = session_root / "captures" / "mqtt_live"
|
||||
raw_path = capture_root / "mqtt.raw.k1mqtt"
|
||||
summary = _read_json_object(capture_root / "mqtt.summary.json")
|
||||
manifest = _read_json_object(session_root / "manifest.redacted.json")
|
||||
raw_bytes = raw_path.stat().st_size if _confined_file(raw_path, session_root) else 0
|
||||
raw_magic_ok = False
|
||||
if raw_bytes >= len(RAW_MAGIC):
|
||||
with raw_path.open("rb") as stream:
|
||||
raw_magic_ok = stream.read(len(RAW_MAGIC)) == RAW_MAGIC
|
||||
|
||||
has_completed_summary = _is_completed_summary(summary)
|
||||
completed = (
|
||||
_validate_completed_capture(capture_root, session_root, raw_path, summary)
|
||||
if has_completed_summary and raw_magic_ok
|
||||
else None
|
||||
)
|
||||
recovered = (
|
||||
_recover_interrupted_capture(capture_root, session_root, raw_path)
|
||||
if raw_magic_ok
|
||||
and completed is None
|
||||
and (not has_completed_summary or summary.get("error") is not None)
|
||||
else None
|
||||
)
|
||||
message_count = (
|
||||
_non_negative_int(summary.get("message_count"))
|
||||
if completed is not None
|
||||
else recovered.message_count if recovered is not None else 0
|
||||
)
|
||||
summary_topic_counts = summary.get("topic_counts")
|
||||
topic_counts = (
|
||||
_normalized_topic_counts(summary_topic_counts)
|
||||
if completed is not None
|
||||
else recovered.topic_counts if recovered is not None else {}
|
||||
)
|
||||
media_sources = _discover_media_sources(session_root)
|
||||
modalities = list(_modalities(topic_counts))
|
||||
if media_sources:
|
||||
modalities.append("video")
|
||||
replayable = bool(
|
||||
not active
|
||||
and
|
||||
raw_magic_ok
|
||||
and raw_bytes > len(RAW_MAGIC)
|
||||
and message_count > 0
|
||||
and any(modality in {"point-cloud", "trajectory"} for modality in modalities)
|
||||
)
|
||||
if completed is not None:
|
||||
status = (
|
||||
"interrupted"
|
||||
if active
|
||||
else _status(replayable, summary.get("error"), summary.get("stop_reason"))
|
||||
)
|
||||
started_at = (
|
||||
_safe_timestamp(summary.get("created_at_utc"))
|
||||
or _safe_timestamp(manifest.get("started_at_utc"))
|
||||
or _timestamp_from_session_name(session_root.name)
|
||||
)
|
||||
completed_at = _safe_timestamp(summary.get("completed_at_utc")) or _safe_timestamp(
|
||||
manifest.get("completed_at_utc")
|
||||
)
|
||||
duration = _duration(summary.get("capture_elapsed_seconds"))
|
||||
declared_hash = _declared_raw_hash(summary)
|
||||
integrity_status = "verified" if declared_hash is not None else "validated-structure"
|
||||
else:
|
||||
status = "interrupted" if recovered is not None or active else "failed"
|
||||
started_at = (
|
||||
recovered.started_at_utc
|
||||
if recovered is not None
|
||||
else _safe_timestamp(manifest.get("started_at_utc"))
|
||||
or _timestamp_from_session_name(session_root.name)
|
||||
)
|
||||
completed_at = recovered.completed_at_utc if recovered is not None else None
|
||||
duration = recovered.duration_seconds if recovered is not None else None
|
||||
declared_hash = None
|
||||
integrity_status = "validated-prefix" if recovered is not None else "unverified"
|
||||
|
||||
replay_raw_bytes = (
|
||||
completed.raw_committed_bytes
|
||||
if completed is not None
|
||||
else recovered.raw_committed_bytes if recovered is not None else 0
|
||||
)
|
||||
replay_metadata_bytes = (
|
||||
completed.metadata_committed_bytes
|
||||
if completed is not None
|
||||
else recovered.metadata_committed_bytes if recovered is not None else 0
|
||||
)
|
||||
|
||||
return LegacySessionCandidate(
|
||||
session_id=session_root.name,
|
||||
display_name=session_root.name,
|
||||
status=status,
|
||||
started_at_utc=started_at,
|
||||
completed_at_utc=completed_at,
|
||||
duration_seconds=duration,
|
||||
modalities=tuple(modalities),
|
||||
replayable=replayable,
|
||||
total_bytes=raw_bytes + sum(source.byte_length for source in media_sources),
|
||||
allowed_root=allowed_root,
|
||||
session_root=session_root,
|
||||
raw_path=raw_path.resolve(strict=False),
|
||||
raw_byte_length=raw_bytes,
|
||||
replay_raw_byte_length=replay_raw_bytes,
|
||||
replay_metadata_byte_length=replay_metadata_bytes,
|
||||
raw_sha256=declared_hash,
|
||||
raw_integrity_status=integrity_status,
|
||||
media_sources=media_sources,
|
||||
)
|
||||
|
||||
|
||||
def _is_completed_summary(summary: dict[str, Any]) -> bool:
|
||||
return (
|
||||
"message_count" in summary
|
||||
and isinstance(summary.get("message_count"), int)
|
||||
and not isinstance(summary.get("message_count"), bool)
|
||||
and isinstance(summary.get("topic_counts"), dict)
|
||||
and isinstance(summary.get("stop_reason"), str)
|
||||
)
|
||||
|
||||
|
||||
def _validate_completed_capture(
|
||||
capture_root: Path,
|
||||
session_root: Path,
|
||||
raw_path: Path,
|
||||
summary: dict[str, Any],
|
||||
) -> _RecoveredCapture | None:
|
||||
metadata_path = capture_root / "mqtt.metadata.jsonl"
|
||||
try:
|
||||
raw_stat = raw_path.lstat()
|
||||
metadata_stat = metadata_path.lstat()
|
||||
except OSError:
|
||||
return None
|
||||
if not stat.S_ISREG(raw_stat.st_mode) or not stat.S_ISREG(metadata_stat.st_mode):
|
||||
return None
|
||||
fingerprint = json.dumps(
|
||||
{
|
||||
"message_count": summary.get("message_count"),
|
||||
"topic_counts": summary.get("topic_counts"),
|
||||
"raw_bytes": summary.get("raw_bytes"),
|
||||
"artifact_hashes": summary.get("artifact_hashes"),
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
return _validate_completed_capture_cached(
|
||||
str(capture_root),
|
||||
str(session_root),
|
||||
str(raw_path),
|
||||
_stat_identity(raw_stat),
|
||||
_stat_identity(metadata_stat),
|
||||
fingerprint,
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
def _validate_completed_capture_cached(
|
||||
capture_root_text: str,
|
||||
session_root_text: str,
|
||||
raw_path_text: str,
|
||||
_raw_identity: tuple[int, int, int, int, int],
|
||||
_metadata_identity: tuple[int, int, int, int, int],
|
||||
summary_fingerprint: str,
|
||||
) -> _RecoveredCapture | None:
|
||||
capture_root = Path(capture_root_text)
|
||||
session_root = Path(session_root_text)
|
||||
raw_path = Path(raw_path_text)
|
||||
summary = json.loads(summary_fingerprint)
|
||||
validated = _scan_capture_prefix(
|
||||
capture_root,
|
||||
session_root,
|
||||
raw_path,
|
||||
tolerate_incomplete_metadata_tail=False,
|
||||
tolerate_raw_crash_tail=False,
|
||||
)
|
||||
if validated is None:
|
||||
return None
|
||||
if validated.message_count != _non_negative_int(summary.get("message_count")):
|
||||
return None
|
||||
if validated.topic_counts != _normalized_topic_counts(summary.get("topic_counts")):
|
||||
return None
|
||||
declared_raw_bytes = summary.get("raw_bytes")
|
||||
if declared_raw_bytes is not None and (
|
||||
not isinstance(declared_raw_bytes, int)
|
||||
or isinstance(declared_raw_bytes, bool)
|
||||
or declared_raw_bytes != validated.raw_committed_bytes
|
||||
):
|
||||
return None
|
||||
raw_hash = _declared_raw_hash(summary)
|
||||
if raw_hash is not None and _sha256_stable(raw_path) != raw_hash:
|
||||
return None
|
||||
metadata_hash = _declared_metadata_hash(summary)
|
||||
metadata_path = capture_root / "mqtt.metadata.jsonl"
|
||||
if metadata_hash is not None and _sha256_stable(metadata_path) != metadata_hash:
|
||||
return None
|
||||
return validated
|
||||
|
||||
|
||||
def _recover_interrupted_capture(
|
||||
capture_root: Path,
|
||||
session_root: Path,
|
||||
raw_path: Path,
|
||||
) -> _RecoveredCapture | None:
|
||||
return _scan_capture_prefix(
|
||||
capture_root,
|
||||
session_root,
|
||||
raw_path,
|
||||
tolerate_incomplete_metadata_tail=True,
|
||||
tolerate_raw_crash_tail=True,
|
||||
)
|
||||
|
||||
|
||||
def _scan_capture_prefix(
|
||||
capture_root: Path,
|
||||
session_root: Path,
|
||||
raw_path: Path,
|
||||
*,
|
||||
tolerate_incomplete_metadata_tail: bool,
|
||||
tolerate_raw_crash_tail: bool,
|
||||
) -> _RecoveredCapture | None:
|
||||
metadata_path = capture_root / "mqtt.metadata.jsonl"
|
||||
if not _confined_file(raw_path, session_root) or not _confined_file(
|
||||
metadata_path, session_root
|
||||
):
|
||||
return None
|
||||
try:
|
||||
raw_size = raw_path.stat().st_size
|
||||
with raw_path.open("rb") as raw_stream, metadata_path.open("rb") as metadata_stream:
|
||||
if raw_stream.read(len(RAW_MAGIC)) != RAW_MAGIC:
|
||||
return None
|
||||
consumed_metadata_bytes = 0
|
||||
message_count = 0
|
||||
topic_counts: dict[str, int] = {}
|
||||
first_epoch_ns: int | None = None
|
||||
last_epoch_ns: int | None = None
|
||||
first_monotonic_ns: int | None = None
|
||||
last_monotonic_ns: int | None = None
|
||||
first_timestamp: str | None = None
|
||||
last_timestamp: str | None = None
|
||||
metadata_committed_bytes = 0
|
||||
while (
|
||||
consumed_metadata_bytes < MAX_RECOVERY_METADATA_BYTES
|
||||
and message_count < MAX_RECOVERY_MESSAGES
|
||||
):
|
||||
remaining = MAX_RECOVERY_METADATA_BYTES - consumed_metadata_bytes
|
||||
read_limit = min(MAX_RECOVERY_METADATA_LINE_BYTES + 1, remaining + 1)
|
||||
line = metadata_stream.readline(read_limit)
|
||||
if not line:
|
||||
break
|
||||
consumed_metadata_bytes += len(line)
|
||||
if (
|
||||
len(line) > MAX_RECOVERY_METADATA_LINE_BYTES
|
||||
or consumed_metadata_bytes > MAX_RECOVERY_METADATA_BYTES
|
||||
):
|
||||
return None
|
||||
if not line.endswith(b"\n"):
|
||||
if tolerate_incomplete_metadata_tail:
|
||||
break
|
||||
return None
|
||||
record = _metadata_record(line)
|
||||
if record is None:
|
||||
return None
|
||||
metadata_committed_bytes = consumed_metadata_bytes
|
||||
if record.get("record_type") != "message":
|
||||
continue
|
||||
expected_sequence = message_count + 1
|
||||
validated = _validate_recovery_frame(
|
||||
raw_stream,
|
||||
raw_size=raw_size,
|
||||
record=record,
|
||||
expected_sequence=expected_sequence,
|
||||
)
|
||||
if validated is None:
|
||||
return None
|
||||
topic, epoch_ns, monotonic_ns, timestamp = validated
|
||||
if last_monotonic_ns is not None and monotonic_ns < last_monotonic_ns:
|
||||
return None
|
||||
message_count = expected_sequence
|
||||
topic_counts[topic] = topic_counts.get(topic, 0) + 1
|
||||
if first_epoch_ns is None:
|
||||
first_epoch_ns = epoch_ns
|
||||
first_monotonic_ns = monotonic_ns
|
||||
first_timestamp = timestamp
|
||||
last_epoch_ns = epoch_ns
|
||||
last_monotonic_ns = monotonic_ns
|
||||
last_timestamp = timestamp
|
||||
if metadata_stream.read(1):
|
||||
return None
|
||||
raw_committed_bytes = raw_stream.tell()
|
||||
if raw_committed_bytes != raw_size and not (
|
||||
tolerate_raw_crash_tail
|
||||
and _tolerable_raw_crash_tail(raw_stream, raw_size=raw_size)
|
||||
):
|
||||
return None
|
||||
except OSError:
|
||||
return None
|
||||
if (
|
||||
message_count == 0
|
||||
or first_epoch_ns is None
|
||||
or last_epoch_ns is None
|
||||
or first_timestamp is None
|
||||
or last_timestamp is None
|
||||
or first_monotonic_ns is None
|
||||
or last_monotonic_ns is None
|
||||
):
|
||||
return None
|
||||
return _RecoveredCapture(
|
||||
message_count=message_count,
|
||||
topic_counts=topic_counts,
|
||||
started_at_utc=first_timestamp,
|
||||
completed_at_utc=last_timestamp,
|
||||
duration_seconds=(last_monotonic_ns - first_monotonic_ns) / 1_000_000_000,
|
||||
raw_committed_bytes=raw_committed_bytes,
|
||||
metadata_committed_bytes=metadata_committed_bytes,
|
||||
)
|
||||
|
||||
|
||||
def _metadata_record(line: bytes) -> dict[str, Any] | None:
|
||||
try:
|
||||
value = json.loads(line.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
return None
|
||||
return value if isinstance(value, dict) else None
|
||||
|
||||
|
||||
def _tolerable_raw_crash_tail(raw_stream: IO[bytes], *, raw_size: int) -> bool:
|
||||
"""Accept at most one raw frame which was not committed by metadata.
|
||||
|
||||
The capture writer flushes raw before metadata and does not begin the next
|
||||
message until the current metadata record is written. Consequently a
|
||||
process crash can leave only a prefix (or all) of one additional frame.
|
||||
Anything beyond that boundary is corruption, not a recoverable tail.
|
||||
"""
|
||||
|
||||
tail_offset = raw_stream.tell()
|
||||
tail_bytes = raw_size - tail_offset
|
||||
if tail_bytes <= 0:
|
||||
return True
|
||||
if tail_bytes > GROUP_COMMIT_MAX_BYTES + MAX_CONFIGURABLE_MESSAGE_BYTES:
|
||||
return False
|
||||
frames = 0
|
||||
while raw_stream.tell() < raw_size:
|
||||
remaining = raw_size - raw_stream.tell()
|
||||
header = raw_stream.read(min(remaining, FRAME_HEADER.size))
|
||||
if len(header) < FRAME_HEADER.size:
|
||||
return frames < GROUP_COMMIT_MAX_MESSAGES
|
||||
topic_length, payload_length = FRAME_HEADER.unpack(header)
|
||||
if not 1 <= topic_length <= MAX_TOPIC_BYTES:
|
||||
return False
|
||||
if payload_length > MAX_CONFIGURABLE_MESSAGE_BYTES:
|
||||
return False
|
||||
frame_bytes = FRAME_HEADER.size + topic_length + payload_length
|
||||
frame_remaining = raw_size - (raw_stream.tell() - FRAME_HEADER.size)
|
||||
available_topic_bytes = min(
|
||||
topic_length,
|
||||
max(0, frame_remaining - FRAME_HEADER.size),
|
||||
)
|
||||
topic_prefix = raw_stream.read(available_topic_bytes)
|
||||
if available_topic_bytes < topic_length:
|
||||
return frames < GROUP_COMMIT_MAX_MESSAGES
|
||||
try:
|
||||
topic = topic_prefix.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return False
|
||||
if not topic:
|
||||
return False
|
||||
payload_available = frame_remaining - FRAME_HEADER.size - topic_length
|
||||
if payload_available < payload_length:
|
||||
return frames < GROUP_COMMIT_MAX_MESSAGES
|
||||
raw_stream.seek(payload_length, 1)
|
||||
frames += 1
|
||||
if frames > GROUP_COMMIT_MAX_MESSAGES or frame_remaining < frame_bytes:
|
||||
return False
|
||||
return frames <= GROUP_COMMIT_MAX_MESSAGES
|
||||
|
||||
|
||||
def _validate_recovery_frame(
|
||||
raw_stream: IO[bytes],
|
||||
*,
|
||||
raw_size: int,
|
||||
record: dict[str, Any],
|
||||
expected_sequence: int,
|
||||
) -> tuple[str, int, int, str] | None:
|
||||
sequence = record.get("sequence")
|
||||
topic = record.get("topic")
|
||||
payload_bytes = record.get("payload_bytes")
|
||||
frame_offset = record.get("raw_frame_offset")
|
||||
payload_offset = record.get("raw_payload_offset")
|
||||
frame_bytes = record.get("raw_frame_bytes")
|
||||
epoch_ns = record.get("received_at_epoch_ns")
|
||||
monotonic_ns = record.get("received_monotonic_ns")
|
||||
if (
|
||||
sequence != expected_sequence
|
||||
or not isinstance(topic, str)
|
||||
or not topic
|
||||
or not isinstance(payload_bytes, int)
|
||||
or isinstance(payload_bytes, bool)
|
||||
or not 0 <= payload_bytes <= MAX_CONFIGURABLE_MESSAGE_BYTES
|
||||
or not isinstance(frame_offset, int)
|
||||
or isinstance(frame_offset, bool)
|
||||
or frame_offset != raw_stream.tell()
|
||||
or not isinstance(payload_offset, int)
|
||||
or isinstance(payload_offset, bool)
|
||||
or not isinstance(frame_bytes, int)
|
||||
or isinstance(frame_bytes, bool)
|
||||
or not isinstance(epoch_ns, int)
|
||||
or isinstance(epoch_ns, bool)
|
||||
or epoch_ns < 0
|
||||
or not isinstance(monotonic_ns, int)
|
||||
or isinstance(monotonic_ns, bool)
|
||||
or monotonic_ns < 0
|
||||
):
|
||||
return None
|
||||
header = raw_stream.read(FRAME_HEADER.size)
|
||||
if len(header) != FRAME_HEADER.size:
|
||||
return None
|
||||
topic_length, raw_payload_bytes = FRAME_HEADER.unpack(header)
|
||||
if (
|
||||
not 1 <= topic_length <= MAX_TOPIC_BYTES
|
||||
or raw_payload_bytes != payload_bytes
|
||||
or payload_offset != frame_offset + FRAME_HEADER.size + topic_length
|
||||
or frame_bytes != FRAME_HEADER.size + topic_length + payload_bytes
|
||||
or frame_offset + frame_bytes > raw_size
|
||||
):
|
||||
return None
|
||||
raw_topic = raw_stream.read(topic_length)
|
||||
try:
|
||||
decoded_topic = raw_topic.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
if decoded_topic != topic:
|
||||
return None
|
||||
raw_stream.seek(payload_bytes, 1)
|
||||
timestamp = _safe_timestamp(record.get("received_at_utc"))
|
||||
if timestamp is None:
|
||||
try:
|
||||
timestamp = datetime.fromtimestamp(epoch_ns / 1_000_000_000, tz=UTC).isoformat()
|
||||
except (OSError, OverflowError, ValueError):
|
||||
return None
|
||||
return topic, epoch_ns, monotonic_ns, timestamp
|
||||
|
||||
|
||||
def _read_json_object(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
if not path.is_file() or path.stat().st_size > MAX_LEGACY_JSON_BYTES:
|
||||
return {}
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
||||
return {}
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _confined_file(path: Path, session_root: Path) -> bool:
|
||||
try:
|
||||
resolved = path.resolve(strict=True)
|
||||
except OSError:
|
||||
return False
|
||||
return resolved.is_file() and resolved.is_relative_to(session_root)
|
||||
|
||||
|
||||
def _non_negative_int(value: object) -> int:
|
||||
return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else 0
|
||||
|
||||
|
||||
def _normalized_topic_counts(value: object) -> dict[str, int]:
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
result: dict[str, int] = {}
|
||||
for topic, raw_count in value.items():
|
||||
count = _non_negative_int(raw_count)
|
||||
if isinstance(topic, str) and count > 0:
|
||||
result[topic] = count
|
||||
return result
|
||||
|
||||
|
||||
def _duration(value: object) -> float | None:
|
||||
if not isinstance(value, (int, float)) or isinstance(value, bool):
|
||||
return None
|
||||
duration = float(value)
|
||||
return duration if math.isfinite(duration) and duration >= 0 else None
|
||||
|
||||
|
||||
def _safe_timestamp(value: object) -> str | None:
|
||||
if not isinstance(value, str) or len(value) > 64:
|
||||
return None
|
||||
candidate = value.strip()
|
||||
try:
|
||||
parsed = datetime.fromisoformat(candidate.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
return None
|
||||
return candidate
|
||||
|
||||
|
||||
def _timestamp_from_session_name(session_id: str) -> str | None:
|
||||
prefix = session_id.split("_", 1)[0]
|
||||
try:
|
||||
parsed = datetime.strptime(prefix, "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC)
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed.isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _declared_raw_hash(summary: dict[str, Any]) -> str | None:
|
||||
hashes = summary.get("artifact_hashes")
|
||||
if not isinstance(hashes, dict):
|
||||
return None
|
||||
value = hashes.get("raw_sha256")
|
||||
return value if isinstance(value, str) and SHA256_PATTERN.fullmatch(value) else None
|
||||
|
||||
|
||||
def _declared_metadata_hash(summary: dict[str, Any]) -> str | None:
|
||||
hashes = summary.get("artifact_hashes")
|
||||
if not isinstance(hashes, dict):
|
||||
return None
|
||||
value = hashes.get("metadata_jsonl_sha256")
|
||||
return value if isinstance(value, str) and SHA256_PATTERN.fullmatch(value) else None
|
||||
|
||||
|
||||
def _sha256_stable(path: Path) -> str:
|
||||
try:
|
||||
current = path.lstat()
|
||||
except OSError:
|
||||
return ""
|
||||
if not stat.S_ISREG(current.st_mode):
|
||||
return ""
|
||||
return _sha256_cached(str(path), _stat_identity(current))
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
def _sha256_cached(
|
||||
path_text: str,
|
||||
expected_identity: tuple[int, int, int, int, int],
|
||||
) -> str:
|
||||
path = Path(path_text)
|
||||
flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
|
||||
try:
|
||||
descriptor = os.open(path, flags)
|
||||
except OSError:
|
||||
return ""
|
||||
try:
|
||||
before = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(before.st_mode) or _stat_identity(before) != expected_identity:
|
||||
return ""
|
||||
digest = hashlib.sha256()
|
||||
while chunk := os.read(descriptor, 1024 * 1024):
|
||||
digest.update(chunk)
|
||||
after = os.fstat(descriptor)
|
||||
try:
|
||||
current = os.lstat(path)
|
||||
except OSError:
|
||||
return ""
|
||||
if _stat_identity(before) != _stat_identity(after) or (
|
||||
current.st_dev,
|
||||
current.st_ino,
|
||||
) != (before.st_dev, before.st_ino):
|
||||
return ""
|
||||
return digest.hexdigest()
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _stat_identity(value: os.stat_result) -> tuple[int, int, int, int, int]:
|
||||
return (value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns, value.st_ctime_ns)
|
||||
|
||||
|
||||
def _active_session_root(root: Path) -> Path | None:
|
||||
marker = root / ".current_session"
|
||||
try:
|
||||
marker_stat = marker.lstat()
|
||||
if not stat.S_ISREG(marker_stat.st_mode) or marker_stat.st_size > 4096:
|
||||
return None
|
||||
value = marker.read_text(encoding="utf-8").strip()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return None
|
||||
if not value:
|
||||
return None
|
||||
raw = Path(value).expanduser()
|
||||
if raw.is_absolute():
|
||||
candidate = raw
|
||||
elif raw.parts and raw.parts[0] == root.name:
|
||||
candidate = root.parent / raw
|
||||
else:
|
||||
candidate = root / raw
|
||||
try:
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError:
|
||||
return None
|
||||
return resolved if resolved.is_dir() and resolved.is_relative_to(root) else None
|
||||
|
||||
|
||||
def _modalities(topic_counts: Mapping[str, int]) -> tuple[SessionModality, ...]:
|
||||
topics = {
|
||||
topic
|
||||
for topic, count in topic_counts.items()
|
||||
if isinstance(topic, str) and _non_negative_int(count) > 0
|
||||
}
|
||||
result: list[SessionModality] = []
|
||||
if any(topic == "RealtimePointcloud" or topic.endswith("/lio_pcl") for topic in topics):
|
||||
result.append("point-cloud")
|
||||
if any(topic == "RealtimePath" or topic.endswith("/lio_pose") for topic in topics):
|
||||
result.append("trajectory")
|
||||
return tuple(result)
|
||||
|
||||
|
||||
def _status(replayable: bool, error: object, stop_reason: object) -> SessionStatus:
|
||||
if error is None and stop_reason in {
|
||||
"duration_elapsed",
|
||||
"external_stop",
|
||||
"keyboard_interrupt",
|
||||
}:
|
||||
return "ready" if replayable else "failed"
|
||||
return "interrupted" if replayable else "failed"
|
||||
|
||||
|
||||
def _discover_media_sources(session_root: Path) -> tuple[LegacyMediaSourceCandidate, ...]:
|
||||
media_root = session_root / "media"
|
||||
try:
|
||||
resolved_media_root = media_root.resolve(strict=True)
|
||||
except OSError:
|
||||
return ()
|
||||
if not resolved_media_root.is_dir() or not resolved_media_root.is_relative_to(session_root):
|
||||
return ()
|
||||
candidates: list[LegacyMediaSourceCandidate] = []
|
||||
for source_root in sorted(resolved_media_root.iterdir()):
|
||||
if not source_root.is_dir() or not MEDIA_SOURCE_PATTERN.fullmatch(source_root.name):
|
||||
continue
|
||||
try:
|
||||
resolved_source_root = source_root.resolve(strict=True)
|
||||
except OSError:
|
||||
continue
|
||||
if not resolved_source_root.is_relative_to(resolved_media_root):
|
||||
continue
|
||||
epochs = [
|
||||
epoch
|
||||
for epoch in sorted(resolved_source_root.iterdir())
|
||||
if epoch.is_dir()
|
||||
and MEDIA_EPOCH_PATTERN.fullmatch(epoch.name)
|
||||
and _validated_media_epoch(epoch, resolved_source_root.name)
|
||||
]
|
||||
if not epochs:
|
||||
continue
|
||||
byte_length = sum(_media_epoch_bytes(epoch) for epoch in epochs)
|
||||
artifact_suffix = hashlib.sha256(resolved_source_root.name.encode()).hexdigest()[:16]
|
||||
candidates.append(
|
||||
LegacyMediaSourceCandidate(
|
||||
source_id=resolved_source_root.name,
|
||||
artifact_id=f"recorded-video-{artifact_suffix}",
|
||||
locator=resolved_source_root,
|
||||
byte_length=byte_length,
|
||||
epoch_count=len(epochs),
|
||||
)
|
||||
)
|
||||
return tuple(candidates)
|
||||
|
||||
|
||||
def _validated_media_epoch(epoch: Path, expected_source_id: str) -> bool:
|
||||
try:
|
||||
resolved_epoch = epoch.resolve(strict=True)
|
||||
init_path = (epoch / "init.mp4").resolve(strict=True)
|
||||
segments_root = (epoch / "segments").resolve(strict=True)
|
||||
index_path = (epoch / "index.jsonl").resolve(strict=True)
|
||||
summary_path = (epoch / "summary.json").resolve(strict=True)
|
||||
except OSError:
|
||||
return False
|
||||
if not all(
|
||||
path.is_relative_to(resolved_epoch)
|
||||
for path in (init_path, segments_root, index_path, summary_path)
|
||||
):
|
||||
return False
|
||||
if (
|
||||
not init_path.is_file()
|
||||
or init_path.stat().st_size <= 0
|
||||
or not segments_root.is_dir()
|
||||
or not index_path.is_file()
|
||||
or not 0 < index_path.stat().st_size <= MAX_MEDIA_INDEX_BYTES
|
||||
or not summary_path.is_file()
|
||||
):
|
||||
return False
|
||||
summary = _read_json_object(summary_path)
|
||||
if summary.get("schema_version") not in {1, "missioncore.camera-recording/v1"}:
|
||||
return False
|
||||
if summary.get("source_id") != expected_source_id:
|
||||
return False
|
||||
segment_count = _non_negative_int(summary.get("segment_count"))
|
||||
if segment_count < 1:
|
||||
return False
|
||||
segments = [
|
||||
path.resolve()
|
||||
for path in sorted(segments_root.iterdir())
|
||||
if path.is_file() and MEDIA_SEGMENT_PATTERN.fullmatch(path.name)
|
||||
]
|
||||
if len(segments) != segment_count or any(
|
||||
not path.is_relative_to(segments_root) or path.stat().st_size <= 0 for path in segments
|
||||
):
|
||||
return False
|
||||
try:
|
||||
index_records = [
|
||||
json.loads(line)
|
||||
for line in index_path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
||||
return False
|
||||
if len(index_records) != segment_count or not all(
|
||||
isinstance(record, dict)
|
||||
and _non_negative_int(record.get("sequence")) > 0
|
||||
for record in index_records
|
||||
):
|
||||
return False
|
||||
sequences = [int(record["sequence"]) for record in index_records]
|
||||
return len(sequences) == len(set(sequences))
|
||||
|
||||
|
||||
def _media_epoch_bytes(epoch: Path) -> int:
|
||||
try:
|
||||
resolved_epoch = epoch.resolve(strict=True)
|
||||
except OSError:
|
||||
return 0
|
||||
total = 0
|
||||
for path in resolved_epoch.rglob("*"):
|
||||
try:
|
||||
resolved = path.resolve(strict=True)
|
||||
except OSError:
|
||||
continue
|
||||
if resolved.is_file() and resolved.is_relative_to(resolved_epoch):
|
||||
total += resolved.stat().st_size
|
||||
return total
|
||||
@@ -14,8 +14,6 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, TypeGuard
|
||||
|
||||
from k1link.viewer.replay import MAX_METADATA_LINE_CHARS
|
||||
|
||||
from .models import RecordedMediaArtifact, ReplayCommand, SessionIntegrityError
|
||||
|
||||
CAMERA_ARCHIVE_SCHEMA = "missioncore.camera-recording/v1"
|
||||
@@ -139,7 +137,7 @@ class RecordedMediaInspector:
|
||||
cached = self._cache.get(key)
|
||||
if cached is not None:
|
||||
identity = _prepared_source_identity(
|
||||
replay.source_path,
|
||||
replay,
|
||||
epoch_paths,
|
||||
cached.manifest.epochs,
|
||||
)
|
||||
@@ -151,15 +149,14 @@ class RecordedMediaInspector:
|
||||
with self._lock:
|
||||
self._cache[key] = _CachedManifest(identity=identity, manifest=manifest)
|
||||
return manifest
|
||||
origin_epoch_ns, origin_monotonic_ns = _raw_timeline_origin(replay.source_path)
|
||||
manifest = _read_manifest(
|
||||
artifact,
|
||||
epoch_paths,
|
||||
origin_epoch_ns=origin_epoch_ns,
|
||||
origin_monotonic_ns=origin_monotonic_ns,
|
||||
origin_epoch_ns=replay.timeline_origin_epoch_ns,
|
||||
origin_monotonic_ns=replay.timeline_origin_monotonic_ns,
|
||||
)
|
||||
identity = _prepared_source_identity(
|
||||
replay.source_path,
|
||||
replay,
|
||||
epoch_paths,
|
||||
manifest.epochs,
|
||||
)
|
||||
@@ -186,7 +183,7 @@ class RecordedMediaInspector:
|
||||
document = _decode_prepared_sidecar(payload)
|
||||
manifest = _manifest_from_sidecar(document, artifact, epoch_paths)
|
||||
identity = _prepared_source_identity(
|
||||
replay.source_path,
|
||||
replay,
|
||||
epoch_paths,
|
||||
manifest.epochs,
|
||||
)
|
||||
@@ -631,15 +628,15 @@ def _epoch_paths(source_path: Path) -> tuple[Path, ...]:
|
||||
|
||||
|
||||
def _prepared_source_identity(
|
||||
raw_path: Path,
|
||||
replay: ReplayCommand,
|
||||
epoch_paths: tuple[Path, ...],
|
||||
epochs: tuple[RecordedMediaEpoch, ...],
|
||||
) -> tuple[tuple[int, int, int, int], ...]:
|
||||
if len(epoch_paths) != len(epochs):
|
||||
raise SessionIntegrityError("recorded media epoch identity is inconsistent")
|
||||
identity: list[tuple[int, int, int, int]] = []
|
||||
for source_file in (raw_path, raw_path.with_name("mqtt.metadata.jsonl")):
|
||||
metadata = _confined_file_stat(source_file, raw_path.parent)
|
||||
for artifact in replay.artifacts:
|
||||
metadata = _session_artifact_stat(artifact.path, replay.session_root)
|
||||
identity.append(
|
||||
(metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mtime_ns)
|
||||
)
|
||||
@@ -671,32 +668,15 @@ def _prepared_source_identity(
|
||||
return tuple(identity)
|
||||
|
||||
|
||||
def _raw_timeline_origin(raw_path: Path) -> tuple[int, int]:
|
||||
metadata_path = raw_path.with_name("mqtt.metadata.jsonl")
|
||||
def _session_artifact_stat(path: Path, session_root: Path) -> os.stat_result:
|
||||
try:
|
||||
line = _read_first_confined_line(
|
||||
metadata_path,
|
||||
raw_path.parent,
|
||||
MAX_METADATA_LINE_CHARS,
|
||||
).decode("utf-8")
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
raise SessionIntegrityError("native capture timing metadata is unavailable") from exc
|
||||
if not line or len(line) > MAX_METADATA_LINE_CHARS or not line.endswith(("\n", "\r")):
|
||||
raise SessionIntegrityError("native capture timing origin is incomplete")
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise SessionIntegrityError("native capture timing origin is invalid") from exc
|
||||
epoch_ns = record.get("received_at_epoch_ns") if isinstance(record, dict) else None
|
||||
monotonic_ns = record.get("received_monotonic_ns") if isinstance(record, dict) else None
|
||||
if (
|
||||
record.get("record_type") != "message"
|
||||
or record.get("sequence") != 1
|
||||
or not _non_negative_int(epoch_ns)
|
||||
or not _non_negative_int(monotonic_ns)
|
||||
):
|
||||
raise SessionIntegrityError("native capture timing origin is invalid")
|
||||
return int(epoch_ns), int(monotonic_ns)
|
||||
session = session_root.resolve(strict=True)
|
||||
parent = path.parent.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise SessionIntegrityError("recording source artifact is missing") from exc
|
||||
if not parent.is_relative_to(session):
|
||||
raise SessionIntegrityError("recording source artifact escapes its session")
|
||||
return _confined_file_stat(path, parent)
|
||||
|
||||
|
||||
def _read_manifest(
|
||||
|
||||
@@ -160,20 +160,44 @@ class WorkspaceLayout:
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReplayArtifact:
|
||||
"""One confined input artifact selected for plugin-owned preparation."""
|
||||
|
||||
artifact_id: str
|
||||
path: Path
|
||||
media_type: str
|
||||
file_byte_length: int
|
||||
replay_byte_length: int
|
||||
expected_sha256: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReplayCommand:
|
||||
"""Internal-only replay command. ``source_path`` never enters an API DTO."""
|
||||
"""Internal replay request containing no vendor format or channel names."""
|
||||
|
||||
session_id: str
|
||||
source_path: Path
|
||||
plugin_id: str
|
||||
allowed_root: Path
|
||||
session_root: Path
|
||||
replay_byte_length: int
|
||||
metadata_byte_length: int
|
||||
expected_source_sha256: str | None
|
||||
primary_artifact_id: str
|
||||
artifacts: tuple[ReplayArtifact, ...]
|
||||
timeline_origin_epoch_ns: int
|
||||
timeline_origin_monotonic_ns: int
|
||||
speed: float
|
||||
loop: bool
|
||||
|
||||
@property
|
||||
def primary_artifact(self) -> ReplayArtifact:
|
||||
matches = tuple(
|
||||
artifact
|
||||
for artifact in self.artifacts
|
||||
if artifact.artifact_id == self.primary_artifact_id
|
||||
)
|
||||
if len(matches) != 1:
|
||||
raise SessionIntegrityError("replay command has no unique primary artifact")
|
||||
return matches[0]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RecordedMediaArtifact:
|
||||
@@ -191,7 +215,19 @@ class RecordedMediaArtifact:
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LegacySessionCandidate:
|
||||
class ObservationArtifactCandidate:
|
||||
artifact_id: str
|
||||
kind: str
|
||||
media_type: str
|
||||
locator: Path
|
||||
byte_length: int
|
||||
replay_byte_length: int
|
||||
sha256: str | None
|
||||
integrity_status: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservationSessionCandidate:
|
||||
session_id: str
|
||||
display_name: str
|
||||
status: SessionStatus
|
||||
@@ -203,13 +239,11 @@ class LegacySessionCandidate:
|
||||
total_bytes: int
|
||||
allowed_root: Path
|
||||
session_root: Path
|
||||
raw_path: Path
|
||||
raw_byte_length: int
|
||||
replay_raw_byte_length: int
|
||||
replay_metadata_byte_length: int
|
||||
raw_sha256: str | None
|
||||
raw_integrity_status: str
|
||||
media_sources: tuple[LegacyMediaSourceCandidate, ...]
|
||||
primary_replay_artifact_id: str | None
|
||||
timeline_origin_epoch_ns: int | None
|
||||
timeline_origin_monotonic_ns: int | None
|
||||
sources: tuple[SessionSource, ...]
|
||||
artifacts: tuple[ObservationArtifactCandidate, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Host-side runtime ABI for observation-capable device plugins.
|
||||
|
||||
The contract deliberately contains no transport name, vendor topic, capture
|
||||
suffix, codec, or viewer implementation. Concrete plugins discover native
|
||||
evidence and export it into the host's canonical recorded-viewer artifact.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Protocol
|
||||
|
||||
from .models import ObservationSessionCandidate
|
||||
|
||||
RecordingProgressPulse = Callable[[], None]
|
||||
RecordingExportResult = Mapping[str, object]
|
||||
|
||||
|
||||
class PluginRecordingExportError(RuntimeError):
|
||||
"""A plugin rejected or failed to convert one native recording."""
|
||||
|
||||
|
||||
class PluginRecordingExportCancelled(PluginRecordingExportError):
|
||||
"""A plugin cooperatively stopped a recording conversion."""
|
||||
|
||||
|
||||
class RecordingExporter(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
source: Path,
|
||||
destination: Path,
|
||||
*,
|
||||
cancel_event: threading.Event | None = None,
|
||||
activity_callback: RecordingProgressPulse | None = None,
|
||||
) -> RecordingExportResult: ...
|
||||
|
||||
|
||||
ObservationArchiveDiscovery = Callable[
|
||||
[Path],
|
||||
tuple[ObservationSessionCandidate, ...],
|
||||
]
|
||||
ObservationArchiveRecovery = Callable[[Path], object]
|
||||
|
||||
|
||||
def _no_recovery(_: Path) -> object:
|
||||
return None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservationArchiveSource:
|
||||
"""One plugin-owned evidence namespace reconciled by the host catalog."""
|
||||
|
||||
plugin_id: str
|
||||
archive_id: str
|
||||
root: Path
|
||||
discover: ObservationArchiveDiscovery
|
||||
recover: ObservationArchiveRecovery = _no_recovery
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservationRuntimeContribution:
|
||||
"""Optional observation capabilities contributed by a device plugin."""
|
||||
|
||||
archives: tuple[ObservationArchiveSource, ...]
|
||||
recording_exporter: RecordingExporter
|
||||
@@ -584,12 +584,20 @@ def _source_identity(command: ReplayCommand) -> tuple[object, ...]:
|
||||
|
||||
identities: list[object] = [
|
||||
command.session_id,
|
||||
str(command.source_path),
|
||||
command.replay_byte_length,
|
||||
command.metadata_byte_length,
|
||||
command.expected_source_sha256,
|
||||
command.plugin_id,
|
||||
command.primary_artifact_id,
|
||||
]
|
||||
for path in (command.source_path, command.source_path.with_name("mqtt.metadata.jsonl")):
|
||||
for artifact in command.artifacts:
|
||||
path = artifact.path
|
||||
identities.extend(
|
||||
(
|
||||
artifact.artifact_id,
|
||||
artifact.media_type,
|
||||
artifact.file_byte_length,
|
||||
artifact.replay_byte_length,
|
||||
artifact.expected_sha256,
|
||||
)
|
||||
)
|
||||
try:
|
||||
value = os.lstat(path)
|
||||
except OSError:
|
||||
|
||||
+276
-171
@@ -16,18 +16,17 @@ from pathlib import Path
|
||||
from typing import Any, cast
|
||||
from uuid import uuid4
|
||||
|
||||
from k1link.viewer.rrd_export import (
|
||||
RrdExportCancelled,
|
||||
RrdExportError,
|
||||
export_k1mqtt_to_rrd,
|
||||
from .models import ReplayArtifact, ReplayCommand
|
||||
from .plugin_contract import (
|
||||
PluginRecordingExportCancelled,
|
||||
PluginRecordingExportError,
|
||||
RecordingExporter,
|
||||
)
|
||||
|
||||
from .models import ReplayCommand
|
||||
|
||||
# v6 adds a real session_time=0 row to the RRD itself. Older sidecars can
|
||||
# declare a zero start while their payload begins at the first decoded sensor
|
||||
# frame, so accepting them would violate the browser playback contract.
|
||||
CACHE_SCHEMA = "missioncore.derived-rerun-recording-cache/v6"
|
||||
CACHE_SCHEMA = "missioncore.derived-rerun-recording-cache/v7"
|
||||
COMPATIBLE_CACHE_SCHEMAS = frozenset({CACHE_SCHEMA})
|
||||
RERUN_RECORDING_MEDIA_TYPE = "application/vnd.rerun.rrd"
|
||||
RERUN_SESSION_TIMELINE = "session_time"
|
||||
@@ -37,7 +36,6 @@ DEFAULT_FREE_SPACE_RESERVE_BYTES = 2 * 1024 * 1024 * 1024
|
||||
|
||||
RrdExporter = Callable[..., Mapping[str, object]]
|
||||
RecordingProgressCallback = Callable[[str, float], None]
|
||||
DEFAULT_RRD_EXPORTER = cast(RrdExporter, export_k1mqtt_to_rrd)
|
||||
|
||||
|
||||
class RecordingMaterializationError(RuntimeError):
|
||||
@@ -71,29 +69,58 @@ class _ValidatedMemoryEntry:
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ValidatedSource:
|
||||
source: Path
|
||||
metadata: Path
|
||||
source_stat: os.stat_result
|
||||
metadata_stat: os.stat_result
|
||||
class _ValidatedArtifact:
|
||||
artifact_id: str
|
||||
path: Path
|
||||
media_type: str
|
||||
file_stat: os.stat_result
|
||||
replay_byte_length: int
|
||||
metadata_byte_length: int
|
||||
expected_source_sha256: str | None
|
||||
expected_sha256: str | None
|
||||
|
||||
@property
|
||||
def identity(self) -> tuple[object, ...]:
|
||||
return (
|
||||
*_stat_identity(self.source_stat),
|
||||
*_stat_identity(self.metadata_stat),
|
||||
self.artifact_id,
|
||||
self.media_type,
|
||||
*_stat_identity(self.file_stat),
|
||||
self.replay_byte_length,
|
||||
self.metadata_byte_length,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ValidatedSource:
|
||||
plugin_id: str
|
||||
primary_artifact_id: str
|
||||
artifacts: tuple[_ValidatedArtifact, ...]
|
||||
|
||||
@property
|
||||
def primary(self) -> _ValidatedArtifact:
|
||||
matches = tuple(
|
||||
artifact
|
||||
for artifact in self.artifacts
|
||||
if artifact.artifact_id == self.primary_artifact_id
|
||||
)
|
||||
if len(matches) != 1:
|
||||
raise RecordingMaterializationError("recording source has no primary artifact")
|
||||
return matches[0]
|
||||
|
||||
@property
|
||||
def identity(self) -> tuple[object, ...]:
|
||||
return (
|
||||
self.plugin_id,
|
||||
self.primary_artifact_id,
|
||||
*(item for artifact in self.artifacts for item in artifact.identity),
|
||||
)
|
||||
|
||||
@property
|
||||
def replay_byte_length(self) -> int:
|
||||
return sum(artifact.replay_byte_length for artifact in self.artifacts)
|
||||
|
||||
|
||||
class SessionRecordingMaterializer:
|
||||
"""Build and validate a per-session seekable RRD under the private data root.
|
||||
|
||||
The native ``.k1mqtt`` capture remains the source of record. Derived RRDs
|
||||
Native plugin evidence remains the source of record. Derived RRDs
|
||||
live below ``data_dir/recordings`` and are reused only when both their
|
||||
source identity and output digest still match an atomically written cache
|
||||
sidecar. Calls for one session are serialized, so concurrent browser
|
||||
@@ -104,7 +131,8 @@ class SessionRecordingMaterializer:
|
||||
self,
|
||||
data_dir: Path,
|
||||
*,
|
||||
exporter: RrdExporter = DEFAULT_RRD_EXPORTER,
|
||||
exporter: RrdExporter | None = None,
|
||||
exporters: Mapping[str, RecordingExporter] | None = None,
|
||||
cache_max_bytes: int | None = None,
|
||||
free_space_reserve_bytes: int | None = None,
|
||||
) -> None:
|
||||
@@ -119,12 +147,12 @@ class SessionRecordingMaterializer:
|
||||
self.recordings_root = recordings_root.resolve()
|
||||
if not self.recordings_root.is_relative_to(private_root):
|
||||
raise RecordingMaterializationError("recording cache escapes the private data root")
|
||||
self._exporter = exporter
|
||||
self._exporter_accepts_cancel = _callable_accepts_keyword(exporter, "cancel_event")
|
||||
self._exporter_accepts_activity = _callable_accepts_keyword(
|
||||
exporter,
|
||||
"activity_callback",
|
||||
)
|
||||
if exporter is not None and exporters is not None:
|
||||
raise ValueError("configure either one test exporter or plugin exporters")
|
||||
self._fallback_exporter = exporter
|
||||
self._exporters = dict(exporters or {})
|
||||
if len(self._exporters) != len(set(self._exporters)):
|
||||
raise ValueError("recording exporter plugin ids must be unique")
|
||||
self.cache_max_bytes = _positive_configuration(
|
||||
cache_max_bytes,
|
||||
environment_name="MISSIONCORE_RRD_CACHE_MAX_BYTES",
|
||||
@@ -152,7 +180,12 @@ class SessionRecordingMaterializer:
|
||||
def supports_cooperative_cancellation(self) -> bool:
|
||||
"""Whether the configured exporter observes a cancellation event."""
|
||||
|
||||
return self._exporter_accepts_cancel
|
||||
exporters: tuple[Callable[..., object], ...] = tuple(self._exporters.values())
|
||||
if self._fallback_exporter is not None:
|
||||
exporters = (*exporters, self._fallback_exporter)
|
||||
return bool(exporters) and all(
|
||||
_callable_accepts_keyword(exporter, "cancel_event") for exporter in exporters
|
||||
)
|
||||
|
||||
def is_recording_available(self, recording: MaterializedRecording) -> bool:
|
||||
"""Cheap no-follow check for a previously validated ready handle."""
|
||||
@@ -463,11 +496,11 @@ class SessionRecordingMaterializer:
|
||||
) from exc
|
||||
|
||||
staged_root: Path | None = None
|
||||
export_source = source.source
|
||||
export_source = source.primary.path
|
||||
try:
|
||||
if (
|
||||
source.replay_byte_length != source.source_stat.st_size
|
||||
or source.metadata_byte_length != source.metadata_stat.st_size
|
||||
if any(
|
||||
artifact.replay_byte_length != artifact.file_stat.st_size
|
||||
for artifact in source.artifacts
|
||||
):
|
||||
staged_root, export_source = _stage_replay_prefix(
|
||||
resolved_session_root,
|
||||
@@ -480,6 +513,7 @@ class SessionRecordingMaterializer:
|
||||
),
|
||||
)
|
||||
summary = self._invoke_exporter(
|
||||
source.plugin_id,
|
||||
export_source,
|
||||
candidate_path,
|
||||
cancel_event=cancel_event,
|
||||
@@ -491,12 +525,12 @@ class SessionRecordingMaterializer:
|
||||
)
|
||||
_raise_if_cancelled(cancel_event)
|
||||
_report_progress(progress_callback, "finalizing", 0.9)
|
||||
except RrdExportCancelled as exc:
|
||||
except PluginRecordingExportCancelled as exc:
|
||||
candidate_path.unlink(missing_ok=True)
|
||||
raise RecordingMaterializationCancelled(
|
||||
"recording preparation was cancelled"
|
||||
) from exc
|
||||
except RrdExportError as exc:
|
||||
except PluginRecordingExportError as exc:
|
||||
candidate_path.unlink(missing_ok=True)
|
||||
raise RecordingMaterializationError("native capture could not be exported") from exc
|
||||
except OSError as exc:
|
||||
@@ -515,13 +549,13 @@ class SessionRecordingMaterializer:
|
||||
raise RecordingMaterializationError("native capture changed during RRD export")
|
||||
_chmod_best_effort(candidate_path, 0o600)
|
||||
source_sha256 = _sha256_prefix_stable(
|
||||
source.source,
|
||||
source.source_stat,
|
||||
source.replay_byte_length,
|
||||
source.primary.path,
|
||||
source.primary.file_stat,
|
||||
source.primary.replay_byte_length,
|
||||
)
|
||||
if (
|
||||
source.expected_source_sha256 is not None
|
||||
and source_sha256 != source.expected_source_sha256
|
||||
source.primary.expected_sha256 is not None
|
||||
and source_sha256 != source.primary.expected_sha256
|
||||
):
|
||||
raise RecordingMaterializationError(
|
||||
"native capture digest no longer matches catalog"
|
||||
@@ -574,18 +608,25 @@ class SessionRecordingMaterializer:
|
||||
|
||||
def _invoke_exporter(
|
||||
self,
|
||||
plugin_id: str,
|
||||
source: Path,
|
||||
destination: Path,
|
||||
*,
|
||||
cancel_event: threading.Event | None,
|
||||
activity_callback: Callable[[], None],
|
||||
) -> Mapping[str, object]:
|
||||
selected = self._fallback_exporter or self._exporters.get(plugin_id)
|
||||
if selected is None:
|
||||
raise PluginRecordingExportError(
|
||||
f"device plugin has no recording exporter: {plugin_id}"
|
||||
)
|
||||
exporter = cast(RrdExporter, selected)
|
||||
kwargs: dict[str, object] = {}
|
||||
if self._exporter_accepts_cancel:
|
||||
if _callable_accepts_keyword(exporter, "cancel_event"):
|
||||
kwargs["cancel_event"] = cancel_event
|
||||
if self._exporter_accepts_activity:
|
||||
if _callable_accepts_keyword(exporter, "activity_callback"):
|
||||
kwargs["activity_callback"] = activity_callback
|
||||
return self._exporter(source, destination, **kwargs)
|
||||
return exporter(source, destination, **kwargs)
|
||||
|
||||
def _load_memory_cache(
|
||||
self,
|
||||
@@ -639,48 +680,53 @@ class SessionRecordingMaterializer:
|
||||
except (OSError, ValueError, json.JSONDecodeError, RecordingMaterializationError):
|
||||
return None
|
||||
|
||||
if document["source_file_byte_length"] != source.source_stat.st_size:
|
||||
if document["plugin_id"] != source.plugin_id:
|
||||
return None
|
||||
if document["source_mtime_ns"] != source.source_stat.st_mtime_ns:
|
||||
if document["primary_artifact_id"] != source.primary_artifact_id:
|
||||
return None
|
||||
if document["source_ctime_ns"] != source.source_stat.st_ctime_ns:
|
||||
return None
|
||||
if document["source_replay_byte_length"] != source.replay_byte_length:
|
||||
return None
|
||||
if document["metadata_file_byte_length"] != source.metadata_stat.st_size:
|
||||
return None
|
||||
if document["metadata_mtime_ns"] != source.metadata_stat.st_mtime_ns:
|
||||
return None
|
||||
if document["metadata_ctime_ns"] != source.metadata_stat.st_ctime_ns:
|
||||
return None
|
||||
if document["metadata_replay_byte_length"] != source.metadata_byte_length:
|
||||
cached_artifacts = document["source_artifacts"]
|
||||
if len(cached_artifacts) != len(source.artifacts):
|
||||
return None
|
||||
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
|
||||
if cached["artifact_id"] != artifact.artifact_id:
|
||||
return None
|
||||
if cached["media_type"] != artifact.media_type:
|
||||
return None
|
||||
if cached["file_byte_length"] != artifact.file_stat.st_size:
|
||||
return None
|
||||
if cached["mtime_ns"] != artifact.file_stat.st_mtime_ns:
|
||||
return None
|
||||
if cached["ctime_ns"] != artifact.file_stat.st_ctime_ns:
|
||||
return None
|
||||
if cached["replay_byte_length"] != artifact.replay_byte_length:
|
||||
return None
|
||||
if document["recording_byte_length"] != recording_stat.st_size:
|
||||
return None
|
||||
if document["recording_mtime_ns"] != recording_stat.st_mtime_ns:
|
||||
return None
|
||||
|
||||
source_sha256 = _sha256_prefix_stable(
|
||||
source.source,
|
||||
source.source_stat,
|
||||
source.replay_byte_length,
|
||||
source.primary.path,
|
||||
source.primary.file_stat,
|
||||
source.primary.replay_byte_length,
|
||||
)
|
||||
if (
|
||||
source.expected_source_sha256 is not None
|
||||
and source_sha256 != source.expected_source_sha256
|
||||
source.primary.expected_sha256 is not None
|
||||
and source_sha256 != source.primary.expected_sha256
|
||||
):
|
||||
raise RecordingMaterializationError(
|
||||
"native capture digest no longer matches catalog"
|
||||
)
|
||||
if source_sha256 != document["source_sha256"]:
|
||||
return None
|
||||
metadata_sha256 = _sha256_prefix_stable(
|
||||
source.metadata,
|
||||
source.metadata_stat,
|
||||
source.metadata_byte_length,
|
||||
)
|
||||
if metadata_sha256 != document["metadata_sha256"]:
|
||||
return None
|
||||
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
|
||||
digest = _sha256_prefix_stable(
|
||||
artifact.path,
|
||||
artifact.file_stat,
|
||||
artifact.replay_byte_length,
|
||||
)
|
||||
if digest != cached["sha256"]:
|
||||
return None
|
||||
recording_sha256 = _sha256_stable(recording_path, recording_stat)
|
||||
if recording_sha256 != document["recording_sha256"]:
|
||||
return None
|
||||
@@ -764,76 +810,103 @@ def _raise_if_cancelled(cancel_event: threading.Event | None) -> None:
|
||||
|
||||
|
||||
def _validate_source(command: ReplayCommand) -> _ValidatedSource:
|
||||
source_path = getattr(command, "source_path", None)
|
||||
plugin_id = getattr(command, "plugin_id", None)
|
||||
allowed_root = getattr(command, "allowed_root", None)
|
||||
session_root = getattr(command, "session_root", None)
|
||||
replay_byte_length = getattr(command, "replay_byte_length", None)
|
||||
metadata_byte_length = getattr(command, "metadata_byte_length", None)
|
||||
expected_source_sha256 = getattr(command, "expected_source_sha256", None)
|
||||
if not isinstance(source_path, Path):
|
||||
raise RecordingMaterializationError("replay command has no native capture")
|
||||
primary_artifact_id = getattr(command, "primary_artifact_id", None)
|
||||
artifacts = getattr(command, "artifacts", None)
|
||||
if not isinstance(plugin_id, str) or SESSION_ID_PATTERN.fullmatch(plugin_id) is None:
|
||||
raise RecordingMaterializationError("replay command has an invalid plugin id")
|
||||
if not isinstance(primary_artifact_id, str) or SESSION_ID_PATTERN.fullmatch(
|
||||
primary_artifact_id
|
||||
) is None:
|
||||
raise RecordingMaterializationError("replay command has an invalid primary artifact")
|
||||
if not isinstance(artifacts, tuple) or not artifacts:
|
||||
raise RecordingMaterializationError("replay command has no source artifacts")
|
||||
try:
|
||||
if not isinstance(allowed_root, Path) or not isinstance(session_root, Path):
|
||||
raise RecordingMaterializationError("replay command has no confinement roots")
|
||||
allowed = allowed_root.expanduser().resolve(strict=True)
|
||||
session = session_root.expanduser().resolve(strict=True)
|
||||
source = source_path.expanduser().absolute()
|
||||
source_parent = source.parent.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise RecordingMaterializationError("native capture is missing") from exc
|
||||
if (
|
||||
not allowed.is_dir()
|
||||
or not session.is_dir()
|
||||
or not session.is_relative_to(allowed)
|
||||
or not source_parent.is_relative_to(session)
|
||||
):
|
||||
raise RecordingMaterializationError("native capture escapes its allowed session root")
|
||||
if source.suffix.casefold() != ".k1mqtt":
|
||||
raise RecordingMaterializationError("native capture has an unsupported format")
|
||||
source_stat = _regular_file_stat_nofollow(source, "native capture")
|
||||
metadata = source.with_name("mqtt.metadata.jsonl")
|
||||
if not metadata.parent.resolve(strict=True).is_relative_to(session):
|
||||
raise RecordingMaterializationError("native metadata escapes its allowed session root")
|
||||
metadata_stat = _regular_file_stat_nofollow(metadata, "native metadata")
|
||||
if (
|
||||
not isinstance(replay_byte_length, int)
|
||||
or isinstance(replay_byte_length, bool)
|
||||
or not 1 <= replay_byte_length <= source_stat.st_size
|
||||
):
|
||||
raise RecordingMaterializationError("native capture replay boundary is invalid")
|
||||
if (
|
||||
not isinstance(metadata_byte_length, int)
|
||||
or isinstance(metadata_byte_length, bool)
|
||||
or not 1 <= metadata_byte_length <= metadata_stat.st_size
|
||||
):
|
||||
raise RecordingMaterializationError("native metadata replay boundary is invalid")
|
||||
if expected_source_sha256 is not None:
|
||||
if not isinstance(expected_source_sha256, str) or not _is_sha256(expected_source_sha256):
|
||||
raise RecordingMaterializationError("native capture expected digest is invalid")
|
||||
if replay_byte_length != source_stat.st_size:
|
||||
raise RecordingMaterializationError(
|
||||
"a full-capture digest cannot describe a replay prefix"
|
||||
raise RecordingMaterializationError("recording confinement root is missing") from exc
|
||||
if not allowed.is_dir() or not session.is_dir() or not session.is_relative_to(allowed):
|
||||
raise RecordingMaterializationError("recording source escapes its allowed session root")
|
||||
|
||||
validated: list[_ValidatedArtifact] = []
|
||||
seen_ids: set[str] = set()
|
||||
seen_names: set[str] = set()
|
||||
for artifact in artifacts:
|
||||
if not isinstance(artifact, ReplayArtifact):
|
||||
raise RecordingMaterializationError("replay command contains an invalid artifact")
|
||||
if (
|
||||
SESSION_ID_PATTERN.fullmatch(artifact.artifact_id) is None
|
||||
or artifact.artifact_id in seen_ids
|
||||
):
|
||||
raise RecordingMaterializationError("replay artifact id is invalid or duplicated")
|
||||
path = artifact.path.expanduser().absolute()
|
||||
try:
|
||||
parent = path.parent.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise RecordingMaterializationError("recording source artifact is missing") from exc
|
||||
if not parent.is_relative_to(session):
|
||||
raise RecordingMaterializationError("recording source artifact escapes its session")
|
||||
file_stat = _regular_file_stat_nofollow(path, "recording source artifact")
|
||||
if artifact.file_byte_length > file_stat.st_size:
|
||||
raise RecordingMaterializationError("recording artifact was truncated after cataloging")
|
||||
if (
|
||||
isinstance(artifact.replay_byte_length, bool)
|
||||
or not 1 <= artifact.replay_byte_length <= file_stat.st_size
|
||||
):
|
||||
raise RecordingMaterializationError("recording artifact replay boundary is invalid")
|
||||
if artifact.expected_sha256 is not None:
|
||||
if not _is_sha256(artifact.expected_sha256):
|
||||
raise RecordingMaterializationError("recording artifact digest is invalid")
|
||||
if artifact.replay_byte_length != file_stat.st_size:
|
||||
raise RecordingMaterializationError(
|
||||
"a full-artifact digest cannot describe a replay prefix"
|
||||
)
|
||||
if path.name in seen_names:
|
||||
raise RecordingMaterializationError("recording artifact filenames are duplicated")
|
||||
seen_ids.add(artifact.artifact_id)
|
||||
seen_names.add(path.name)
|
||||
validated.append(
|
||||
_ValidatedArtifact(
|
||||
artifact_id=artifact.artifact_id,
|
||||
path=path,
|
||||
media_type=artifact.media_type,
|
||||
file_stat=file_stat,
|
||||
replay_byte_length=artifact.replay_byte_length,
|
||||
expected_sha256=artifact.expected_sha256,
|
||||
)
|
||||
)
|
||||
if sum(artifact.artifact_id == primary_artifact_id for artifact in validated) != 1:
|
||||
raise RecordingMaterializationError("recording primary artifact is unavailable")
|
||||
return _ValidatedSource(
|
||||
source=source,
|
||||
metadata=metadata,
|
||||
source_stat=source_stat,
|
||||
metadata_stat=metadata_stat,
|
||||
replay_byte_length=replay_byte_length,
|
||||
metadata_byte_length=metadata_byte_length,
|
||||
expected_source_sha256=expected_source_sha256,
|
||||
plugin_id=plugin_id,
|
||||
primary_artifact_id=primary_artifact_id,
|
||||
artifacts=tuple(validated),
|
||||
)
|
||||
|
||||
|
||||
def _validate_source_state(source: _ValidatedSource) -> _ValidatedSource:
|
||||
return _ValidatedSource(
|
||||
source=source.source,
|
||||
metadata=source.metadata,
|
||||
source_stat=_regular_file_stat_nofollow(source.source, "native capture"),
|
||||
metadata_stat=_regular_file_stat_nofollow(source.metadata, "native metadata"),
|
||||
replay_byte_length=source.replay_byte_length,
|
||||
metadata_byte_length=source.metadata_byte_length,
|
||||
expected_source_sha256=source.expected_source_sha256,
|
||||
plugin_id=source.plugin_id,
|
||||
primary_artifact_id=source.primary_artifact_id,
|
||||
artifacts=tuple(
|
||||
_ValidatedArtifact(
|
||||
artifact_id=artifact.artifact_id,
|
||||
path=artifact.path,
|
||||
media_type=artifact.media_type,
|
||||
file_stat=_regular_file_stat_nofollow(
|
||||
artifact.path,
|
||||
"recording source artifact",
|
||||
),
|
||||
replay_byte_length=artifact.replay_byte_length,
|
||||
expected_sha256=artifact.expected_sha256,
|
||||
)
|
||||
for artifact in source.artifacts
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -880,20 +953,25 @@ def _cache_document(
|
||||
return {
|
||||
"schema_version": CACHE_SCHEMA,
|
||||
"session_id": recording.session_id,
|
||||
"source_file_byte_length": source.source_stat.st_size,
|
||||
"source_replay_byte_length": source.replay_byte_length,
|
||||
"source_mtime_ns": source.source_stat.st_mtime_ns,
|
||||
"source_ctime_ns": source.source_stat.st_ctime_ns,
|
||||
"plugin_id": source.plugin_id,
|
||||
"primary_artifact_id": source.primary_artifact_id,
|
||||
"source_sha256": recording.source_sha256,
|
||||
"metadata_file_byte_length": source.metadata_stat.st_size,
|
||||
"metadata_replay_byte_length": source.metadata_byte_length,
|
||||
"metadata_mtime_ns": source.metadata_stat.st_mtime_ns,
|
||||
"metadata_ctime_ns": source.metadata_stat.st_ctime_ns,
|
||||
"metadata_sha256": _sha256_prefix_stable(
|
||||
source.metadata,
|
||||
source.metadata_stat,
|
||||
source.metadata_byte_length,
|
||||
),
|
||||
"source_artifacts": [
|
||||
{
|
||||
"artifact_id": artifact.artifact_id,
|
||||
"media_type": artifact.media_type,
|
||||
"file_byte_length": artifact.file_stat.st_size,
|
||||
"replay_byte_length": artifact.replay_byte_length,
|
||||
"mtime_ns": artifact.file_stat.st_mtime_ns,
|
||||
"ctime_ns": artifact.file_stat.st_ctime_ns,
|
||||
"sha256": _sha256_prefix_stable(
|
||||
artifact.path,
|
||||
artifact.file_stat,
|
||||
artifact.replay_byte_length,
|
||||
),
|
||||
}
|
||||
for artifact in source.artifacts
|
||||
],
|
||||
"recording_byte_length": recording.byte_length,
|
||||
"recording_mtime_ns": recording_stat.st_mtime_ns,
|
||||
"recording_sha256": recording.sha256,
|
||||
@@ -909,16 +987,10 @@ def _validate_cache_document(value: object, session_id: str) -> dict[str, Any]:
|
||||
expected_keys = {
|
||||
"schema_version",
|
||||
"session_id",
|
||||
"source_file_byte_length",
|
||||
"source_replay_byte_length",
|
||||
"source_mtime_ns",
|
||||
"source_ctime_ns",
|
||||
"plugin_id",
|
||||
"primary_artifact_id",
|
||||
"source_sha256",
|
||||
"metadata_file_byte_length",
|
||||
"metadata_replay_byte_length",
|
||||
"metadata_mtime_ns",
|
||||
"metadata_ctime_ns",
|
||||
"metadata_sha256",
|
||||
"source_artifacts",
|
||||
"recording_byte_length",
|
||||
"recording_mtime_ns",
|
||||
"recording_sha256",
|
||||
@@ -932,15 +1004,10 @@ def _validate_cache_document(value: object, session_id: str) -> dict[str, Any]:
|
||||
raise ValueError("recording cache sidecar identity does not match")
|
||||
if value["timeline"] != RERUN_SESSION_TIMELINE:
|
||||
raise ValueError("recording cache timeline is unsupported")
|
||||
for key in ("plugin_id", "primary_artifact_id"):
|
||||
if not isinstance(value[key], str) or SESSION_ID_PATTERN.fullmatch(value[key]) is None:
|
||||
raise ValueError("recording cache contains an invalid identifier")
|
||||
for key in (
|
||||
"source_file_byte_length",
|
||||
"source_replay_byte_length",
|
||||
"source_mtime_ns",
|
||||
"source_ctime_ns",
|
||||
"metadata_file_byte_length",
|
||||
"metadata_replay_byte_length",
|
||||
"metadata_mtime_ns",
|
||||
"metadata_ctime_ns",
|
||||
"recording_byte_length",
|
||||
"recording_mtime_ns",
|
||||
"timeline_start_ns",
|
||||
@@ -948,10 +1015,51 @@ def _validate_cache_document(value: object, session_id: str) -> dict[str, Any]:
|
||||
):
|
||||
if not isinstance(value[key], int) or isinstance(value[key], bool) or value[key] < 0:
|
||||
raise ValueError("recording cache contains an invalid integer")
|
||||
for key in ("source_sha256", "metadata_sha256", "recording_sha256"):
|
||||
for key in ("source_sha256", "recording_sha256"):
|
||||
digest = value[key]
|
||||
if not isinstance(digest, str) or not _is_sha256(digest):
|
||||
raise ValueError("recording cache contains an invalid digest")
|
||||
source_artifacts = value["source_artifacts"]
|
||||
if not isinstance(source_artifacts, list) or not source_artifacts:
|
||||
raise ValueError("recording cache source artifacts are invalid")
|
||||
artifact_ids: set[str] = set()
|
||||
artifact_keys = {
|
||||
"artifact_id",
|
||||
"media_type",
|
||||
"file_byte_length",
|
||||
"replay_byte_length",
|
||||
"mtime_ns",
|
||||
"ctime_ns",
|
||||
"sha256",
|
||||
}
|
||||
for artifact in source_artifacts:
|
||||
if not isinstance(artifact, dict) or set(artifact) != artifact_keys:
|
||||
raise ValueError("recording cache source artifact is invalid")
|
||||
artifact_id = artifact["artifact_id"]
|
||||
if (
|
||||
not isinstance(artifact_id, str)
|
||||
or SESSION_ID_PATTERN.fullmatch(artifact_id) is None
|
||||
or artifact_id in artifact_ids
|
||||
):
|
||||
raise ValueError("recording cache source artifact id is invalid")
|
||||
artifact_ids.add(artifact_id)
|
||||
if not isinstance(artifact["media_type"], str) or not artifact["media_type"]:
|
||||
raise ValueError("recording cache source media type is invalid")
|
||||
for key in (
|
||||
"file_byte_length",
|
||||
"replay_byte_length",
|
||||
"mtime_ns",
|
||||
"ctime_ns",
|
||||
):
|
||||
item = artifact[key]
|
||||
if not isinstance(item, int) or isinstance(item, bool) or item < 0:
|
||||
raise ValueError("recording cache source artifact boundary is invalid")
|
||||
if not 1 <= artifact["replay_byte_length"] <= artifact["file_byte_length"]:
|
||||
raise ValueError("recording cache source replay boundary is invalid")
|
||||
if not isinstance(artifact["sha256"], str) or not _is_sha256(artifact["sha256"]):
|
||||
raise ValueError("recording cache source digest is invalid")
|
||||
if value["primary_artifact_id"] not in artifact_ids:
|
||||
raise ValueError("recording cache primary artifact is unavailable")
|
||||
if value["timeline_end_ns"] < value["timeline_start_ns"]:
|
||||
raise ValueError("recording cache timeline bounds are invalid")
|
||||
return cast(dict[str, Any], value)
|
||||
@@ -1061,26 +1169,23 @@ def _stage_replay_prefix(
|
||||
staged_root = session_cache_root / f".source.{uuid4().hex}.tmp"
|
||||
try:
|
||||
staged_root.mkdir(mode=0o700)
|
||||
staged_source = staged_root / "mqtt.raw.k1mqtt"
|
||||
staged_metadata = staged_root / "mqtt.metadata.jsonl"
|
||||
_copy_prefix_nofollow(
|
||||
source.source,
|
||||
source.source_stat,
|
||||
staged_source,
|
||||
source.replay_byte_length,
|
||||
cancel_event=cancel_event,
|
||||
activity_callback=activity_callback,
|
||||
)
|
||||
_copy_prefix_nofollow(
|
||||
source.metadata,
|
||||
source.metadata_stat,
|
||||
staged_metadata,
|
||||
source.metadata_byte_length,
|
||||
cancel_event=cancel_event,
|
||||
activity_callback=activity_callback,
|
||||
)
|
||||
staged_primary: Path | None = None
|
||||
for artifact in source.artifacts:
|
||||
staged = staged_root / artifact.path.name
|
||||
_copy_prefix_nofollow(
|
||||
artifact.path,
|
||||
artifact.file_stat,
|
||||
staged,
|
||||
artifact.replay_byte_length,
|
||||
cancel_event=cancel_event,
|
||||
activity_callback=activity_callback,
|
||||
)
|
||||
if artifact.artifact_id == source.primary_artifact_id:
|
||||
staged_primary = staged
|
||||
if staged_primary is None:
|
||||
raise RecordingMaterializationError("staged recording has no primary artifact")
|
||||
_fsync_directory(staged_root)
|
||||
return staged_root, staged_source
|
||||
return staged_root, staged_primary
|
||||
except BaseException:
|
||||
shutil.rmtree(staged_root, ignore_errors=True)
|
||||
raise
|
||||
|
||||
+185
-126
@@ -12,11 +12,12 @@ from typing import Any, cast
|
||||
|
||||
from k1link.artifacts import utc_now_iso
|
||||
|
||||
from .legacy import discover_legacy_viewer_sessions
|
||||
from .models import (
|
||||
LayoutConflictError,
|
||||
LegacySessionCandidate,
|
||||
ObservationArtifactCandidate,
|
||||
ObservationSessionCandidate,
|
||||
RecordedMediaArtifact,
|
||||
ReplayArtifact,
|
||||
ReplayCommand,
|
||||
SessionArtifact,
|
||||
SessionDetail,
|
||||
@@ -30,6 +31,7 @@ from .models import (
|
||||
SessionSummary,
|
||||
WorkspaceLayout,
|
||||
)
|
||||
from .plugin_contract import ObservationArchiveSource
|
||||
|
||||
IDENTIFIER_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
MAX_LAYOUT_BYTES = 256 * 1024
|
||||
@@ -38,6 +40,8 @@ DATABASE_NAME = "mission-core.sqlite3"
|
||||
SCHEMA_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS observation_sessions (
|
||||
session_id TEXT PRIMARY KEY,
|
||||
plugin_id TEXT NOT NULL,
|
||||
archive_id TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL CHECK (status IN ('ready', 'interrupted', 'failed')),
|
||||
started_at_utc TEXT,
|
||||
@@ -48,8 +52,9 @@ CREATE TABLE IF NOT EXISTS observation_sessions (
|
||||
origin TEXT NOT NULL,
|
||||
source_count INTEGER NOT NULL,
|
||||
total_bytes INTEGER NOT NULL,
|
||||
replay_raw_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
replay_metadata_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
primary_replay_artifact_id TEXT,
|
||||
timeline_origin_epoch_ns INTEGER,
|
||||
timeline_origin_monotonic_ns INTEGER,
|
||||
allowed_root TEXT NOT NULL,
|
||||
session_root TEXT NOT NULL,
|
||||
created_at_utc TEXT NOT NULL,
|
||||
@@ -68,6 +73,7 @@ CREATE TABLE IF NOT EXISTS observation_session_artifacts (
|
||||
sha256 TEXT,
|
||||
integrity_status TEXT NOT NULL,
|
||||
locator TEXT NOT NULL,
|
||||
replay_byte_length INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (session_id, artifact_id)
|
||||
);
|
||||
|
||||
@@ -129,20 +135,22 @@ class SessionStore:
|
||||
self._lock = threading.RLock()
|
||||
self._initialize()
|
||||
|
||||
def import_legacy_viewer_live(self, root: Path) -> tuple[str, ...]:
|
||||
allowed_root = root.expanduser().resolve()
|
||||
candidates = discover_legacy_viewer_sessions(allowed_root)
|
||||
def reconcile_archive(self, source: ObservationArchiveSource) -> tuple[str, ...]:
|
||||
"""Reconcile one plugin-owned evidence namespace into the host catalog."""
|
||||
|
||||
allowed_root = source.root.expanduser().resolve()
|
||||
candidates = source.discover(allowed_root)
|
||||
imported: list[str] = []
|
||||
for candidate in candidates:
|
||||
self._upsert_legacy(candidate)
|
||||
self._upsert_candidate(source, candidate)
|
||||
imported.append(candidate.session_id)
|
||||
with self._lock, self._connect() as connection:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
discovered = set(imported)
|
||||
indexed = connection.execute(
|
||||
"SELECT session_id FROM observation_sessions "
|
||||
"WHERE origin = 'legacy-viewer-live' AND allowed_root = ?",
|
||||
(str(allowed_root),),
|
||||
"WHERE plugin_id = ? AND archive_id = ? AND allowed_root = ?",
|
||||
(source.plugin_id, source.archive_id, str(allowed_root)),
|
||||
).fetchall()
|
||||
stale = [row["session_id"] for row in indexed if row["session_id"] not in discovered]
|
||||
connection.executemany(
|
||||
@@ -240,29 +248,57 @@ class SessionStore:
|
||||
if not 0 <= speed <= 100:
|
||||
raise ValueError("speed must be within 0..100")
|
||||
with self._connect() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT s.allowed_root, s.session_root, s.replay_raw_bytes, "
|
||||
"s.replay_metadata_bytes, a.locator, a.sha256 "
|
||||
"FROM observation_sessions AS s "
|
||||
"JOIN observation_session_artifacts AS a ON a.session_id = s.session_id "
|
||||
"WHERE s.session_id = ? AND a.artifact_id = 'raw-mqtt'",
|
||||
session = connection.execute(
|
||||
"SELECT plugin_id, allowed_root, session_root, "
|
||||
"primary_replay_artifact_id, timeline_origin_epoch_ns, "
|
||||
"timeline_origin_monotonic_ns FROM observation_sessions WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
artifact_rows = connection.execute(
|
||||
"SELECT artifact_id, media_type, byte_length, replay_byte_length, locator, sha256 "
|
||||
"FROM observation_session_artifacts WHERE session_id = ? "
|
||||
"AND replay_byte_length > 0 ORDER BY artifact_id",
|
||||
(session_id,),
|
||||
).fetchall()
|
||||
if session is None or not artifact_rows:
|
||||
raise SessionNotReplayableError("observation session replay artifact is unavailable")
|
||||
source_path = _resolve_confined_artifact(
|
||||
Path(row["allowed_root"]),
|
||||
Path(row["session_root"]),
|
||||
Path(row["locator"]),
|
||||
primary_artifact_id = session["primary_replay_artifact_id"]
|
||||
epoch_ns = session["timeline_origin_epoch_ns"]
|
||||
monotonic_ns = session["timeline_origin_monotonic_ns"]
|
||||
if (
|
||||
not isinstance(primary_artifact_id, str)
|
||||
or not isinstance(epoch_ns, int)
|
||||
or not isinstance(monotonic_ns, int)
|
||||
):
|
||||
raise SessionNotReplayableError("observation session replay contract is incomplete")
|
||||
allowed_root = Path(session["allowed_root"])
|
||||
session_root = Path(session["session_root"])
|
||||
artifacts = tuple(
|
||||
ReplayArtifact(
|
||||
artifact_id=row["artifact_id"],
|
||||
path=_resolve_confined_artifact(
|
||||
allowed_root,
|
||||
session_root,
|
||||
Path(row["locator"]),
|
||||
),
|
||||
media_type=row["media_type"],
|
||||
file_byte_length=int(row["byte_length"]),
|
||||
replay_byte_length=int(row["replay_byte_length"]),
|
||||
expected_sha256=row["sha256"],
|
||||
)
|
||||
for row in artifact_rows
|
||||
)
|
||||
if sum(artifact.artifact_id == primary_artifact_id for artifact in artifacts) != 1:
|
||||
raise SessionNotReplayableError("observation session primary artifact is unavailable")
|
||||
return ReplayCommand(
|
||||
session_id=session_id,
|
||||
source_path=source_path,
|
||||
allowed_root=Path(row["allowed_root"]),
|
||||
session_root=Path(row["session_root"]),
|
||||
replay_byte_length=int(row["replay_raw_bytes"]),
|
||||
metadata_byte_length=int(row["replay_metadata_bytes"]),
|
||||
expected_source_sha256=row["sha256"],
|
||||
plugin_id=session["plugin_id"],
|
||||
allowed_root=allowed_root,
|
||||
session_root=session_root,
|
||||
primary_artifact_id=primary_artifact_id,
|
||||
artifacts=artifacts,
|
||||
timeline_origin_epoch_ns=epoch_ns,
|
||||
timeline_origin_monotonic_ns=monotonic_ns,
|
||||
speed=float(speed),
|
||||
loop=loop,
|
||||
)
|
||||
@@ -387,58 +423,92 @@ class SessionStore:
|
||||
def _initialize(self) -> None:
|
||||
with self._connect() as connection:
|
||||
connection.executescript(SCHEMA_SQL)
|
||||
columns = {
|
||||
session_columns = {
|
||||
row["name"]
|
||||
for row in connection.execute("PRAGMA table_info(observation_sessions)")
|
||||
}
|
||||
if "replay_raw_bytes" not in columns:
|
||||
for name, declaration in (
|
||||
("plugin_id", "TEXT NOT NULL DEFAULT ''"),
|
||||
("archive_id", "TEXT NOT NULL DEFAULT ''"),
|
||||
("primary_replay_artifact_id", "TEXT"),
|
||||
("timeline_origin_epoch_ns", "INTEGER"),
|
||||
("timeline_origin_monotonic_ns", "INTEGER"),
|
||||
):
|
||||
if name in session_columns:
|
||||
continue
|
||||
connection.execute(
|
||||
"ALTER TABLE observation_sessions "
|
||||
"ADD COLUMN replay_raw_bytes INTEGER NOT NULL DEFAULT 0"
|
||||
f"ALTER TABLE observation_sessions ADD COLUMN {name} {declaration}" # noqa: S608
|
||||
)
|
||||
if "replay_metadata_bytes" not in columns:
|
||||
artifact_columns = {
|
||||
row["name"]
|
||||
for row in connection.execute(
|
||||
"PRAGMA table_info(observation_session_artifacts)"
|
||||
)
|
||||
}
|
||||
if "replay_byte_length" not in artifact_columns:
|
||||
connection.execute(
|
||||
"ALTER TABLE observation_sessions "
|
||||
"ADD COLUMN replay_metadata_bytes INTEGER NOT NULL DEFAULT 0"
|
||||
"ALTER TABLE observation_session_artifacts "
|
||||
"ADD COLUMN replay_byte_length INTEGER NOT NULL DEFAULT 0"
|
||||
)
|
||||
connection.commit()
|
||||
with _ignore_os_error():
|
||||
self.database_path.chmod(0o600)
|
||||
|
||||
def _upsert_legacy(self, candidate: LegacySessionCandidate) -> None:
|
||||
_validate_identifier(candidate.session_id, "legacy session id")
|
||||
def _upsert_candidate(
|
||||
self,
|
||||
source: ObservationArchiveSource,
|
||||
candidate: ObservationSessionCandidate,
|
||||
) -> None:
|
||||
_validate_identifier(candidate.session_id, "observation session id")
|
||||
_validate_identifier(source.plugin_id, "device plugin id")
|
||||
_validate_identifier(source.archive_id, "observation archive id")
|
||||
allowed_root = candidate.allowed_root.resolve()
|
||||
session_root = candidate.session_root.resolve()
|
||||
if not session_root.is_relative_to(allowed_root):
|
||||
raise SessionIntegrityError("legacy session root escapes its allowed root")
|
||||
sources = _legacy_sources(candidate)
|
||||
artifacts = _legacy_artifacts(candidate, session_root)
|
||||
raise SessionIntegrityError("observation session root escapes its allowed root")
|
||||
if allowed_root != source.root.expanduser().resolve():
|
||||
raise SessionIntegrityError("plugin candidate does not belong to its archive root")
|
||||
sources = candidate.sources
|
||||
artifacts = _validated_candidate_artifacts(candidate, session_root)
|
||||
_validate_candidate_replay(candidate, artifacts)
|
||||
now = utc_now_iso()
|
||||
modalities_json = json.dumps(list(candidate.modalities), separators=(",", ":"))
|
||||
|
||||
with self._lock, self._connect() as connection:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
existing = connection.execute(
|
||||
"SELECT origin, allowed_root, session_root, created_at_utc "
|
||||
"SELECT plugin_id, archive_id, allowed_root, session_root, created_at_utc "
|
||||
"FROM observation_sessions WHERE session_id = ?",
|
||||
(candidate.session_id,),
|
||||
).fetchone()
|
||||
unclaimed_pre_plugin_row = existing is not None and (
|
||||
existing["plugin_id"] == "" and existing["archive_id"] == ""
|
||||
)
|
||||
if existing is not None and (
|
||||
existing["origin"] != "legacy-viewer-live"
|
||||
or Path(existing["allowed_root"]).resolve() != allowed_root
|
||||
Path(existing["allowed_root"]).resolve() != allowed_root
|
||||
or Path(existing["session_root"]).resolve() != session_root
|
||||
or (
|
||||
not unclaimed_pre_plugin_row
|
||||
and (
|
||||
existing["plugin_id"] != source.plugin_id
|
||||
or existing["archive_id"] != source.archive_id
|
||||
)
|
||||
)
|
||||
):
|
||||
connection.rollback()
|
||||
raise SessionIntegrityError("session id is already bound to another origin")
|
||||
raise SessionIntegrityError("session id is already bound to another archive")
|
||||
created_at = existing["created_at_utc"] if existing is not None else now
|
||||
connection.execute(
|
||||
"INSERT INTO observation_sessions "
|
||||
"(session_id, display_name, status, started_at_utc, completed_at_utc, "
|
||||
"(session_id, plugin_id, archive_id, display_name, status, "
|
||||
"started_at_utc, completed_at_utc, "
|
||||
"duration_seconds, modalities_json, replayable, origin, source_count, "
|
||||
"total_bytes, replay_raw_bytes, replay_metadata_bytes, allowed_root, "
|
||||
"total_bytes, primary_replay_artifact_id, timeline_origin_epoch_ns, "
|
||||
"timeline_origin_monotonic_ns, allowed_root, "
|
||||
"session_root, created_at_utc, updated_at_utc) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
|
||||
"ON CONFLICT(session_id) DO UPDATE SET "
|
||||
"plugin_id = excluded.plugin_id, archive_id = excluded.archive_id, "
|
||||
"display_name = excluded.display_name, status = excluded.status, "
|
||||
"started_at_utc = excluded.started_at_utc, "
|
||||
"completed_at_utc = excluded.completed_at_utc, "
|
||||
@@ -446,11 +516,14 @@ class SessionStore:
|
||||
"modalities_json = excluded.modalities_json, "
|
||||
"replayable = excluded.replayable, source_count = excluded.source_count, "
|
||||
"total_bytes = excluded.total_bytes, "
|
||||
"replay_raw_bytes = excluded.replay_raw_bytes, "
|
||||
"replay_metadata_bytes = excluded.replay_metadata_bytes, "
|
||||
"primary_replay_artifact_id = excluded.primary_replay_artifact_id, "
|
||||
"timeline_origin_epoch_ns = excluded.timeline_origin_epoch_ns, "
|
||||
"timeline_origin_monotonic_ns = excluded.timeline_origin_monotonic_ns, "
|
||||
"updated_at_utc = excluded.updated_at_utc",
|
||||
(
|
||||
candidate.session_id,
|
||||
source.plugin_id,
|
||||
source.archive_id,
|
||||
candidate.display_name,
|
||||
candidate.status,
|
||||
candidate.started_at_utc,
|
||||
@@ -458,11 +531,12 @@ class SessionStore:
|
||||
candidate.duration_seconds,
|
||||
modalities_json,
|
||||
int(candidate.replayable),
|
||||
"legacy-viewer-live",
|
||||
source.archive_id,
|
||||
len(sources),
|
||||
candidate.total_bytes,
|
||||
candidate.replay_raw_byte_length,
|
||||
candidate.replay_metadata_byte_length,
|
||||
candidate.primary_replay_artifact_id,
|
||||
candidate.timeline_origin_epoch_ns,
|
||||
candidate.timeline_origin_monotonic_ns,
|
||||
str(allowed_root),
|
||||
str(session_root),
|
||||
created_at,
|
||||
@@ -480,9 +554,20 @@ class SessionStore:
|
||||
connection.executemany(
|
||||
"INSERT INTO observation_session_artifacts "
|
||||
"(session_id, artifact_id, kind, media_type, byte_length, sha256, "
|
||||
"integrity_status, locator) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
"integrity_status, locator, replay_byte_length) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
[
|
||||
(candidate.session_id, *artifact)
|
||||
(
|
||||
candidate.session_id,
|
||||
artifact.artifact_id,
|
||||
artifact.kind,
|
||||
artifact.media_type,
|
||||
artifact.byte_length,
|
||||
artifact.sha256,
|
||||
artifact.integrity_status,
|
||||
str(artifact.locator),
|
||||
artifact.replay_byte_length,
|
||||
)
|
||||
for artifact in artifacts
|
||||
],
|
||||
)
|
||||
@@ -519,82 +604,56 @@ class SessionStore:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _legacy_sources(candidate: LegacySessionCandidate) -> tuple[SessionSource, ...]:
|
||||
rows: list[SessionSource] = []
|
||||
if "point-cloud" in candidate.modalities:
|
||||
rows.append(
|
||||
SessionSource(
|
||||
source_id="sensor.lidar.primary",
|
||||
semantic_channel_id="spatial.point-cloud.recorded",
|
||||
modality="point-cloud",
|
||||
status="recorded",
|
||||
seekable=candidate.replayable,
|
||||
artifact_id="raw-mqtt",
|
||||
)
|
||||
)
|
||||
if "trajectory" in candidate.modalities:
|
||||
rows.append(
|
||||
SessionSource(
|
||||
source_id="spatial.trajectory",
|
||||
semantic_channel_id="spatial.pose.recorded",
|
||||
modality="trajectory",
|
||||
status="recorded",
|
||||
seekable=candidate.replayable,
|
||||
artifact_id="raw-mqtt",
|
||||
)
|
||||
)
|
||||
rows.extend(
|
||||
SessionSource(
|
||||
source_id=media.source_id,
|
||||
semantic_channel_id="camera.video.recorded",
|
||||
modality="video",
|
||||
status="recorded",
|
||||
seekable=True,
|
||||
artifact_id=media.artifact_id,
|
||||
)
|
||||
for media in candidate.media_sources
|
||||
)
|
||||
if len({source.source_id for source in rows}) != len(rows):
|
||||
raise SessionIntegrityError("legacy session contains duplicate source identifiers")
|
||||
return tuple(rows)
|
||||
|
||||
|
||||
def _legacy_artifacts(
|
||||
candidate: LegacySessionCandidate,
|
||||
def _validated_candidate_artifacts(
|
||||
candidate: ObservationSessionCandidate,
|
||||
session_root: Path,
|
||||
) -> tuple[tuple[str, str, str, int, str | None, str, str], ...]:
|
||||
artifacts: list[tuple[str, str, str, int, str | None, str, str]] = [
|
||||
(
|
||||
"raw-mqtt",
|
||||
"raw-transport",
|
||||
"application/x-nodedc-k1mqtt",
|
||||
candidate.raw_byte_length,
|
||||
candidate.raw_sha256,
|
||||
candidate.raw_integrity_status,
|
||||
str(candidate.raw_path),
|
||||
)
|
||||
]
|
||||
for media in candidate.media_sources:
|
||||
) -> tuple[ObservationArtifactCandidate, ...]:
|
||||
artifacts = candidate.artifacts
|
||||
artifact_ids: set[str] = set()
|
||||
for artifact in artifacts:
|
||||
_validate_identifier(artifact.artifact_id, "observation artifact id")
|
||||
if artifact.artifact_id in artifact_ids:
|
||||
raise SessionIntegrityError("observation session contains duplicate artifacts")
|
||||
artifact_ids.add(artifact.artifact_id)
|
||||
if artifact.byte_length < 0 or not 0 <= artifact.replay_byte_length <= artifact.byte_length:
|
||||
raise SessionIntegrityError("observation artifact has invalid byte boundaries")
|
||||
try:
|
||||
locator = media.locator.resolve(strict=True)
|
||||
locator = artifact.locator.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise SessionIntegrityError("legacy video artifact is missing") from exc
|
||||
if not locator.is_dir() or not locator.is_relative_to(session_root):
|
||||
raise SessionIntegrityError("legacy video artifact escapes its session root")
|
||||
artifacts.append(
|
||||
(
|
||||
media.artifact_id,
|
||||
"recorded-video",
|
||||
"video/mp4",
|
||||
media.byte_length,
|
||||
None,
|
||||
"validated-structure",
|
||||
str(locator),
|
||||
)
|
||||
)
|
||||
if len({artifact[0] for artifact in artifacts}) != len(artifacts):
|
||||
raise SessionIntegrityError("legacy session contains duplicate artifact identifiers")
|
||||
return tuple(artifacts)
|
||||
raise SessionIntegrityError("observation artifact is missing") from exc
|
||||
if not locator.is_relative_to(session_root) or not (locator.is_file() or locator.is_dir()):
|
||||
raise SessionIntegrityError("observation artifact escapes its session root")
|
||||
if artifact.replay_byte_length > 0 and not locator.is_file():
|
||||
raise SessionIntegrityError("replay input artifact must be a regular file")
|
||||
source_ids: set[str] = set()
|
||||
for source in candidate.sources:
|
||||
_validate_identifier(source.source_id, "observation source id")
|
||||
if source.source_id in source_ids:
|
||||
raise SessionIntegrityError("observation session contains duplicate sources")
|
||||
if source.artifact_id not in artifact_ids:
|
||||
raise SessionIntegrityError("observation source references an unknown artifact")
|
||||
source_ids.add(source.source_id)
|
||||
return artifacts
|
||||
|
||||
|
||||
def _validate_candidate_replay(
|
||||
candidate: ObservationSessionCandidate,
|
||||
artifacts: tuple[ObservationArtifactCandidate, ...],
|
||||
) -> None:
|
||||
primary_id = candidate.primary_replay_artifact_id
|
||||
replay_artifacts = tuple(artifact for artifact in artifacts if artifact.replay_byte_length > 0)
|
||||
if candidate.replayable:
|
||||
if (
|
||||
primary_id is None
|
||||
or sum(artifact.artifact_id == primary_id for artifact in replay_artifacts) != 1
|
||||
or candidate.timeline_origin_epoch_ns is None
|
||||
or candidate.timeline_origin_monotonic_ns is None
|
||||
):
|
||||
raise SessionIntegrityError("replayable observation contract is incomplete")
|
||||
if candidate.timeline_origin_epoch_ns < 0 or candidate.timeline_origin_monotonic_ns < 0:
|
||||
raise SessionIntegrityError("observation timeline origin is invalid")
|
||||
elif primary_id is not None or replay_artifacts:
|
||||
raise SessionIntegrityError("non-replayable observation declares replay artifacts")
|
||||
|
||||
|
||||
def _summary_from_row(row: sqlite3.Row) -> SessionSummary:
|
||||
|
||||
Reference in New Issue
Block a user