SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(plugins): isolate device integrations

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-17 19:29:32 +03:00
parent f9ffb7bd1c
commit 24a47318f2
122 changed files with 3304 additions and 1892 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/sessions/recording.py
+276 -171
View File
@@ -16,18 +16,17 @@ from pathlib import Path
from typing import Any, cast
from uuid import uuid4
from k1link.viewer.rrd_export import (
RrdExportCancelled,
RrdExportError,
export_k1mqtt_to_rrd,
from .models import ReplayArtifact, ReplayCommand
from .plugin_contract import (
PluginRecordingExportCancelled,
PluginRecordingExportError,
RecordingExporter,
)
from .models import ReplayCommand
# v6 adds a real session_time=0 row to the RRD itself. Older sidecars can
# declare a zero start while their payload begins at the first decoded sensor
# frame, so accepting them would violate the browser playback contract.
CACHE_SCHEMA = "missioncore.derived-rerun-recording-cache/v6"
CACHE_SCHEMA = "missioncore.derived-rerun-recording-cache/v7"
COMPATIBLE_CACHE_SCHEMAS = frozenset({CACHE_SCHEMA})
RERUN_RECORDING_MEDIA_TYPE = "application/vnd.rerun.rrd"
RERUN_SESSION_TIMELINE = "session_time"
@@ -37,7 +36,6 @@ DEFAULT_FREE_SPACE_RESERVE_BYTES = 2 * 1024 * 1024 * 1024
RrdExporter = Callable[..., Mapping[str, object]]
RecordingProgressCallback = Callable[[str, float], None]
DEFAULT_RRD_EXPORTER = cast(RrdExporter, export_k1mqtt_to_rrd)
class RecordingMaterializationError(RuntimeError):
@@ -71,29 +69,58 @@ class _ValidatedMemoryEntry:
@dataclass(frozen=True, slots=True)
class _ValidatedSource:
source: Path
metadata: Path
source_stat: os.stat_result
metadata_stat: os.stat_result
class _ValidatedArtifact:
artifact_id: str
path: Path
media_type: str
file_stat: os.stat_result
replay_byte_length: int
metadata_byte_length: int
expected_source_sha256: str | None
expected_sha256: str | None
@property
def identity(self) -> tuple[object, ...]:
return (
*_stat_identity(self.source_stat),
*_stat_identity(self.metadata_stat),
self.artifact_id,
self.media_type,
*_stat_identity(self.file_stat),
self.replay_byte_length,
self.metadata_byte_length,
)
@dataclass(frozen=True, slots=True)
class _ValidatedSource:
plugin_id: str
primary_artifact_id: str
artifacts: tuple[_ValidatedArtifact, ...]
@property
def primary(self) -> _ValidatedArtifact:
matches = tuple(
artifact
for artifact in self.artifacts
if artifact.artifact_id == self.primary_artifact_id
)
if len(matches) != 1:
raise RecordingMaterializationError("recording source has no primary artifact")
return matches[0]
@property
def identity(self) -> tuple[object, ...]:
return (
self.plugin_id,
self.primary_artifact_id,
*(item for artifact in self.artifacts for item in artifact.identity),
)
@property
def replay_byte_length(self) -> int:
return sum(artifact.replay_byte_length for artifact in self.artifacts)
class SessionRecordingMaterializer:
"""Build and validate a per-session seekable RRD under the private data root.
The native ``.k1mqtt`` capture remains the source of record. Derived RRDs
Native plugin evidence remains the source of record. Derived RRDs
live below ``data_dir/recordings`` and are reused only when both their
source identity and output digest still match an atomically written cache
sidecar. Calls for one session are serialized, so concurrent browser
@@ -104,7 +131,8 @@ class SessionRecordingMaterializer:
self,
data_dir: Path,
*,
exporter: RrdExporter = DEFAULT_RRD_EXPORTER,
exporter: RrdExporter | None = None,
exporters: Mapping[str, RecordingExporter] | None = None,
cache_max_bytes: int | None = None,
free_space_reserve_bytes: int | None = None,
) -> None:
@@ -119,12 +147,12 @@ class SessionRecordingMaterializer:
self.recordings_root = recordings_root.resolve()
if not self.recordings_root.is_relative_to(private_root):
raise RecordingMaterializationError("recording cache escapes the private data root")
self._exporter = exporter
self._exporter_accepts_cancel = _callable_accepts_keyword(exporter, "cancel_event")
self._exporter_accepts_activity = _callable_accepts_keyword(
exporter,
"activity_callback",
)
if exporter is not None and exporters is not None:
raise ValueError("configure either one test exporter or plugin exporters")
self._fallback_exporter = exporter
self._exporters = dict(exporters or {})
if len(self._exporters) != len(set(self._exporters)):
raise ValueError("recording exporter plugin ids must be unique")
self.cache_max_bytes = _positive_configuration(
cache_max_bytes,
environment_name="MISSIONCORE_RRD_CACHE_MAX_BYTES",
@@ -152,7 +180,12 @@ class SessionRecordingMaterializer:
def supports_cooperative_cancellation(self) -> bool:
"""Whether the configured exporter observes a cancellation event."""
return self._exporter_accepts_cancel
exporters: tuple[Callable[..., object], ...] = tuple(self._exporters.values())
if self._fallback_exporter is not None:
exporters = (*exporters, self._fallback_exporter)
return bool(exporters) and all(
_callable_accepts_keyword(exporter, "cancel_event") for exporter in exporters
)
def is_recording_available(self, recording: MaterializedRecording) -> bool:
"""Cheap no-follow check for a previously validated ready handle."""
@@ -463,11 +496,11 @@ class SessionRecordingMaterializer:
) from exc
staged_root: Path | None = None
export_source = source.source
export_source = source.primary.path
try:
if (
source.replay_byte_length != source.source_stat.st_size
or source.metadata_byte_length != source.metadata_stat.st_size
if any(
artifact.replay_byte_length != artifact.file_stat.st_size
for artifact in source.artifacts
):
staged_root, export_source = _stage_replay_prefix(
resolved_session_root,
@@ -480,6 +513,7 @@ class SessionRecordingMaterializer:
),
)
summary = self._invoke_exporter(
source.plugin_id,
export_source,
candidate_path,
cancel_event=cancel_event,
@@ -491,12 +525,12 @@ class SessionRecordingMaterializer:
)
_raise_if_cancelled(cancel_event)
_report_progress(progress_callback, "finalizing", 0.9)
except RrdExportCancelled as exc:
except PluginRecordingExportCancelled as exc:
candidate_path.unlink(missing_ok=True)
raise RecordingMaterializationCancelled(
"recording preparation was cancelled"
) from exc
except RrdExportError as exc:
except PluginRecordingExportError as exc:
candidate_path.unlink(missing_ok=True)
raise RecordingMaterializationError("native capture could not be exported") from exc
except OSError as exc:
@@ -515,13 +549,13 @@ class SessionRecordingMaterializer:
raise RecordingMaterializationError("native capture changed during RRD export")
_chmod_best_effort(candidate_path, 0o600)
source_sha256 = _sha256_prefix_stable(
source.source,
source.source_stat,
source.replay_byte_length,
source.primary.path,
source.primary.file_stat,
source.primary.replay_byte_length,
)
if (
source.expected_source_sha256 is not None
and source_sha256 != source.expected_source_sha256
source.primary.expected_sha256 is not None
and source_sha256 != source.primary.expected_sha256
):
raise RecordingMaterializationError(
"native capture digest no longer matches catalog"
@@ -574,18 +608,25 @@ class SessionRecordingMaterializer:
def _invoke_exporter(
self,
plugin_id: str,
source: Path,
destination: Path,
*,
cancel_event: threading.Event | None,
activity_callback: Callable[[], None],
) -> Mapping[str, object]:
selected = self._fallback_exporter or self._exporters.get(plugin_id)
if selected is None:
raise PluginRecordingExportError(
f"device plugin has no recording exporter: {plugin_id}"
)
exporter = cast(RrdExporter, selected)
kwargs: dict[str, object] = {}
if self._exporter_accepts_cancel:
if _callable_accepts_keyword(exporter, "cancel_event"):
kwargs["cancel_event"] = cancel_event
if self._exporter_accepts_activity:
if _callable_accepts_keyword(exporter, "activity_callback"):
kwargs["activity_callback"] = activity_callback
return self._exporter(source, destination, **kwargs)
return exporter(source, destination, **kwargs)
def _load_memory_cache(
self,
@@ -639,48 +680,53 @@ class SessionRecordingMaterializer:
except (OSError, ValueError, json.JSONDecodeError, RecordingMaterializationError):
return None
if document["source_file_byte_length"] != source.source_stat.st_size:
if document["plugin_id"] != source.plugin_id:
return None
if document["source_mtime_ns"] != source.source_stat.st_mtime_ns:
if document["primary_artifact_id"] != source.primary_artifact_id:
return None
if document["source_ctime_ns"] != source.source_stat.st_ctime_ns:
return None
if document["source_replay_byte_length"] != source.replay_byte_length:
return None
if document["metadata_file_byte_length"] != source.metadata_stat.st_size:
return None
if document["metadata_mtime_ns"] != source.metadata_stat.st_mtime_ns:
return None
if document["metadata_ctime_ns"] != source.metadata_stat.st_ctime_ns:
return None
if document["metadata_replay_byte_length"] != source.metadata_byte_length:
cached_artifacts = document["source_artifacts"]
if len(cached_artifacts) != len(source.artifacts):
return None
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
if cached["artifact_id"] != artifact.artifact_id:
return None
if cached["media_type"] != artifact.media_type:
return None
if cached["file_byte_length"] != artifact.file_stat.st_size:
return None
if cached["mtime_ns"] != artifact.file_stat.st_mtime_ns:
return None
if cached["ctime_ns"] != artifact.file_stat.st_ctime_ns:
return None
if cached["replay_byte_length"] != artifact.replay_byte_length:
return None
if document["recording_byte_length"] != recording_stat.st_size:
return None
if document["recording_mtime_ns"] != recording_stat.st_mtime_ns:
return None
source_sha256 = _sha256_prefix_stable(
source.source,
source.source_stat,
source.replay_byte_length,
source.primary.path,
source.primary.file_stat,
source.primary.replay_byte_length,
)
if (
source.expected_source_sha256 is not None
and source_sha256 != source.expected_source_sha256
source.primary.expected_sha256 is not None
and source_sha256 != source.primary.expected_sha256
):
raise RecordingMaterializationError(
"native capture digest no longer matches catalog"
)
if source_sha256 != document["source_sha256"]:
return None
metadata_sha256 = _sha256_prefix_stable(
source.metadata,
source.metadata_stat,
source.metadata_byte_length,
)
if metadata_sha256 != document["metadata_sha256"]:
return None
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
digest = _sha256_prefix_stable(
artifact.path,
artifact.file_stat,
artifact.replay_byte_length,
)
if digest != cached["sha256"]:
return None
recording_sha256 = _sha256_stable(recording_path, recording_stat)
if recording_sha256 != document["recording_sha256"]:
return None
@@ -764,76 +810,103 @@ def _raise_if_cancelled(cancel_event: threading.Event | None) -> None:
def _validate_source(command: ReplayCommand) -> _ValidatedSource:
source_path = getattr(command, "source_path", None)
plugin_id = getattr(command, "plugin_id", None)
allowed_root = getattr(command, "allowed_root", None)
session_root = getattr(command, "session_root", None)
replay_byte_length = getattr(command, "replay_byte_length", None)
metadata_byte_length = getattr(command, "metadata_byte_length", None)
expected_source_sha256 = getattr(command, "expected_source_sha256", None)
if not isinstance(source_path, Path):
raise RecordingMaterializationError("replay command has no native capture")
primary_artifact_id = getattr(command, "primary_artifact_id", None)
artifacts = getattr(command, "artifacts", None)
if not isinstance(plugin_id, str) or SESSION_ID_PATTERN.fullmatch(plugin_id) is None:
raise RecordingMaterializationError("replay command has an invalid plugin id")
if not isinstance(primary_artifact_id, str) or SESSION_ID_PATTERN.fullmatch(
primary_artifact_id
) is None:
raise RecordingMaterializationError("replay command has an invalid primary artifact")
if not isinstance(artifacts, tuple) or not artifacts:
raise RecordingMaterializationError("replay command has no source artifacts")
try:
if not isinstance(allowed_root, Path) or not isinstance(session_root, Path):
raise RecordingMaterializationError("replay command has no confinement roots")
allowed = allowed_root.expanduser().resolve(strict=True)
session = session_root.expanduser().resolve(strict=True)
source = source_path.expanduser().absolute()
source_parent = source.parent.resolve(strict=True)
except OSError as exc:
raise RecordingMaterializationError("native capture is missing") from exc
if (
not allowed.is_dir()
or not session.is_dir()
or not session.is_relative_to(allowed)
or not source_parent.is_relative_to(session)
):
raise RecordingMaterializationError("native capture escapes its allowed session root")
if source.suffix.casefold() != ".k1mqtt":
raise RecordingMaterializationError("native capture has an unsupported format")
source_stat = _regular_file_stat_nofollow(source, "native capture")
metadata = source.with_name("mqtt.metadata.jsonl")
if not metadata.parent.resolve(strict=True).is_relative_to(session):
raise RecordingMaterializationError("native metadata escapes its allowed session root")
metadata_stat = _regular_file_stat_nofollow(metadata, "native metadata")
if (
not isinstance(replay_byte_length, int)
or isinstance(replay_byte_length, bool)
or not 1 <= replay_byte_length <= source_stat.st_size
):
raise RecordingMaterializationError("native capture replay boundary is invalid")
if (
not isinstance(metadata_byte_length, int)
or isinstance(metadata_byte_length, bool)
or not 1 <= metadata_byte_length <= metadata_stat.st_size
):
raise RecordingMaterializationError("native metadata replay boundary is invalid")
if expected_source_sha256 is not None:
if not isinstance(expected_source_sha256, str) or not _is_sha256(expected_source_sha256):
raise RecordingMaterializationError("native capture expected digest is invalid")
if replay_byte_length != source_stat.st_size:
raise RecordingMaterializationError(
"a full-capture digest cannot describe a replay prefix"
raise RecordingMaterializationError("recording confinement root is missing") from exc
if not allowed.is_dir() or not session.is_dir() or not session.is_relative_to(allowed):
raise RecordingMaterializationError("recording source escapes its allowed session root")
validated: list[_ValidatedArtifact] = []
seen_ids: set[str] = set()
seen_names: set[str] = set()
for artifact in artifacts:
if not isinstance(artifact, ReplayArtifact):
raise RecordingMaterializationError("replay command contains an invalid artifact")
if (
SESSION_ID_PATTERN.fullmatch(artifact.artifact_id) is None
or artifact.artifact_id in seen_ids
):
raise RecordingMaterializationError("replay artifact id is invalid or duplicated")
path = artifact.path.expanduser().absolute()
try:
parent = path.parent.resolve(strict=True)
except OSError as exc:
raise RecordingMaterializationError("recording source artifact is missing") from exc
if not parent.is_relative_to(session):
raise RecordingMaterializationError("recording source artifact escapes its session")
file_stat = _regular_file_stat_nofollow(path, "recording source artifact")
if artifact.file_byte_length > file_stat.st_size:
raise RecordingMaterializationError("recording artifact was truncated after cataloging")
if (
isinstance(artifact.replay_byte_length, bool)
or not 1 <= artifact.replay_byte_length <= file_stat.st_size
):
raise RecordingMaterializationError("recording artifact replay boundary is invalid")
if artifact.expected_sha256 is not None:
if not _is_sha256(artifact.expected_sha256):
raise RecordingMaterializationError("recording artifact digest is invalid")
if artifact.replay_byte_length != file_stat.st_size:
raise RecordingMaterializationError(
"a full-artifact digest cannot describe a replay prefix"
)
if path.name in seen_names:
raise RecordingMaterializationError("recording artifact filenames are duplicated")
seen_ids.add(artifact.artifact_id)
seen_names.add(path.name)
validated.append(
_ValidatedArtifact(
artifact_id=artifact.artifact_id,
path=path,
media_type=artifact.media_type,
file_stat=file_stat,
replay_byte_length=artifact.replay_byte_length,
expected_sha256=artifact.expected_sha256,
)
)
if sum(artifact.artifact_id == primary_artifact_id for artifact in validated) != 1:
raise RecordingMaterializationError("recording primary artifact is unavailable")
return _ValidatedSource(
source=source,
metadata=metadata,
source_stat=source_stat,
metadata_stat=metadata_stat,
replay_byte_length=replay_byte_length,
metadata_byte_length=metadata_byte_length,
expected_source_sha256=expected_source_sha256,
plugin_id=plugin_id,
primary_artifact_id=primary_artifact_id,
artifacts=tuple(validated),
)
def _validate_source_state(source: _ValidatedSource) -> _ValidatedSource:
return _ValidatedSource(
source=source.source,
metadata=source.metadata,
source_stat=_regular_file_stat_nofollow(source.source, "native capture"),
metadata_stat=_regular_file_stat_nofollow(source.metadata, "native metadata"),
replay_byte_length=source.replay_byte_length,
metadata_byte_length=source.metadata_byte_length,
expected_source_sha256=source.expected_source_sha256,
plugin_id=source.plugin_id,
primary_artifact_id=source.primary_artifact_id,
artifacts=tuple(
_ValidatedArtifact(
artifact_id=artifact.artifact_id,
path=artifact.path,
media_type=artifact.media_type,
file_stat=_regular_file_stat_nofollow(
artifact.path,
"recording source artifact",
),
replay_byte_length=artifact.replay_byte_length,
expected_sha256=artifact.expected_sha256,
)
for artifact in source.artifacts
),
)
@@ -880,20 +953,25 @@ def _cache_document(
return {
"schema_version": CACHE_SCHEMA,
"session_id": recording.session_id,
"source_file_byte_length": source.source_stat.st_size,
"source_replay_byte_length": source.replay_byte_length,
"source_mtime_ns": source.source_stat.st_mtime_ns,
"source_ctime_ns": source.source_stat.st_ctime_ns,
"plugin_id": source.plugin_id,
"primary_artifact_id": source.primary_artifact_id,
"source_sha256": recording.source_sha256,
"metadata_file_byte_length": source.metadata_stat.st_size,
"metadata_replay_byte_length": source.metadata_byte_length,
"metadata_mtime_ns": source.metadata_stat.st_mtime_ns,
"metadata_ctime_ns": source.metadata_stat.st_ctime_ns,
"metadata_sha256": _sha256_prefix_stable(
source.metadata,
source.metadata_stat,
source.metadata_byte_length,
),
"source_artifacts": [
{
"artifact_id": artifact.artifact_id,
"media_type": artifact.media_type,
"file_byte_length": artifact.file_stat.st_size,
"replay_byte_length": artifact.replay_byte_length,
"mtime_ns": artifact.file_stat.st_mtime_ns,
"ctime_ns": artifact.file_stat.st_ctime_ns,
"sha256": _sha256_prefix_stable(
artifact.path,
artifact.file_stat,
artifact.replay_byte_length,
),
}
for artifact in source.artifacts
],
"recording_byte_length": recording.byte_length,
"recording_mtime_ns": recording_stat.st_mtime_ns,
"recording_sha256": recording.sha256,
@@ -909,16 +987,10 @@ def _validate_cache_document(value: object, session_id: str) -> dict[str, Any]:
expected_keys = {
"schema_version",
"session_id",
"source_file_byte_length",
"source_replay_byte_length",
"source_mtime_ns",
"source_ctime_ns",
"plugin_id",
"primary_artifact_id",
"source_sha256",
"metadata_file_byte_length",
"metadata_replay_byte_length",
"metadata_mtime_ns",
"metadata_ctime_ns",
"metadata_sha256",
"source_artifacts",
"recording_byte_length",
"recording_mtime_ns",
"recording_sha256",
@@ -932,15 +1004,10 @@ def _validate_cache_document(value: object, session_id: str) -> dict[str, Any]:
raise ValueError("recording cache sidecar identity does not match")
if value["timeline"] != RERUN_SESSION_TIMELINE:
raise ValueError("recording cache timeline is unsupported")
for key in ("plugin_id", "primary_artifact_id"):
if not isinstance(value[key], str) or SESSION_ID_PATTERN.fullmatch(value[key]) is None:
raise ValueError("recording cache contains an invalid identifier")
for key in (
"source_file_byte_length",
"source_replay_byte_length",
"source_mtime_ns",
"source_ctime_ns",
"metadata_file_byte_length",
"metadata_replay_byte_length",
"metadata_mtime_ns",
"metadata_ctime_ns",
"recording_byte_length",
"recording_mtime_ns",
"timeline_start_ns",
@@ -948,10 +1015,51 @@ def _validate_cache_document(value: object, session_id: str) -> dict[str, Any]:
):
if not isinstance(value[key], int) or isinstance(value[key], bool) or value[key] < 0:
raise ValueError("recording cache contains an invalid integer")
for key in ("source_sha256", "metadata_sha256", "recording_sha256"):
for key in ("source_sha256", "recording_sha256"):
digest = value[key]
if not isinstance(digest, str) or not _is_sha256(digest):
raise ValueError("recording cache contains an invalid digest")
source_artifacts = value["source_artifacts"]
if not isinstance(source_artifacts, list) or not source_artifacts:
raise ValueError("recording cache source artifacts are invalid")
artifact_ids: set[str] = set()
artifact_keys = {
"artifact_id",
"media_type",
"file_byte_length",
"replay_byte_length",
"mtime_ns",
"ctime_ns",
"sha256",
}
for artifact in source_artifacts:
if not isinstance(artifact, dict) or set(artifact) != artifact_keys:
raise ValueError("recording cache source artifact is invalid")
artifact_id = artifact["artifact_id"]
if (
not isinstance(artifact_id, str)
or SESSION_ID_PATTERN.fullmatch(artifact_id) is None
or artifact_id in artifact_ids
):
raise ValueError("recording cache source artifact id is invalid")
artifact_ids.add(artifact_id)
if not isinstance(artifact["media_type"], str) or not artifact["media_type"]:
raise ValueError("recording cache source media type is invalid")
for key in (
"file_byte_length",
"replay_byte_length",
"mtime_ns",
"ctime_ns",
):
item = artifact[key]
if not isinstance(item, int) or isinstance(item, bool) or item < 0:
raise ValueError("recording cache source artifact boundary is invalid")
if not 1 <= artifact["replay_byte_length"] <= artifact["file_byte_length"]:
raise ValueError("recording cache source replay boundary is invalid")
if not isinstance(artifact["sha256"], str) or not _is_sha256(artifact["sha256"]):
raise ValueError("recording cache source digest is invalid")
if value["primary_artifact_id"] not in artifact_ids:
raise ValueError("recording cache primary artifact is unavailable")
if value["timeline_end_ns"] < value["timeline_start_ns"]:
raise ValueError("recording cache timeline bounds are invalid")
return cast(dict[str, Any], value)
@@ -1061,26 +1169,23 @@ def _stage_replay_prefix(
staged_root = session_cache_root / f".source.{uuid4().hex}.tmp"
try:
staged_root.mkdir(mode=0o700)
staged_source = staged_root / "mqtt.raw.k1mqtt"
staged_metadata = staged_root / "mqtt.metadata.jsonl"
_copy_prefix_nofollow(
source.source,
source.source_stat,
staged_source,
source.replay_byte_length,
cancel_event=cancel_event,
activity_callback=activity_callback,
)
_copy_prefix_nofollow(
source.metadata,
source.metadata_stat,
staged_metadata,
source.metadata_byte_length,
cancel_event=cancel_event,
activity_callback=activity_callback,
)
staged_primary: Path | None = None
for artifact in source.artifacts:
staged = staged_root / artifact.path.name
_copy_prefix_nofollow(
artifact.path,
artifact.file_stat,
staged,
artifact.replay_byte_length,
cancel_event=cancel_event,
activity_callback=activity_callback,
)
if artifact.artifact_id == source.primary_artifact_id:
staged_primary = staged
if staged_primary is None:
raise RecordingMaterializationError("staged recording has no primary artifact")
_fsync_directory(staged_root)
return staged_root, staged_source
return staged_root, staged_primary
except BaseException:
shutil.rmtree(staged_root, ignore_errors=True)
raise