feat(plugins): isolate device integrations

This commit is contained in:
DCCONSTRUCTIONS
2026-07-17 19:29:32 +03:00
parent f9ffb7bd1c
commit 24a47318f2
122 changed files with 3304 additions and 1892 deletions
+1
View File
@@ -0,0 +1 @@
"""Concrete device integrations loaded only through reviewed plugin manifests."""
@@ -0,0 +1,5 @@
"""XGRIDS/LixelKity K1 compatibility plugin implementation."""
from .observation import build_xgrids_k1_observation, xgrids_k1_archive_source
__all__ = ["build_xgrids_k1_observation", "xgrids_k1_archive_source"]
@@ -1,6 +1,6 @@
"""Bounded, offline analysis of sensitive K1 evidence artifacts."""
from k1link.analyze.stream_summary import (
from k1link.device_plugins.xgrids_k1.analyze.stream_summary import (
DEFAULT_STREAM_SUMMARY_MAX_PAYLOAD_BYTES,
MAX_STREAM_SUMMARY_PAYLOAD_BYTES,
StreamSummary,
@@ -9,8 +9,8 @@ from pathlib import Path
from typing import TypedDict
from k1link.artifacts import utc_now_iso
from k1link.mqtt import DEFAULT_MAX_MESSAGE_BYTES, iter_capture_frames
from k1link.protocol import (
from k1link.device_plugins.xgrids_k1.mqtt import DEFAULT_MAX_MESSAGE_BYTES, iter_capture_frames
from k1link.device_plugins.xgrids_k1.protocol import (
DecodeLimits,
StreamDecodeError,
decode_lio_pcl,
@@ -13,7 +13,7 @@ from functools import lru_cache
from pathlib import Path
from typing import IO, Any
from k1link.mqtt.capture import (
from k1link.device_plugins.xgrids_k1.mqtt.capture import (
FRAME_HEADER,
GROUP_COMMIT_MAX_BYTES,
GROUP_COMMIT_MAX_MESSAGES,
@@ -21,10 +21,8 @@ from k1link.mqtt.capture import (
MAX_TOPIC_BYTES,
RAW_MAGIC,
)
from .models import (
from k1link.sessions.models import (
LegacyMediaSourceCandidate,
LegacySessionCandidate,
SessionModality,
SessionStatus,
)
@@ -41,6 +39,30 @@ MAX_RECOVERY_METADATA_LINE_BYTES = 64 * 1024
MAX_RECOVERY_MESSAGES = 500_000
@dataclass(frozen=True, slots=True)
class LegacySessionCandidate:
"""K1 archive discovery result retained inside the compatibility adapter."""
session_id: str
display_name: str
status: SessionStatus
started_at_utc: str | None
completed_at_utc: str | None
duration_seconds: float | None
modalities: tuple[SessionModality, ...]
replayable: bool
total_bytes: int
allowed_root: Path
session_root: Path
raw_path: Path
raw_byte_length: int
replay_raw_byte_length: int
replay_metadata_byte_length: int
raw_sha256: str | None
raw_integrity_status: str
media_sources: tuple[LegacyMediaSourceCandidate, ...]
@dataclass(frozen=True, slots=True)
class _RecoveredCapture:
message_count: int
@@ -14,7 +14,7 @@ from typing import IO, Any, Final, Literal
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
from k1link.mqtt import validate_private_ipv4
from k1link.device_plugins.xgrids_k1.mqtt import validate_private_ipv4
from k1link.web.camera_archive import (
CameraArchiveError,
CameraArchiveKind,
@@ -16,28 +16,28 @@ from rich.console import Console
from rich.table import Table
from k1link import __version__
from k1link.analyze import (
from k1link.artifacts import write_json_atomic
from k1link.device_plugins.xgrids_k1.analyze import (
DEFAULT_STREAM_SUMMARY_MAX_PAYLOAD_BYTES,
MAX_STREAM_SUMMARY_PAYLOAD_BYTES,
summarize_mqtt_streams,
)
from k1link.artifacts import write_json_atomic
from k1link.ble.gatt import dump_metadata
from k1link.ble.scanner import scan
from k1link.ble.wifi_provisioning import (
from k1link.device_plugins.xgrids_k1.ble.gatt import dump_metadata
from k1link.device_plugins.xgrids_k1.ble.scanner import scan
from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import (
PROFILE_ID,
WriteMode,
provision_wifi_once,
)
from k1link.macos_credentials import CredentialDialogError, prompt_wifi_credentials
from k1link.mqtt import (
from k1link.device_plugins.xgrids_k1.mqtt import (
DEFAULT_MAX_MESSAGE_BYTES,
MAX_CONFIGURABLE_MESSAGE_BYTES,
CaptureError,
capture_mqtt,
)
from k1link.net.snapshot import snapshot
from k1link.usb.snapshot import snapshot as usb_snapshot
from k1link.device_plugins.xgrids_k1.net.snapshot import snapshot
from k1link.device_plugins.xgrids_k1.usb.snapshot import snapshot as usb_snapshot
from k1link.macos_credentials import CredentialDialogError, prompt_wifi_credentials
app = typer.Typer(
name="k1link",
@@ -15,16 +15,31 @@ from pathlib import Path
from typing import Any, Literal, Protocol, cast
from bleak.exc import BleakError
from missioncore_plugin_sdk.v0alpha2 import RuntimeActionInvocation
from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
from k1link.artifacts import write_json_atomic
from k1link.ble.scanner import scan
from k1link.ble.wifi_provisioning import AP_FALLBACK_IPV4, provision_wifi_once
from k1link.mqtt import validate_private_ipv4
from k1link.protocol.normalizer import normalize_k1_message
from k1link.device_plugins.xgrids_k1.ble.scanner import scan
from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import (
AP_FALLBACK_IPV4,
provision_wifi_once,
)
from k1link.device_plugins.xgrids_k1.camera import (
CAMERA_EXCLUSIVE_GROUP,
CAMERA_SOURCE_LABELS,
CAMERA_SOURCE_PATHS,
CameraSourceId,
XgridsK1CameraGateway,
build_xgrids_k1_camera_router,
)
from k1link.device_plugins.xgrids_k1.mqtt import validate_private_ipv4
from k1link.device_plugins.xgrids_k1.protocol.normalizer import normalize_k1_message
from k1link.device_plugins.xgrids_k1.viewer.runtime import (
VisualizationRuntime,
new_live_session_dir,
)
from k1link.sessions import ActiveSessionLease, resolve_missioncore_evidence_dir
from k1link.viewer.rerun_bridge import RerunSceneSettings
from k1link.viewer.runtime import VisualizationRuntime, new_live_session_dir
from k1link.web.device_lifecycle import (
TERMINAL_ACQUISITION_STATES,
AcquisitionRecord,
@@ -38,14 +53,6 @@ from k1link.web.plugin_runtime import (
PluginActionNotFoundError,
PluginExecutionError,
)
from k1link.web.xgrids_k1_camera import (
CAMERA_EXCLUSIVE_GROUP,
CAMERA_SOURCE_LABELS,
CAMERA_SOURCE_PATHS,
CameraSourceId,
XgridsK1CameraGateway,
build_xgrids_k1_camera_router,
)
XGRIDS_K1_PLUGIN_ID = "nodedc.device.xgrids-lixelkity-k1"
XGRIDS_K1_MODEL_ID = "xgrids.lixelkity-k1"
@@ -1497,9 +1504,14 @@ class XgridsK1PluginFacade:
def __init__(self, service: XgridsK1ServicePort) -> None:
self.service = service
async def invoke(self, action_id: str, payload: Mapping[str, Any]) -> dict[str, Any]:
async def invoke(self, invocation: RuntimeActionInvocation) -> dict[str, Any]:
if invocation.plugin_id != self.plugin_id:
raise PluginActionNotFoundError("runtime invocation targets another plugin")
try:
return await self._invoke_validated(action_id, payload)
return await self._invoke_validated(
invocation.action_id,
invocation.parameters,
)
except (ValidationError, ValueError, PluginActionNotFoundError):
raise
except (BleakError, OSError, TimeoutError, RuntimeError) as exc:
@@ -1803,7 +1815,8 @@ def _validate_installed_compatibility_profile(repository_root: Path) -> None:
def build_xgrids_k1_plugin(repository_root: Path) -> DevicePluginRuntimeContribution:
"""Manifest entrypoint for the reviewed XGRIDS compatibility adapter."""
from k1link.web.xgrids_k1_legacy_api import build_xgrids_k1_legacy_router
from k1link.device_plugins.xgrids_k1 import build_xgrids_k1_observation
from k1link.device_plugins.xgrids_k1.legacy_api import build_xgrids_k1_legacy_router
_validate_installed_compatibility_profile(repository_root)
service = XgridsK1CompatibilityService(repository_root)
@@ -1817,5 +1830,6 @@ def build_xgrids_k1_plugin(repository_root: Path) -> DevicePluginRuntimeContribu
XGRIDS_K1_PLUGIN_ID,
),
),
observation=build_xgrids_k1_observation(repository_root),
close=service.close,
)
@@ -5,8 +5,7 @@ from typing import Any
from fastapi import APIRouter, HTTPException, WebSocket, WebSocketDisconnect
from k1link.web.plugin_runtime import PluginExecutionError
from k1link.web.xgrids_k1_facade import (
from k1link.device_plugins.xgrids_k1.facade import (
ACTION_DISCOVERY_SCAN,
ACTION_NETWORK_PROVISION,
ACTION_STATE_READ,
@@ -21,6 +20,7 @@ from k1link.web.xgrids_k1_facade import (
ViewerSettingsRequest,
XgridsK1PluginFacade,
)
from k1link.web.plugin_runtime import PluginExecutionError, invoke_device_plugin_adapter
def build_xgrids_k1_legacy_router(adapter: XgridsK1PluginFacade) -> APIRouter:
@@ -30,19 +30,27 @@ def build_xgrids_k1_legacy_router(adapter: XgridsK1PluginFacade) -> APIRouter:
@router.get("/api/state", deprecated=True)
async def get_state() -> dict[str, Any]:
return await adapter.invoke(ACTION_STATE_READ, {})
return await invoke_device_plugin_adapter(adapter, ACTION_STATE_READ, {})
@router.post("/api/ble/scan", deprecated=True)
async def scan_ble(request: BleScanRequest) -> dict[str, Any]:
try:
return await adapter.invoke(ACTION_DISCOVERY_SCAN, request.model_dump())
return await invoke_device_plugin_adapter(
adapter,
ACTION_DISCOVERY_SCAN,
request.model_dump(),
)
except (PluginExecutionError, ValueError) as exc:
raise HTTPException(status_code=502, detail=f"Ошибка поиска BLE: {exc}") from exc
@router.post("/api/connect", deprecated=True)
async def connect(request: ConnectRequest) -> dict[str, Any]:
try:
return await adapter.invoke(ACTION_NETWORK_PROVISION, request.model_dump())
return await invoke_device_plugin_adapter(
adapter,
ACTION_NETWORK_PROVISION,
request.model_dump(),
)
except (PluginExecutionError, ValueError) as exc:
raise HTTPException(
status_code=502,
@@ -52,31 +60,51 @@ def build_xgrids_k1_legacy_router(adapter: XgridsK1PluginFacade) -> APIRouter:
@router.post("/api/session/live", deprecated=True)
async def start_live(request: LiveRequest) -> dict[str, Any]:
try:
return await adapter.invoke(ACTION_STREAM_START_LIVE, request.model_dump())
return await invoke_device_plugin_adapter(
adapter,
ACTION_STREAM_START_LIVE,
request.model_dump(),
)
except (PluginExecutionError, ValueError) as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.post("/api/session/replay", deprecated=True)
async def start_replay(request: ReplayRequest) -> dict[str, Any]:
try:
return await adapter.invoke(ACTION_STREAM_START_REPLAY, request.model_dump())
return await invoke_device_plugin_adapter(
adapter,
ACTION_STREAM_START_REPLAY,
request.model_dump(),
)
except (PluginExecutionError, ValueError) as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.post("/api/session/stop", deprecated=True)
async def stop_session() -> dict[str, Any]:
return await adapter.invoke(ACTION_STREAM_STOP, {})
return await invoke_device_plugin_adapter(adapter, ACTION_STREAM_STOP, {})
@router.post("/api/viewer/settings", deprecated=True)
async def update_viewer_settings(request: ViewerSettingsRequest) -> dict[str, Any]:
return await adapter.invoke(ACTION_VIEWER_SETTINGS_UPDATE, request.model_dump())
return await invoke_device_plugin_adapter(
adapter,
ACTION_VIEWER_SETTINGS_UPDATE,
request.model_dump(),
)
@router.websocket("/api/events")
async def events(websocket: WebSocket) -> None:
await websocket.accept()
try:
while True:
await websocket.send_json({"state": await adapter.invoke(ACTION_STATE_READ, {})})
await websocket.send_json(
{
"state": await invoke_device_plugin_adapter(
adapter,
ACTION_STATE_READ,
{},
)
}
)
await asyncio.sleep(0.5)
except WebSocketDisconnect:
return
@@ -1,6 +1,6 @@
"""Read-only MQTT evidence capture for an owner-controlled K1."""
from k1link.mqtt.capture import (
from k1link.device_plugins.xgrids_k1.mqtt.capture import (
DEFAULT_MAX_MESSAGE_BYTES,
MAX_CONFIGURABLE_MESSAGE_BYTES,
REPORT_TOPICS,
@@ -0,0 +1,260 @@
"""K1-native observation archive and Rerun preparation adapter."""
from __future__ import annotations
import json
import os
import stat
import threading
from pathlib import Path
from k1link.device_plugins.xgrids_k1.archive import (
LegacySessionCandidate,
discover_legacy_viewer_sessions,
)
from k1link.device_plugins.xgrids_k1.rrd_export import (
RrdExportCancelled,
RrdExportError,
export_k1mqtt_to_rrd,
)
from k1link.sessions.active import recover_stale_active_session_marker
from k1link.sessions.models import (
ObservationArtifactCandidate,
ObservationSessionCandidate,
SessionIntegrityError,
SessionSource,
)
from k1link.sessions.plugin_contract import (
ObservationArchiveSource,
ObservationRuntimeContribution,
PluginRecordingExportCancelled,
PluginRecordingExportError,
)
from k1link.sessions.store import resolve_missioncore_evidence_dir
from k1link.web.camera_archive import recover_incomplete_camera_archives
XGRIDS_K1_PLUGIN_ID = "nodedc.device.xgrids-lixelkity-k1"
MAX_TIMELINE_ORIGIN_LINE_BYTES = 64 * 1024
def build_xgrids_k1_observation(repository_root: Path) -> ObservationRuntimeContribution:
"""Compose every K1 evidence root behind the generic observation ABI."""
roots = (
("xgrids-k1.viewer-live.repository", repository_root.resolve() / "sessions"),
(
"xgrids-k1.viewer-live.evidence",
resolve_missioncore_evidence_dir(repository_root),
),
)
return ObservationRuntimeContribution(
archives=tuple(
ObservationArchiveSource(
plugin_id=XGRIDS_K1_PLUGIN_ID,
archive_id=archive_id,
root=root,
discover=_discover_archive,
recover=_recover_archive,
)
for archive_id, root in roots
),
recording_exporter=_export_recording,
)
def xgrids_k1_archive_source(
root: Path,
*,
archive_id: str = "xgrids-k1.viewer-live",
) -> ObservationArchiveSource:
"""Build one explicit archive source for tests, tools, or migration jobs."""
return ObservationArchiveSource(
plugin_id=XGRIDS_K1_PLUGIN_ID,
archive_id=archive_id,
root=root,
discover=_discover_archive,
recover=_recover_archive,
)
def _recover_archive(root: Path) -> None:
recover_stale_active_session_marker(root)
recover_incomplete_camera_archives(root)
def _discover_archive(root: Path) -> tuple[ObservationSessionCandidate, ...]:
return tuple(
_to_host_candidate(candidate) for candidate in discover_legacy_viewer_sessions(root)
)
def _to_host_candidate(candidate: LegacySessionCandidate) -> ObservationSessionCandidate:
session_id = candidate.session_id
raw_path = candidate.raw_path
raw_bytes = candidate.raw_byte_length
replay_raw_bytes = candidate.replay_raw_byte_length
replay_metadata_bytes = candidate.replay_metadata_byte_length
replayable = candidate.replayable
metadata_path = raw_path.with_name("mqtt.metadata.jsonl")
artifacts = [
ObservationArtifactCandidate(
artifact_id="raw-transport-primary",
kind="raw-transport",
media_type="application/x-nodedc-k1mqtt",
locator=raw_path,
byte_length=raw_bytes,
replay_byte_length=replay_raw_bytes if replayable else 0,
sha256=candidate.raw_sha256,
integrity_status=candidate.raw_integrity_status,
)
]
if replay_metadata_bytes > 0:
artifacts.append(
ObservationArtifactCandidate(
artifact_id="raw-transport-index",
kind="raw-transport-index",
media_type="application/x-ndjson",
locator=metadata_path,
byte_length=_regular_file_size(metadata_path),
replay_byte_length=replay_metadata_bytes if replayable else 0,
sha256=None,
integrity_status=candidate.raw_integrity_status,
)
)
media_sources = candidate.media_sources
artifacts.extend(
ObservationArtifactCandidate(
artifact_id=media.artifact_id,
kind="recorded-video",
media_type="video/mp4",
locator=media.locator,
byte_length=media.byte_length,
replay_byte_length=0,
sha256=None,
integrity_status="validated-structure",
)
for media in media_sources
)
sources: list[SessionSource] = []
modalities = candidate.modalities
if "point-cloud" in modalities:
sources.append(
SessionSource(
source_id="sensor.lidar.primary",
semantic_channel_id="spatial.point-cloud.recorded",
modality="point-cloud",
status="recorded",
seekable=replayable,
artifact_id="raw-transport-primary",
)
)
if "trajectory" in modalities:
sources.append(
SessionSource(
source_id="spatial.trajectory",
semantic_channel_id="spatial.pose.recorded",
modality="trajectory",
status="recorded",
seekable=replayable,
artifact_id="raw-transport-primary",
)
)
sources.extend(
SessionSource(
source_id=media.source_id,
semantic_channel_id="camera.video.recorded",
modality="video",
status="recorded",
seekable=True,
artifact_id=media.artifact_id,
)
for media in media_sources
)
timeline_origin = _timeline_origin(metadata_path) if replayable else None
return ObservationSessionCandidate(
session_id=session_id,
display_name=candidate.display_name,
status=candidate.status,
started_at_utc=candidate.started_at_utc,
completed_at_utc=candidate.completed_at_utc,
duration_seconds=candidate.duration_seconds,
modalities=modalities,
replayable=replayable,
total_bytes=candidate.total_bytes,
allowed_root=candidate.allowed_root,
session_root=candidate.session_root,
primary_replay_artifact_id="raw-transport-primary" if replayable else None,
timeline_origin_epoch_ns=None if timeline_origin is None else timeline_origin[0],
timeline_origin_monotonic_ns=None if timeline_origin is None else timeline_origin[1],
sources=tuple(sources),
artifacts=tuple(artifacts),
)
def _regular_file_size(path: Path) -> int:
try:
value = path.lstat()
except OSError as exc:
raise SessionIntegrityError("K1 transport index is unavailable") from exc
if not stat.S_ISREG(value.st_mode) or stat.S_ISLNK(value.st_mode):
raise SessionIntegrityError("K1 transport index is not a regular file")
return value.st_size
def _timeline_origin(path: Path) -> tuple[int, int]:
flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
try:
descriptor = os.open(path, flags)
try:
payload = os.read(descriptor, MAX_TIMELINE_ORIGIN_LINE_BYTES + 1)
finally:
os.close(descriptor)
first_line = payload.splitlines(keepends=True)[0]
if len(first_line) > MAX_TIMELINE_ORIGIN_LINE_BYTES or not first_line.endswith(
(b"\n", b"\r")
):
raise ValueError
document = json.loads(first_line)
epoch_ns = document.get("received_at_epoch_ns")
monotonic_ns = document.get("received_monotonic_ns")
if (
document.get("record_type") != "message"
or document.get("sequence") != 1
or not _non_negative_int(epoch_ns)
or not _non_negative_int(monotonic_ns)
):
raise ValueError
return int(epoch_ns), int(monotonic_ns)
except (IndexError, OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
raise SessionIntegrityError("K1 transport timeline origin is invalid") from exc
def _non_negative_int(value: object) -> bool:
return isinstance(value, int) and not isinstance(value, bool) and value >= 0
def _export_recording(
source: Path,
destination: Path,
*,
cancel_event: threading.Event | None = None,
activity_callback: object | None = None,
) -> dict[str, object]:
try:
return dict(
export_k1mqtt_to_rrd(
source,
destination,
cancel_event=cancel_event,
activity_callback=activity_callback if callable(activity_callback) else None,
)
)
except RrdExportCancelled as exc:
raise PluginRecordingExportCancelled("K1 recording export was cancelled") from exc
except RrdExportError as exc:
raise PluginRecordingExportError("K1 recording export failed") from exc
@@ -1,6 +1,6 @@
"""Verified protocol decoders for captured K1 application streams."""
from k1link.protocol.streams import (
from k1link.device_plugins.xgrids_k1.protocol.streams import (
DecodeLimits,
LegacyPoint,
LegacyPointCloudFrame,
@@ -9,7 +9,7 @@ from k1link.data_plane import (
DecodedPoseView,
NormalizationError,
)
from k1link.protocol.streams import (
from k1link.device_plugins.xgrids_k1.protocol.streams import (
LegacyPointCloudFrame,
LegacyPoseFrame,
LioPointCloudFrame,
@@ -7,7 +7,7 @@ from typing import NamedTuple
import lz4.block
from k1link.protocol.protobuf_wire import (
from k1link.device_plugins.xgrids_k1.protocol.protobuf_wire import (
ProtobufWireError,
ProtoField,
decode_zigzag64,
@@ -16,8 +16,12 @@ import rerun as rr
from rerun import blueprint as rrb
from k1link.data_plane import DecodedPointCloudView, DecodedPoseView, NormalizationError
from k1link.protocol.normalizer import normalize_k1_message
from k1link.viewer.replay import ReplayFormatError, detect_replay_format, iter_replay_messages
from k1link.device_plugins.xgrids_k1.protocol.normalizer import normalize_k1_message
from k1link.device_plugins.xgrids_k1.viewer.replay import (
ReplayFormatError,
detect_replay_format,
iter_replay_messages,
)
from k1link.viewer.rerun_bridge import (
MAX_TRAJECTORY_POSES,
TRAJECTORY_APPEND_INTERVAL_NS,
@@ -0,0 +1,11 @@
"""K1-native capture, replay, and compatibility visualization runtime."""
from .messages import StreamMessage
from .replay import ReplayFormatError, detect_replay_format, iter_replay_messages
__all__ = [
"ReplayFormatError",
"StreamMessage",
"detect_replay_format",
"iter_replay_messages",
]
@@ -29,7 +29,7 @@ from foxglove.messages import (
Vector3,
)
from k1link.protocol.streams import (
from k1link.device_plugins.xgrids_k1.protocol.streams import (
LegacyPointCloudFrame,
LegacyPoseFrame,
LioPointCloudFrame,
@@ -40,7 +40,7 @@ from k1link.protocol.streams import (
decode_lio_pcl,
decode_lio_pose,
)
from k1link.viewer.messages import StreamMessage
from k1link.device_plugins.xgrids_k1.viewer.messages import StreamMessage
from k1link.viewer.metrics import BridgeMetrics
POINT_STRUCT = struct.Struct("<fffB3x")
@@ -7,9 +7,9 @@ from decimal import Decimal, InvalidOperation
from pathlib import Path
from typing import IO, Literal
from k1link.mqtt import CaptureFormatError, iter_capture_frames
from k1link.mqtt.capture import RAW_MAGIC
from k1link.viewer.messages import StreamMessage
from k1link.device_plugins.xgrids_k1.mqtt import CaptureFormatError, iter_capture_frames
from k1link.device_plugins.xgrids_k1.mqtt.capture import RAW_MAGIC
from k1link.device_plugins.xgrids_k1.viewer.messages import StreamMessage
MAX_REPLAY_PAYLOAD_BYTES = 2 * 1024 * 1024
MAX_LEGACY_LINE_BYTES = MAX_REPLAY_PAYLOAD_BYTES * 2 + 64 * 1024
@@ -11,10 +11,10 @@ from typing import Literal, Protocol, TypedDict
from k1link.artifacts import utc_now_iso, write_json_atomic
from k1link.data_plane import DecodedDataPlaneView, NormalizationError
from k1link.mqtt import CapturedMqttMessage, CaptureError, capture_mqtt
from k1link.viewer.messages import StreamMessage
from k1link.device_plugins.xgrids_k1.mqtt import CapturedMqttMessage, CaptureError, capture_mqtt
from k1link.device_plugins.xgrids_k1.viewer.messages import StreamMessage
from k1link.device_plugins.xgrids_k1.viewer.replay import iter_replay_messages
from k1link.viewer.metrics import BridgeMetrics, MetricsSnapshot
from k1link.viewer.replay import iter_replay_messages
from k1link.viewer.rerun_bridge import DEFAULT_GRPC_PORT, RerunBridge, RerunSceneSettings
RuntimePhase = Literal[
+18
View File
@@ -14,12 +14,22 @@ from .media import (
)
from .models import (
LayoutConflictError,
ObservationArtifactCandidate,
ObservationSessionCandidate,
RecordedMediaArtifact,
ReplayArtifact,
ReplayCommand,
SessionIntegrityError,
SessionNotFoundError,
SessionNotReplayableError,
)
from .plugin_contract import (
ObservationArchiveSource,
ObservationRuntimeContribution,
PluginRecordingExportCancelled,
PluginRecordingExportError,
RecordingExporter,
)
from .preparation import (
RecordingPreparationQueueFull,
RecordingPreparationSnapshot,
@@ -42,6 +52,12 @@ __all__ = [
"ActiveSessionLease",
"ActiveSessionLeaseError",
"MaterializedRecording",
"ObservationArchiveSource",
"ObservationArtifactCandidate",
"ObservationRuntimeContribution",
"ObservationSessionCandidate",
"PluginRecordingExportCancelled",
"PluginRecordingExportError",
"RecordingMaterializationCancelled",
"RecordedMediaArtifact",
"RECORDED_MEDIA_MANIFEST_SCHEMA",
@@ -49,6 +65,8 @@ __all__ = [
"RecordedMediaInspector",
"RecordedMediaManifest",
"ReplayCommand",
"ReplayArtifact",
"RecordingExporter",
"RecordingMaterializationError",
"RecordingPreparationQueueFull",
"RecordingPreparationSnapshot",
+16 -36
View File
@@ -14,8 +14,6 @@ from dataclasses import dataclass
from pathlib import Path
from typing import Any, TypeGuard
from k1link.viewer.replay import MAX_METADATA_LINE_CHARS
from .models import RecordedMediaArtifact, ReplayCommand, SessionIntegrityError
CAMERA_ARCHIVE_SCHEMA = "missioncore.camera-recording/v1"
@@ -139,7 +137,7 @@ class RecordedMediaInspector:
cached = self._cache.get(key)
if cached is not None:
identity = _prepared_source_identity(
replay.source_path,
replay,
epoch_paths,
cached.manifest.epochs,
)
@@ -151,15 +149,14 @@ class RecordedMediaInspector:
with self._lock:
self._cache[key] = _CachedManifest(identity=identity, manifest=manifest)
return manifest
origin_epoch_ns, origin_monotonic_ns = _raw_timeline_origin(replay.source_path)
manifest = _read_manifest(
artifact,
epoch_paths,
origin_epoch_ns=origin_epoch_ns,
origin_monotonic_ns=origin_monotonic_ns,
origin_epoch_ns=replay.timeline_origin_epoch_ns,
origin_monotonic_ns=replay.timeline_origin_monotonic_ns,
)
identity = _prepared_source_identity(
replay.source_path,
replay,
epoch_paths,
manifest.epochs,
)
@@ -186,7 +183,7 @@ class RecordedMediaInspector:
document = _decode_prepared_sidecar(payload)
manifest = _manifest_from_sidecar(document, artifact, epoch_paths)
identity = _prepared_source_identity(
replay.source_path,
replay,
epoch_paths,
manifest.epochs,
)
@@ -631,15 +628,15 @@ def _epoch_paths(source_path: Path) -> tuple[Path, ...]:
def _prepared_source_identity(
raw_path: Path,
replay: ReplayCommand,
epoch_paths: tuple[Path, ...],
epochs: tuple[RecordedMediaEpoch, ...],
) -> tuple[tuple[int, int, int, int], ...]:
if len(epoch_paths) != len(epochs):
raise SessionIntegrityError("recorded media epoch identity is inconsistent")
identity: list[tuple[int, int, int, int]] = []
for source_file in (raw_path, raw_path.with_name("mqtt.metadata.jsonl")):
metadata = _confined_file_stat(source_file, raw_path.parent)
for artifact in replay.artifacts:
metadata = _session_artifact_stat(artifact.path, replay.session_root)
identity.append(
(metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mtime_ns)
)
@@ -671,32 +668,15 @@ def _prepared_source_identity(
return tuple(identity)
def _raw_timeline_origin(raw_path: Path) -> tuple[int, int]:
metadata_path = raw_path.with_name("mqtt.metadata.jsonl")
def _session_artifact_stat(path: Path, session_root: Path) -> os.stat_result:
try:
line = _read_first_confined_line(
metadata_path,
raw_path.parent,
MAX_METADATA_LINE_CHARS,
).decode("utf-8")
except (OSError, UnicodeDecodeError) as exc:
raise SessionIntegrityError("native capture timing metadata is unavailable") from exc
if not line or len(line) > MAX_METADATA_LINE_CHARS or not line.endswith(("\n", "\r")):
raise SessionIntegrityError("native capture timing origin is incomplete")
try:
record = json.loads(line)
except json.JSONDecodeError as exc:
raise SessionIntegrityError("native capture timing origin is invalid") from exc
epoch_ns = record.get("received_at_epoch_ns") if isinstance(record, dict) else None
monotonic_ns = record.get("received_monotonic_ns") if isinstance(record, dict) else None
if (
record.get("record_type") != "message"
or record.get("sequence") != 1
or not _non_negative_int(epoch_ns)
or not _non_negative_int(monotonic_ns)
):
raise SessionIntegrityError("native capture timing origin is invalid")
return int(epoch_ns), int(monotonic_ns)
session = session_root.resolve(strict=True)
parent = path.parent.resolve(strict=True)
except OSError as exc:
raise SessionIntegrityError("recording source artifact is missing") from exc
if not parent.is_relative_to(session):
raise SessionIntegrityError("recording source artifact escapes its session")
return _confined_file_stat(path, parent)
def _read_manifest(
+47 -13
View File
@@ -160,20 +160,44 @@ class WorkspaceLayout:
}
@dataclass(frozen=True, slots=True)
class ReplayArtifact:
"""One confined input artifact selected for plugin-owned preparation."""
artifact_id: str
path: Path
media_type: str
file_byte_length: int
replay_byte_length: int
expected_sha256: str | None
@dataclass(frozen=True, slots=True)
class ReplayCommand:
"""Internal-only replay command. ``source_path`` never enters an API DTO."""
"""Internal replay request containing no vendor format or channel names."""
session_id: str
source_path: Path
plugin_id: str
allowed_root: Path
session_root: Path
replay_byte_length: int
metadata_byte_length: int
expected_source_sha256: str | None
primary_artifact_id: str
artifacts: tuple[ReplayArtifact, ...]
timeline_origin_epoch_ns: int
timeline_origin_monotonic_ns: int
speed: float
loop: bool
@property
def primary_artifact(self) -> ReplayArtifact:
matches = tuple(
artifact
for artifact in self.artifacts
if artifact.artifact_id == self.primary_artifact_id
)
if len(matches) != 1:
raise SessionIntegrityError("replay command has no unique primary artifact")
return matches[0]
@dataclass(frozen=True, slots=True)
class RecordedMediaArtifact:
@@ -191,7 +215,19 @@ class RecordedMediaArtifact:
@dataclass(frozen=True, slots=True)
class LegacySessionCandidate:
class ObservationArtifactCandidate:
artifact_id: str
kind: str
media_type: str
locator: Path
byte_length: int
replay_byte_length: int
sha256: str | None
integrity_status: str
@dataclass(frozen=True, slots=True)
class ObservationSessionCandidate:
session_id: str
display_name: str
status: SessionStatus
@@ -203,13 +239,11 @@ class LegacySessionCandidate:
total_bytes: int
allowed_root: Path
session_root: Path
raw_path: Path
raw_byte_length: int
replay_raw_byte_length: int
replay_metadata_byte_length: int
raw_sha256: str | None
raw_integrity_status: str
media_sources: tuple[LegacyMediaSourceCandidate, ...]
primary_replay_artifact_id: str | None
timeline_origin_epoch_ns: int | None
timeline_origin_monotonic_ns: int | None
sources: tuple[SessionSource, ...]
artifacts: tuple[ObservationArtifactCandidate, ...]
@dataclass(frozen=True, slots=True)
+68
View File
@@ -0,0 +1,68 @@
"""Host-side runtime ABI for observation-capable device plugins.
The contract deliberately contains no transport name, vendor topic, capture
suffix, codec, or viewer implementation. Concrete plugins discover native
evidence and export it into the host's canonical recorded-viewer artifact.
"""
from __future__ import annotations
import threading
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Protocol
from .models import ObservationSessionCandidate
RecordingProgressPulse = Callable[[], None]
RecordingExportResult = Mapping[str, object]
class PluginRecordingExportError(RuntimeError):
"""A plugin rejected or failed to convert one native recording."""
class PluginRecordingExportCancelled(PluginRecordingExportError):
"""A plugin cooperatively stopped a recording conversion."""
class RecordingExporter(Protocol):
def __call__(
self,
source: Path,
destination: Path,
*,
cancel_event: threading.Event | None = None,
activity_callback: RecordingProgressPulse | None = None,
) -> RecordingExportResult: ...
ObservationArchiveDiscovery = Callable[
[Path],
tuple[ObservationSessionCandidate, ...],
]
ObservationArchiveRecovery = Callable[[Path], object]
def _no_recovery(_: Path) -> object:
return None
@dataclass(frozen=True, slots=True)
class ObservationArchiveSource:
"""One plugin-owned evidence namespace reconciled by the host catalog."""
plugin_id: str
archive_id: str
root: Path
discover: ObservationArchiveDiscovery
recover: ObservationArchiveRecovery = _no_recovery
@dataclass(frozen=True, slots=True)
class ObservationRuntimeContribution:
"""Optional observation capabilities contributed by a device plugin."""
archives: tuple[ObservationArchiveSource, ...]
recording_exporter: RecordingExporter
+13 -5
View File
@@ -584,12 +584,20 @@ def _source_identity(command: ReplayCommand) -> tuple[object, ...]:
identities: list[object] = [
command.session_id,
str(command.source_path),
command.replay_byte_length,
command.metadata_byte_length,
command.expected_source_sha256,
command.plugin_id,
command.primary_artifact_id,
]
for path in (command.source_path, command.source_path.with_name("mqtt.metadata.jsonl")):
for artifact in command.artifacts:
path = artifact.path
identities.extend(
(
artifact.artifact_id,
artifact.media_type,
artifact.file_byte_length,
artifact.replay_byte_length,
artifact.expected_sha256,
)
)
try:
value = os.lstat(path)
except OSError:
+276 -171
View File
@@ -16,18 +16,17 @@ from pathlib import Path
from typing import Any, cast
from uuid import uuid4
from k1link.viewer.rrd_export import (
RrdExportCancelled,
RrdExportError,
export_k1mqtt_to_rrd,
from .models import ReplayArtifact, ReplayCommand
from .plugin_contract import (
PluginRecordingExportCancelled,
PluginRecordingExportError,
RecordingExporter,
)
from .models import ReplayCommand
# v6 adds a real session_time=0 row to the RRD itself. Older sidecars can
# declare a zero start while their payload begins at the first decoded sensor
# frame, so accepting them would violate the browser playback contract.
CACHE_SCHEMA = "missioncore.derived-rerun-recording-cache/v6"
CACHE_SCHEMA = "missioncore.derived-rerun-recording-cache/v7"
COMPATIBLE_CACHE_SCHEMAS = frozenset({CACHE_SCHEMA})
RERUN_RECORDING_MEDIA_TYPE = "application/vnd.rerun.rrd"
RERUN_SESSION_TIMELINE = "session_time"
@@ -37,7 +36,6 @@ DEFAULT_FREE_SPACE_RESERVE_BYTES = 2 * 1024 * 1024 * 1024
RrdExporter = Callable[..., Mapping[str, object]]
RecordingProgressCallback = Callable[[str, float], None]
DEFAULT_RRD_EXPORTER = cast(RrdExporter, export_k1mqtt_to_rrd)
class RecordingMaterializationError(RuntimeError):
@@ -71,29 +69,58 @@ class _ValidatedMemoryEntry:
@dataclass(frozen=True, slots=True)
class _ValidatedSource:
source: Path
metadata: Path
source_stat: os.stat_result
metadata_stat: os.stat_result
class _ValidatedArtifact:
artifact_id: str
path: Path
media_type: str
file_stat: os.stat_result
replay_byte_length: int
metadata_byte_length: int
expected_source_sha256: str | None
expected_sha256: str | None
@property
def identity(self) -> tuple[object, ...]:
return (
*_stat_identity(self.source_stat),
*_stat_identity(self.metadata_stat),
self.artifact_id,
self.media_type,
*_stat_identity(self.file_stat),
self.replay_byte_length,
self.metadata_byte_length,
)
@dataclass(frozen=True, slots=True)
class _ValidatedSource:
plugin_id: str
primary_artifact_id: str
artifacts: tuple[_ValidatedArtifact, ...]
@property
def primary(self) -> _ValidatedArtifact:
matches = tuple(
artifact
for artifact in self.artifacts
if artifact.artifact_id == self.primary_artifact_id
)
if len(matches) != 1:
raise RecordingMaterializationError("recording source has no primary artifact")
return matches[0]
@property
def identity(self) -> tuple[object, ...]:
return (
self.plugin_id,
self.primary_artifact_id,
*(item for artifact in self.artifacts for item in artifact.identity),
)
@property
def replay_byte_length(self) -> int:
return sum(artifact.replay_byte_length for artifact in self.artifacts)
class SessionRecordingMaterializer:
"""Build and validate a per-session seekable RRD under the private data root.
The native ``.k1mqtt`` capture remains the source of record. Derived RRDs
Native plugin evidence remains the source of record. Derived RRDs
live below ``data_dir/recordings`` and are reused only when both their
source identity and output digest still match an atomically written cache
sidecar. Calls for one session are serialized, so concurrent browser
@@ -104,7 +131,8 @@ class SessionRecordingMaterializer:
self,
data_dir: Path,
*,
exporter: RrdExporter = DEFAULT_RRD_EXPORTER,
exporter: RrdExporter | None = None,
exporters: Mapping[str, RecordingExporter] | None = None,
cache_max_bytes: int | None = None,
free_space_reserve_bytes: int | None = None,
) -> None:
@@ -119,12 +147,12 @@ class SessionRecordingMaterializer:
self.recordings_root = recordings_root.resolve()
if not self.recordings_root.is_relative_to(private_root):
raise RecordingMaterializationError("recording cache escapes the private data root")
self._exporter = exporter
self._exporter_accepts_cancel = _callable_accepts_keyword(exporter, "cancel_event")
self._exporter_accepts_activity = _callable_accepts_keyword(
exporter,
"activity_callback",
)
if exporter is not None and exporters is not None:
raise ValueError("configure either one test exporter or plugin exporters")
self._fallback_exporter = exporter
self._exporters = dict(exporters or {})
if len(self._exporters) != len(set(self._exporters)):
raise ValueError("recording exporter plugin ids must be unique")
self.cache_max_bytes = _positive_configuration(
cache_max_bytes,
environment_name="MISSIONCORE_RRD_CACHE_MAX_BYTES",
@@ -152,7 +180,12 @@ class SessionRecordingMaterializer:
def supports_cooperative_cancellation(self) -> bool:
"""Whether the configured exporter observes a cancellation event."""
return self._exporter_accepts_cancel
exporters: tuple[Callable[..., object], ...] = tuple(self._exporters.values())
if self._fallback_exporter is not None:
exporters = (*exporters, self._fallback_exporter)
return bool(exporters) and all(
_callable_accepts_keyword(exporter, "cancel_event") for exporter in exporters
)
def is_recording_available(self, recording: MaterializedRecording) -> bool:
"""Cheap no-follow check for a previously validated ready handle."""
@@ -463,11 +496,11 @@ class SessionRecordingMaterializer:
) from exc
staged_root: Path | None = None
export_source = source.source
export_source = source.primary.path
try:
if (
source.replay_byte_length != source.source_stat.st_size
or source.metadata_byte_length != source.metadata_stat.st_size
if any(
artifact.replay_byte_length != artifact.file_stat.st_size
for artifact in source.artifacts
):
staged_root, export_source = _stage_replay_prefix(
resolved_session_root,
@@ -480,6 +513,7 @@ class SessionRecordingMaterializer:
),
)
summary = self._invoke_exporter(
source.plugin_id,
export_source,
candidate_path,
cancel_event=cancel_event,
@@ -491,12 +525,12 @@ class SessionRecordingMaterializer:
)
_raise_if_cancelled(cancel_event)
_report_progress(progress_callback, "finalizing", 0.9)
except RrdExportCancelled as exc:
except PluginRecordingExportCancelled as exc:
candidate_path.unlink(missing_ok=True)
raise RecordingMaterializationCancelled(
"recording preparation was cancelled"
) from exc
except RrdExportError as exc:
except PluginRecordingExportError as exc:
candidate_path.unlink(missing_ok=True)
raise RecordingMaterializationError("native capture could not be exported") from exc
except OSError as exc:
@@ -515,13 +549,13 @@ class SessionRecordingMaterializer:
raise RecordingMaterializationError("native capture changed during RRD export")
_chmod_best_effort(candidate_path, 0o600)
source_sha256 = _sha256_prefix_stable(
source.source,
source.source_stat,
source.replay_byte_length,
source.primary.path,
source.primary.file_stat,
source.primary.replay_byte_length,
)
if (
source.expected_source_sha256 is not None
and source_sha256 != source.expected_source_sha256
source.primary.expected_sha256 is not None
and source_sha256 != source.primary.expected_sha256
):
raise RecordingMaterializationError(
"native capture digest no longer matches catalog"
@@ -574,18 +608,25 @@ class SessionRecordingMaterializer:
def _invoke_exporter(
self,
plugin_id: str,
source: Path,
destination: Path,
*,
cancel_event: threading.Event | None,
activity_callback: Callable[[], None],
) -> Mapping[str, object]:
selected = self._fallback_exporter or self._exporters.get(plugin_id)
if selected is None:
raise PluginRecordingExportError(
f"device plugin has no recording exporter: {plugin_id}"
)
exporter = cast(RrdExporter, selected)
kwargs: dict[str, object] = {}
if self._exporter_accepts_cancel:
if _callable_accepts_keyword(exporter, "cancel_event"):
kwargs["cancel_event"] = cancel_event
if self._exporter_accepts_activity:
if _callable_accepts_keyword(exporter, "activity_callback"):
kwargs["activity_callback"] = activity_callback
return self._exporter(source, destination, **kwargs)
return exporter(source, destination, **kwargs)
def _load_memory_cache(
self,
@@ -639,48 +680,53 @@ class SessionRecordingMaterializer:
except (OSError, ValueError, json.JSONDecodeError, RecordingMaterializationError):
return None
if document["source_file_byte_length"] != source.source_stat.st_size:
if document["plugin_id"] != source.plugin_id:
return None
if document["source_mtime_ns"] != source.source_stat.st_mtime_ns:
if document["primary_artifact_id"] != source.primary_artifact_id:
return None
if document["source_ctime_ns"] != source.source_stat.st_ctime_ns:
return None
if document["source_replay_byte_length"] != source.replay_byte_length:
return None
if document["metadata_file_byte_length"] != source.metadata_stat.st_size:
return None
if document["metadata_mtime_ns"] != source.metadata_stat.st_mtime_ns:
return None
if document["metadata_ctime_ns"] != source.metadata_stat.st_ctime_ns:
return None
if document["metadata_replay_byte_length"] != source.metadata_byte_length:
cached_artifacts = document["source_artifacts"]
if len(cached_artifacts) != len(source.artifacts):
return None
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
if cached["artifact_id"] != artifact.artifact_id:
return None
if cached["media_type"] != artifact.media_type:
return None
if cached["file_byte_length"] != artifact.file_stat.st_size:
return None
if cached["mtime_ns"] != artifact.file_stat.st_mtime_ns:
return None
if cached["ctime_ns"] != artifact.file_stat.st_ctime_ns:
return None
if cached["replay_byte_length"] != artifact.replay_byte_length:
return None
if document["recording_byte_length"] != recording_stat.st_size:
return None
if document["recording_mtime_ns"] != recording_stat.st_mtime_ns:
return None
source_sha256 = _sha256_prefix_stable(
source.source,
source.source_stat,
source.replay_byte_length,
source.primary.path,
source.primary.file_stat,
source.primary.replay_byte_length,
)
if (
source.expected_source_sha256 is not None
and source_sha256 != source.expected_source_sha256
source.primary.expected_sha256 is not None
and source_sha256 != source.primary.expected_sha256
):
raise RecordingMaterializationError(
"native capture digest no longer matches catalog"
)
if source_sha256 != document["source_sha256"]:
return None
metadata_sha256 = _sha256_prefix_stable(
source.metadata,
source.metadata_stat,
source.metadata_byte_length,
)
if metadata_sha256 != document["metadata_sha256"]:
return None
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
digest = _sha256_prefix_stable(
artifact.path,
artifact.file_stat,
artifact.replay_byte_length,
)
if digest != cached["sha256"]:
return None
recording_sha256 = _sha256_stable(recording_path, recording_stat)
if recording_sha256 != document["recording_sha256"]:
return None
@@ -764,76 +810,103 @@ def _raise_if_cancelled(cancel_event: threading.Event | None) -> None:
def _validate_source(command: ReplayCommand) -> _ValidatedSource:
source_path = getattr(command, "source_path", None)
plugin_id = getattr(command, "plugin_id", None)
allowed_root = getattr(command, "allowed_root", None)
session_root = getattr(command, "session_root", None)
replay_byte_length = getattr(command, "replay_byte_length", None)
metadata_byte_length = getattr(command, "metadata_byte_length", None)
expected_source_sha256 = getattr(command, "expected_source_sha256", None)
if not isinstance(source_path, Path):
raise RecordingMaterializationError("replay command has no native capture")
primary_artifact_id = getattr(command, "primary_artifact_id", None)
artifacts = getattr(command, "artifacts", None)
if not isinstance(plugin_id, str) or SESSION_ID_PATTERN.fullmatch(plugin_id) is None:
raise RecordingMaterializationError("replay command has an invalid plugin id")
if not isinstance(primary_artifact_id, str) or SESSION_ID_PATTERN.fullmatch(
primary_artifact_id
) is None:
raise RecordingMaterializationError("replay command has an invalid primary artifact")
if not isinstance(artifacts, tuple) or not artifacts:
raise RecordingMaterializationError("replay command has no source artifacts")
try:
if not isinstance(allowed_root, Path) or not isinstance(session_root, Path):
raise RecordingMaterializationError("replay command has no confinement roots")
allowed = allowed_root.expanduser().resolve(strict=True)
session = session_root.expanduser().resolve(strict=True)
source = source_path.expanduser().absolute()
source_parent = source.parent.resolve(strict=True)
except OSError as exc:
raise RecordingMaterializationError("native capture is missing") from exc
if (
not allowed.is_dir()
or not session.is_dir()
or not session.is_relative_to(allowed)
or not source_parent.is_relative_to(session)
):
raise RecordingMaterializationError("native capture escapes its allowed session root")
if source.suffix.casefold() != ".k1mqtt":
raise RecordingMaterializationError("native capture has an unsupported format")
source_stat = _regular_file_stat_nofollow(source, "native capture")
metadata = source.with_name("mqtt.metadata.jsonl")
if not metadata.parent.resolve(strict=True).is_relative_to(session):
raise RecordingMaterializationError("native metadata escapes its allowed session root")
metadata_stat = _regular_file_stat_nofollow(metadata, "native metadata")
if (
not isinstance(replay_byte_length, int)
or isinstance(replay_byte_length, bool)
or not 1 <= replay_byte_length <= source_stat.st_size
):
raise RecordingMaterializationError("native capture replay boundary is invalid")
if (
not isinstance(metadata_byte_length, int)
or isinstance(metadata_byte_length, bool)
or not 1 <= metadata_byte_length <= metadata_stat.st_size
):
raise RecordingMaterializationError("native metadata replay boundary is invalid")
if expected_source_sha256 is not None:
if not isinstance(expected_source_sha256, str) or not _is_sha256(expected_source_sha256):
raise RecordingMaterializationError("native capture expected digest is invalid")
if replay_byte_length != source_stat.st_size:
raise RecordingMaterializationError(
"a full-capture digest cannot describe a replay prefix"
raise RecordingMaterializationError("recording confinement root is missing") from exc
if not allowed.is_dir() or not session.is_dir() or not session.is_relative_to(allowed):
raise RecordingMaterializationError("recording source escapes its allowed session root")
validated: list[_ValidatedArtifact] = []
seen_ids: set[str] = set()
seen_names: set[str] = set()
for artifact in artifacts:
if not isinstance(artifact, ReplayArtifact):
raise RecordingMaterializationError("replay command contains an invalid artifact")
if (
SESSION_ID_PATTERN.fullmatch(artifact.artifact_id) is None
or artifact.artifact_id in seen_ids
):
raise RecordingMaterializationError("replay artifact id is invalid or duplicated")
path = artifact.path.expanduser().absolute()
try:
parent = path.parent.resolve(strict=True)
except OSError as exc:
raise RecordingMaterializationError("recording source artifact is missing") from exc
if not parent.is_relative_to(session):
raise RecordingMaterializationError("recording source artifact escapes its session")
file_stat = _regular_file_stat_nofollow(path, "recording source artifact")
if artifact.file_byte_length > file_stat.st_size:
raise RecordingMaterializationError("recording artifact was truncated after cataloging")
if (
isinstance(artifact.replay_byte_length, bool)
or not 1 <= artifact.replay_byte_length <= file_stat.st_size
):
raise RecordingMaterializationError("recording artifact replay boundary is invalid")
if artifact.expected_sha256 is not None:
if not _is_sha256(artifact.expected_sha256):
raise RecordingMaterializationError("recording artifact digest is invalid")
if artifact.replay_byte_length != file_stat.st_size:
raise RecordingMaterializationError(
"a full-artifact digest cannot describe a replay prefix"
)
if path.name in seen_names:
raise RecordingMaterializationError("recording artifact filenames are duplicated")
seen_ids.add(artifact.artifact_id)
seen_names.add(path.name)
validated.append(
_ValidatedArtifact(
artifact_id=artifact.artifact_id,
path=path,
media_type=artifact.media_type,
file_stat=file_stat,
replay_byte_length=artifact.replay_byte_length,
expected_sha256=artifact.expected_sha256,
)
)
if sum(artifact.artifact_id == primary_artifact_id for artifact in validated) != 1:
raise RecordingMaterializationError("recording primary artifact is unavailable")
return _ValidatedSource(
source=source,
metadata=metadata,
source_stat=source_stat,
metadata_stat=metadata_stat,
replay_byte_length=replay_byte_length,
metadata_byte_length=metadata_byte_length,
expected_source_sha256=expected_source_sha256,
plugin_id=plugin_id,
primary_artifact_id=primary_artifact_id,
artifacts=tuple(validated),
)
def _validate_source_state(source: _ValidatedSource) -> _ValidatedSource:
return _ValidatedSource(
source=source.source,
metadata=source.metadata,
source_stat=_regular_file_stat_nofollow(source.source, "native capture"),
metadata_stat=_regular_file_stat_nofollow(source.metadata, "native metadata"),
replay_byte_length=source.replay_byte_length,
metadata_byte_length=source.metadata_byte_length,
expected_source_sha256=source.expected_source_sha256,
plugin_id=source.plugin_id,
primary_artifact_id=source.primary_artifact_id,
artifacts=tuple(
_ValidatedArtifact(
artifact_id=artifact.artifact_id,
path=artifact.path,
media_type=artifact.media_type,
file_stat=_regular_file_stat_nofollow(
artifact.path,
"recording source artifact",
),
replay_byte_length=artifact.replay_byte_length,
expected_sha256=artifact.expected_sha256,
)
for artifact in source.artifacts
),
)
@@ -880,20 +953,25 @@ def _cache_document(
return {
"schema_version": CACHE_SCHEMA,
"session_id": recording.session_id,
"source_file_byte_length": source.source_stat.st_size,
"source_replay_byte_length": source.replay_byte_length,
"source_mtime_ns": source.source_stat.st_mtime_ns,
"source_ctime_ns": source.source_stat.st_ctime_ns,
"plugin_id": source.plugin_id,
"primary_artifact_id": source.primary_artifact_id,
"source_sha256": recording.source_sha256,
"metadata_file_byte_length": source.metadata_stat.st_size,
"metadata_replay_byte_length": source.metadata_byte_length,
"metadata_mtime_ns": source.metadata_stat.st_mtime_ns,
"metadata_ctime_ns": source.metadata_stat.st_ctime_ns,
"metadata_sha256": _sha256_prefix_stable(
source.metadata,
source.metadata_stat,
source.metadata_byte_length,
),
"source_artifacts": [
{
"artifact_id": artifact.artifact_id,
"media_type": artifact.media_type,
"file_byte_length": artifact.file_stat.st_size,
"replay_byte_length": artifact.replay_byte_length,
"mtime_ns": artifact.file_stat.st_mtime_ns,
"ctime_ns": artifact.file_stat.st_ctime_ns,
"sha256": _sha256_prefix_stable(
artifact.path,
artifact.file_stat,
artifact.replay_byte_length,
),
}
for artifact in source.artifacts
],
"recording_byte_length": recording.byte_length,
"recording_mtime_ns": recording_stat.st_mtime_ns,
"recording_sha256": recording.sha256,
@@ -909,16 +987,10 @@ def _validate_cache_document(value: object, session_id: str) -> dict[str, Any]:
expected_keys = {
"schema_version",
"session_id",
"source_file_byte_length",
"source_replay_byte_length",
"source_mtime_ns",
"source_ctime_ns",
"plugin_id",
"primary_artifact_id",
"source_sha256",
"metadata_file_byte_length",
"metadata_replay_byte_length",
"metadata_mtime_ns",
"metadata_ctime_ns",
"metadata_sha256",
"source_artifacts",
"recording_byte_length",
"recording_mtime_ns",
"recording_sha256",
@@ -932,15 +1004,10 @@ def _validate_cache_document(value: object, session_id: str) -> dict[str, Any]:
raise ValueError("recording cache sidecar identity does not match")
if value["timeline"] != RERUN_SESSION_TIMELINE:
raise ValueError("recording cache timeline is unsupported")
for key in ("plugin_id", "primary_artifact_id"):
if not isinstance(value[key], str) or SESSION_ID_PATTERN.fullmatch(value[key]) is None:
raise ValueError("recording cache contains an invalid identifier")
for key in (
"source_file_byte_length",
"source_replay_byte_length",
"source_mtime_ns",
"source_ctime_ns",
"metadata_file_byte_length",
"metadata_replay_byte_length",
"metadata_mtime_ns",
"metadata_ctime_ns",
"recording_byte_length",
"recording_mtime_ns",
"timeline_start_ns",
@@ -948,10 +1015,51 @@ def _validate_cache_document(value: object, session_id: str) -> dict[str, Any]:
):
if not isinstance(value[key], int) or isinstance(value[key], bool) or value[key] < 0:
raise ValueError("recording cache contains an invalid integer")
for key in ("source_sha256", "metadata_sha256", "recording_sha256"):
for key in ("source_sha256", "recording_sha256"):
digest = value[key]
if not isinstance(digest, str) or not _is_sha256(digest):
raise ValueError("recording cache contains an invalid digest")
source_artifacts = value["source_artifacts"]
if not isinstance(source_artifacts, list) or not source_artifacts:
raise ValueError("recording cache source artifacts are invalid")
artifact_ids: set[str] = set()
artifact_keys = {
"artifact_id",
"media_type",
"file_byte_length",
"replay_byte_length",
"mtime_ns",
"ctime_ns",
"sha256",
}
for artifact in source_artifacts:
if not isinstance(artifact, dict) or set(artifact) != artifact_keys:
raise ValueError("recording cache source artifact is invalid")
artifact_id = artifact["artifact_id"]
if (
not isinstance(artifact_id, str)
or SESSION_ID_PATTERN.fullmatch(artifact_id) is None
or artifact_id in artifact_ids
):
raise ValueError("recording cache source artifact id is invalid")
artifact_ids.add(artifact_id)
if not isinstance(artifact["media_type"], str) or not artifact["media_type"]:
raise ValueError("recording cache source media type is invalid")
for key in (
"file_byte_length",
"replay_byte_length",
"mtime_ns",
"ctime_ns",
):
item = artifact[key]
if not isinstance(item, int) or isinstance(item, bool) or item < 0:
raise ValueError("recording cache source artifact boundary is invalid")
if not 1 <= artifact["replay_byte_length"] <= artifact["file_byte_length"]:
raise ValueError("recording cache source replay boundary is invalid")
if not isinstance(artifact["sha256"], str) or not _is_sha256(artifact["sha256"]):
raise ValueError("recording cache source digest is invalid")
if value["primary_artifact_id"] not in artifact_ids:
raise ValueError("recording cache primary artifact is unavailable")
if value["timeline_end_ns"] < value["timeline_start_ns"]:
raise ValueError("recording cache timeline bounds are invalid")
return cast(dict[str, Any], value)
@@ -1061,26 +1169,23 @@ def _stage_replay_prefix(
staged_root = session_cache_root / f".source.{uuid4().hex}.tmp"
try:
staged_root.mkdir(mode=0o700)
staged_source = staged_root / "mqtt.raw.k1mqtt"
staged_metadata = staged_root / "mqtt.metadata.jsonl"
_copy_prefix_nofollow(
source.source,
source.source_stat,
staged_source,
source.replay_byte_length,
cancel_event=cancel_event,
activity_callback=activity_callback,
)
_copy_prefix_nofollow(
source.metadata,
source.metadata_stat,
staged_metadata,
source.metadata_byte_length,
cancel_event=cancel_event,
activity_callback=activity_callback,
)
staged_primary: Path | None = None
for artifact in source.artifacts:
staged = staged_root / artifact.path.name
_copy_prefix_nofollow(
artifact.path,
artifact.file_stat,
staged,
artifact.replay_byte_length,
cancel_event=cancel_event,
activity_callback=activity_callback,
)
if artifact.artifact_id == source.primary_artifact_id:
staged_primary = staged
if staged_primary is None:
raise RecordingMaterializationError("staged recording has no primary artifact")
_fsync_directory(staged_root)
return staged_root, staged_source
return staged_root, staged_primary
except BaseException:
shutil.rmtree(staged_root, ignore_errors=True)
raise
+185 -126
View File
@@ -12,11 +12,12 @@ from typing import Any, cast
from k1link.artifacts import utc_now_iso
from .legacy import discover_legacy_viewer_sessions
from .models import (
LayoutConflictError,
LegacySessionCandidate,
ObservationArtifactCandidate,
ObservationSessionCandidate,
RecordedMediaArtifact,
ReplayArtifact,
ReplayCommand,
SessionArtifact,
SessionDetail,
@@ -30,6 +31,7 @@ from .models import (
SessionSummary,
WorkspaceLayout,
)
from .plugin_contract import ObservationArchiveSource
IDENTIFIER_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
MAX_LAYOUT_BYTES = 256 * 1024
@@ -38,6 +40,8 @@ DATABASE_NAME = "mission-core.sqlite3"
SCHEMA_SQL = """
CREATE TABLE IF NOT EXISTS observation_sessions (
session_id TEXT PRIMARY KEY,
plugin_id TEXT NOT NULL,
archive_id TEXT NOT NULL,
display_name TEXT NOT NULL,
status TEXT NOT NULL CHECK (status IN ('ready', 'interrupted', 'failed')),
started_at_utc TEXT,
@@ -48,8 +52,9 @@ CREATE TABLE IF NOT EXISTS observation_sessions (
origin TEXT NOT NULL,
source_count INTEGER NOT NULL,
total_bytes INTEGER NOT NULL,
replay_raw_bytes INTEGER NOT NULL DEFAULT 0,
replay_metadata_bytes INTEGER NOT NULL DEFAULT 0,
primary_replay_artifact_id TEXT,
timeline_origin_epoch_ns INTEGER,
timeline_origin_monotonic_ns INTEGER,
allowed_root TEXT NOT NULL,
session_root TEXT NOT NULL,
created_at_utc TEXT NOT NULL,
@@ -68,6 +73,7 @@ CREATE TABLE IF NOT EXISTS observation_session_artifacts (
sha256 TEXT,
integrity_status TEXT NOT NULL,
locator TEXT NOT NULL,
replay_byte_length INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (session_id, artifact_id)
);
@@ -129,20 +135,22 @@ class SessionStore:
self._lock = threading.RLock()
self._initialize()
def import_legacy_viewer_live(self, root: Path) -> tuple[str, ...]:
allowed_root = root.expanduser().resolve()
candidates = discover_legacy_viewer_sessions(allowed_root)
def reconcile_archive(self, source: ObservationArchiveSource) -> tuple[str, ...]:
"""Reconcile one plugin-owned evidence namespace into the host catalog."""
allowed_root = source.root.expanduser().resolve()
candidates = source.discover(allowed_root)
imported: list[str] = []
for candidate in candidates:
self._upsert_legacy(candidate)
self._upsert_candidate(source, candidate)
imported.append(candidate.session_id)
with self._lock, self._connect() as connection:
connection.execute("BEGIN IMMEDIATE")
discovered = set(imported)
indexed = connection.execute(
"SELECT session_id FROM observation_sessions "
"WHERE origin = 'legacy-viewer-live' AND allowed_root = ?",
(str(allowed_root),),
"WHERE plugin_id = ? AND archive_id = ? AND allowed_root = ?",
(source.plugin_id, source.archive_id, str(allowed_root)),
).fetchall()
stale = [row["session_id"] for row in indexed if row["session_id"] not in discovered]
connection.executemany(
@@ -240,29 +248,57 @@ class SessionStore:
if not 0 <= speed <= 100:
raise ValueError("speed must be within 0..100")
with self._connect() as connection:
row = connection.execute(
"SELECT s.allowed_root, s.session_root, s.replay_raw_bytes, "
"s.replay_metadata_bytes, a.locator, a.sha256 "
"FROM observation_sessions AS s "
"JOIN observation_session_artifacts AS a ON a.session_id = s.session_id "
"WHERE s.session_id = ? AND a.artifact_id = 'raw-mqtt'",
session = connection.execute(
"SELECT plugin_id, allowed_root, session_root, "
"primary_replay_artifact_id, timeline_origin_epoch_ns, "
"timeline_origin_monotonic_ns FROM observation_sessions WHERE session_id = ?",
(session_id,),
).fetchone()
if row is None:
artifact_rows = connection.execute(
"SELECT artifact_id, media_type, byte_length, replay_byte_length, locator, sha256 "
"FROM observation_session_artifacts WHERE session_id = ? "
"AND replay_byte_length > 0 ORDER BY artifact_id",
(session_id,),
).fetchall()
if session is None or not artifact_rows:
raise SessionNotReplayableError("observation session replay artifact is unavailable")
source_path = _resolve_confined_artifact(
Path(row["allowed_root"]),
Path(row["session_root"]),
Path(row["locator"]),
primary_artifact_id = session["primary_replay_artifact_id"]
epoch_ns = session["timeline_origin_epoch_ns"]
monotonic_ns = session["timeline_origin_monotonic_ns"]
if (
not isinstance(primary_artifact_id, str)
or not isinstance(epoch_ns, int)
or not isinstance(monotonic_ns, int)
):
raise SessionNotReplayableError("observation session replay contract is incomplete")
allowed_root = Path(session["allowed_root"])
session_root = Path(session["session_root"])
artifacts = tuple(
ReplayArtifact(
artifact_id=row["artifact_id"],
path=_resolve_confined_artifact(
allowed_root,
session_root,
Path(row["locator"]),
),
media_type=row["media_type"],
file_byte_length=int(row["byte_length"]),
replay_byte_length=int(row["replay_byte_length"]),
expected_sha256=row["sha256"],
)
for row in artifact_rows
)
if sum(artifact.artifact_id == primary_artifact_id for artifact in artifacts) != 1:
raise SessionNotReplayableError("observation session primary artifact is unavailable")
return ReplayCommand(
session_id=session_id,
source_path=source_path,
allowed_root=Path(row["allowed_root"]),
session_root=Path(row["session_root"]),
replay_byte_length=int(row["replay_raw_bytes"]),
metadata_byte_length=int(row["replay_metadata_bytes"]),
expected_source_sha256=row["sha256"],
plugin_id=session["plugin_id"],
allowed_root=allowed_root,
session_root=session_root,
primary_artifact_id=primary_artifact_id,
artifacts=artifacts,
timeline_origin_epoch_ns=epoch_ns,
timeline_origin_monotonic_ns=monotonic_ns,
speed=float(speed),
loop=loop,
)
@@ -387,58 +423,92 @@ class SessionStore:
def _initialize(self) -> None:
with self._connect() as connection:
connection.executescript(SCHEMA_SQL)
columns = {
session_columns = {
row["name"]
for row in connection.execute("PRAGMA table_info(observation_sessions)")
}
if "replay_raw_bytes" not in columns:
for name, declaration in (
("plugin_id", "TEXT NOT NULL DEFAULT ''"),
("archive_id", "TEXT NOT NULL DEFAULT ''"),
("primary_replay_artifact_id", "TEXT"),
("timeline_origin_epoch_ns", "INTEGER"),
("timeline_origin_monotonic_ns", "INTEGER"),
):
if name in session_columns:
continue
connection.execute(
"ALTER TABLE observation_sessions "
"ADD COLUMN replay_raw_bytes INTEGER NOT NULL DEFAULT 0"
f"ALTER TABLE observation_sessions ADD COLUMN {name} {declaration}" # noqa: S608
)
if "replay_metadata_bytes" not in columns:
artifact_columns = {
row["name"]
for row in connection.execute(
"PRAGMA table_info(observation_session_artifacts)"
)
}
if "replay_byte_length" not in artifact_columns:
connection.execute(
"ALTER TABLE observation_sessions "
"ADD COLUMN replay_metadata_bytes INTEGER NOT NULL DEFAULT 0"
"ALTER TABLE observation_session_artifacts "
"ADD COLUMN replay_byte_length INTEGER NOT NULL DEFAULT 0"
)
connection.commit()
with _ignore_os_error():
self.database_path.chmod(0o600)
def _upsert_legacy(self, candidate: LegacySessionCandidate) -> None:
_validate_identifier(candidate.session_id, "legacy session id")
def _upsert_candidate(
self,
source: ObservationArchiveSource,
candidate: ObservationSessionCandidate,
) -> None:
_validate_identifier(candidate.session_id, "observation session id")
_validate_identifier(source.plugin_id, "device plugin id")
_validate_identifier(source.archive_id, "observation archive id")
allowed_root = candidate.allowed_root.resolve()
session_root = candidate.session_root.resolve()
if not session_root.is_relative_to(allowed_root):
raise SessionIntegrityError("legacy session root escapes its allowed root")
sources = _legacy_sources(candidate)
artifacts = _legacy_artifacts(candidate, session_root)
raise SessionIntegrityError("observation session root escapes its allowed root")
if allowed_root != source.root.expanduser().resolve():
raise SessionIntegrityError("plugin candidate does not belong to its archive root")
sources = candidate.sources
artifacts = _validated_candidate_artifacts(candidate, session_root)
_validate_candidate_replay(candidate, artifacts)
now = utc_now_iso()
modalities_json = json.dumps(list(candidate.modalities), separators=(",", ":"))
with self._lock, self._connect() as connection:
connection.execute("BEGIN IMMEDIATE")
existing = connection.execute(
"SELECT origin, allowed_root, session_root, created_at_utc "
"SELECT plugin_id, archive_id, allowed_root, session_root, created_at_utc "
"FROM observation_sessions WHERE session_id = ?",
(candidate.session_id,),
).fetchone()
unclaimed_pre_plugin_row = existing is not None and (
existing["plugin_id"] == "" and existing["archive_id"] == ""
)
if existing is not None and (
existing["origin"] != "legacy-viewer-live"
or Path(existing["allowed_root"]).resolve() != allowed_root
Path(existing["allowed_root"]).resolve() != allowed_root
or Path(existing["session_root"]).resolve() != session_root
or (
not unclaimed_pre_plugin_row
and (
existing["plugin_id"] != source.plugin_id
or existing["archive_id"] != source.archive_id
)
)
):
connection.rollback()
raise SessionIntegrityError("session id is already bound to another origin")
raise SessionIntegrityError("session id is already bound to another archive")
created_at = existing["created_at_utc"] if existing is not None else now
connection.execute(
"INSERT INTO observation_sessions "
"(session_id, display_name, status, started_at_utc, completed_at_utc, "
"(session_id, plugin_id, archive_id, display_name, status, "
"started_at_utc, completed_at_utc, "
"duration_seconds, modalities_json, replayable, origin, source_count, "
"total_bytes, replay_raw_bytes, replay_metadata_bytes, allowed_root, "
"total_bytes, primary_replay_artifact_id, timeline_origin_epoch_ns, "
"timeline_origin_monotonic_ns, allowed_root, "
"session_root, created_at_utc, updated_at_utc) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
"ON CONFLICT(session_id) DO UPDATE SET "
"plugin_id = excluded.plugin_id, archive_id = excluded.archive_id, "
"display_name = excluded.display_name, status = excluded.status, "
"started_at_utc = excluded.started_at_utc, "
"completed_at_utc = excluded.completed_at_utc, "
@@ -446,11 +516,14 @@ class SessionStore:
"modalities_json = excluded.modalities_json, "
"replayable = excluded.replayable, source_count = excluded.source_count, "
"total_bytes = excluded.total_bytes, "
"replay_raw_bytes = excluded.replay_raw_bytes, "
"replay_metadata_bytes = excluded.replay_metadata_bytes, "
"primary_replay_artifact_id = excluded.primary_replay_artifact_id, "
"timeline_origin_epoch_ns = excluded.timeline_origin_epoch_ns, "
"timeline_origin_monotonic_ns = excluded.timeline_origin_monotonic_ns, "
"updated_at_utc = excluded.updated_at_utc",
(
candidate.session_id,
source.plugin_id,
source.archive_id,
candidate.display_name,
candidate.status,
candidate.started_at_utc,
@@ -458,11 +531,12 @@ class SessionStore:
candidate.duration_seconds,
modalities_json,
int(candidate.replayable),
"legacy-viewer-live",
source.archive_id,
len(sources),
candidate.total_bytes,
candidate.replay_raw_byte_length,
candidate.replay_metadata_byte_length,
candidate.primary_replay_artifact_id,
candidate.timeline_origin_epoch_ns,
candidate.timeline_origin_monotonic_ns,
str(allowed_root),
str(session_root),
created_at,
@@ -480,9 +554,20 @@ class SessionStore:
connection.executemany(
"INSERT INTO observation_session_artifacts "
"(session_id, artifact_id, kind, media_type, byte_length, sha256, "
"integrity_status, locator) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
"integrity_status, locator, replay_byte_length) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
[
(candidate.session_id, *artifact)
(
candidate.session_id,
artifact.artifact_id,
artifact.kind,
artifact.media_type,
artifact.byte_length,
artifact.sha256,
artifact.integrity_status,
str(artifact.locator),
artifact.replay_byte_length,
)
for artifact in artifacts
],
)
@@ -519,82 +604,56 @@ class SessionStore:
connection.close()
def _legacy_sources(candidate: LegacySessionCandidate) -> tuple[SessionSource, ...]:
rows: list[SessionSource] = []
if "point-cloud" in candidate.modalities:
rows.append(
SessionSource(
source_id="sensor.lidar.primary",
semantic_channel_id="spatial.point-cloud.recorded",
modality="point-cloud",
status="recorded",
seekable=candidate.replayable,
artifact_id="raw-mqtt",
)
)
if "trajectory" in candidate.modalities:
rows.append(
SessionSource(
source_id="spatial.trajectory",
semantic_channel_id="spatial.pose.recorded",
modality="trajectory",
status="recorded",
seekable=candidate.replayable,
artifact_id="raw-mqtt",
)
)
rows.extend(
SessionSource(
source_id=media.source_id,
semantic_channel_id="camera.video.recorded",
modality="video",
status="recorded",
seekable=True,
artifact_id=media.artifact_id,
)
for media in candidate.media_sources
)
if len({source.source_id for source in rows}) != len(rows):
raise SessionIntegrityError("legacy session contains duplicate source identifiers")
return tuple(rows)
def _legacy_artifacts(
candidate: LegacySessionCandidate,
def _validated_candidate_artifacts(
candidate: ObservationSessionCandidate,
session_root: Path,
) -> tuple[tuple[str, str, str, int, str | None, str, str], ...]:
artifacts: list[tuple[str, str, str, int, str | None, str, str]] = [
(
"raw-mqtt",
"raw-transport",
"application/x-nodedc-k1mqtt",
candidate.raw_byte_length,
candidate.raw_sha256,
candidate.raw_integrity_status,
str(candidate.raw_path),
)
]
for media in candidate.media_sources:
) -> tuple[ObservationArtifactCandidate, ...]:
artifacts = candidate.artifacts
artifact_ids: set[str] = set()
for artifact in artifacts:
_validate_identifier(artifact.artifact_id, "observation artifact id")
if artifact.artifact_id in artifact_ids:
raise SessionIntegrityError("observation session contains duplicate artifacts")
artifact_ids.add(artifact.artifact_id)
if artifact.byte_length < 0 or not 0 <= artifact.replay_byte_length <= artifact.byte_length:
raise SessionIntegrityError("observation artifact has invalid byte boundaries")
try:
locator = media.locator.resolve(strict=True)
locator = artifact.locator.resolve(strict=True)
except OSError as exc:
raise SessionIntegrityError("legacy video artifact is missing") from exc
if not locator.is_dir() or not locator.is_relative_to(session_root):
raise SessionIntegrityError("legacy video artifact escapes its session root")
artifacts.append(
(
media.artifact_id,
"recorded-video",
"video/mp4",
media.byte_length,
None,
"validated-structure",
str(locator),
)
)
if len({artifact[0] for artifact in artifacts}) != len(artifacts):
raise SessionIntegrityError("legacy session contains duplicate artifact identifiers")
return tuple(artifacts)
raise SessionIntegrityError("observation artifact is missing") from exc
if not locator.is_relative_to(session_root) or not (locator.is_file() or locator.is_dir()):
raise SessionIntegrityError("observation artifact escapes its session root")
if artifact.replay_byte_length > 0 and not locator.is_file():
raise SessionIntegrityError("replay input artifact must be a regular file")
source_ids: set[str] = set()
for source in candidate.sources:
_validate_identifier(source.source_id, "observation source id")
if source.source_id in source_ids:
raise SessionIntegrityError("observation session contains duplicate sources")
if source.artifact_id not in artifact_ids:
raise SessionIntegrityError("observation source references an unknown artifact")
source_ids.add(source.source_id)
return artifacts
def _validate_candidate_replay(
candidate: ObservationSessionCandidate,
artifacts: tuple[ObservationArtifactCandidate, ...],
) -> None:
primary_id = candidate.primary_replay_artifact_id
replay_artifacts = tuple(artifact for artifact in artifacts if artifact.replay_byte_length > 0)
if candidate.replayable:
if (
primary_id is None
or sum(artifact.artifact_id == primary_id for artifact in replay_artifacts) != 1
or candidate.timeline_origin_epoch_ns is None
or candidate.timeline_origin_monotonic_ns is None
):
raise SessionIntegrityError("replayable observation contract is incomplete")
if candidate.timeline_origin_epoch_ns < 0 or candidate.timeline_origin_monotonic_ns < 0:
raise SessionIntegrityError("observation timeline origin is invalid")
elif primary_id is not None or replay_artifacts:
raise SessionIntegrityError("non-replayable observation declares replay artifacts")
def _summary_from_row(row: sqlite3.Row) -> SessionSummary:
+7 -11
View File
@@ -1,15 +1,11 @@
"""Live/replay visualization bridge for verified K1 MQTT streams."""
"""Mission Core viewer consumers and recorded-viewer contracts."""
from k1link.viewer.messages import StreamMessage
from k1link.viewer.replay import (
ReplayFormatError,
detect_replay_format,
iter_replay_messages,
)
from .recorded import RecordedBlueprintError, recorded_blueprint_rrd
from .rerun_bridge import RerunBridge, RerunSceneSettings
__all__ = [
"ReplayFormatError",
"StreamMessage",
"detect_replay_format",
"iter_replay_messages",
"RecordedBlueprintError",
"RerunBridge",
"RerunSceneSettings",
"recorded_blueprint_rrd",
]
+124
View File
@@ -0,0 +1,124 @@
"""Vendor-neutral Rerun blueprint for prepared observation recordings."""
from __future__ import annotations
from contextlib import suppress
from uuid import UUID
import rerun as rr
from rerun import blueprint as rrb
from k1link.viewer.rerun_bridge import RerunSceneSettings, _parse_hex_color
APPLICATION_ID = "nodedc_mission_core_recorded"
SESSION_TIMELINE = "session_time"
RECORDED_SPATIAL_VIEW_ID = UUID("5f5f11d5-3b0a-4a81-887b-2be767cba1c0")
RECORDED_ROOT_CONTAINER_ID = UUID("b02f2aca-8471-4dcb-b786-53df5a320fc8")
RECORDED_POINTS_VISUALIZER_ID = UUID("ca037ec0-8761-4417-86ee-846fa2875303")
class RecordedBlueprintError(RuntimeError):
"""A viewer blueprint update could not be serialized safely."""
def recorded_blueprint(
settings: RerunSceneSettings,
*,
include_initial_playback_state: bool = True,
) -> rrb.Blueprint:
accumulation = max(0.0, settings.accumulation_seconds)
time_ranges: list[rr.VisibleTimeRange] | None = None
if accumulation > 0:
time_ranges = [
rr.VisibleTimeRange(
SESSION_TIMELINE,
start=rr.TimeRangeBoundary.cursor_relative(seconds=-accumulation),
end=rr.TimeRangeBoundary.cursor_relative(),
)
]
point_visualizer = rr.Points3D.from_fields(
radii=rr.Radius.ui_points(settings.point_size),
colors=(
[_parse_hex_color(settings.custom_color)]
if settings.palette == "custom"
else None
),
).visualizer()
point_visualizer.id = RECORDED_POINTS_VISUALIZER_ID
spatial_view = rrb.Spatial3DView(
origin="/world",
name="Пространственная сцена",
background=[7, 8, 10, 255],
line_grid=rrb.LineGrid3D(
visible=settings.show_grid,
color=[86, 91, 99, 110],
stroke_width=0.75,
),
overrides={
"/world/points": [
rrb.EntityBehavior(visible=settings.show_points),
point_visualizer,
],
"/world/trajectory": rrb.EntityBehavior(
visible=settings.show_trajectory,
),
},
time_ranges=time_ranges,
)
spatial_view.id = RECORDED_SPATIAL_VIEW_ID
root_container = rrb.Tabs(spatial_view)
root_container.id = RECORDED_ROOT_CONTAINER_ID
if include_initial_playback_state:
return rrb.Blueprint(
root_container,
rrb.TimePanel(
timeline=SESSION_TIMELINE,
play_state="paused",
state="hidden",
),
auto_layout=False,
auto_views=False,
collapse_panels=True,
)
return rrb.Blueprint(
root_container,
auto_layout=False,
auto_views=False,
collapse_panels=False,
)
def recorded_blueprint_rrd(
settings: RerunSceneSettings,
*,
application_id: str = APPLICATION_ID,
recording_id: str,
) -> bytes:
"""Serialize a bounded active blueprint update without recorded data."""
recording = rr.RecordingStream(
application_id,
recording_id=recording_id,
send_properties=False,
)
stream = rr.binary_stream(recording)
try:
recording.send_blueprint(
recorded_blueprint(
settings,
include_initial_playback_state=False,
),
make_active=True,
make_default=False,
)
payload = stream.read(flush=True, flush_timeout_sec=5.0)
except Exception as exc:
raise RecordedBlueprintError("failed to serialize recorded blueprint") from exc
finally:
with suppress(Exception):
recording.disconnect()
if not payload or not payload.startswith(b"RRF2") or len(payload) > 1_048_576:
raise RecordedBlueprintError("serialized recorded blueprint is invalid")
return payload
+9 -14
View File
@@ -22,10 +22,7 @@ from k1link.sessions import (
SessionRecordingMaterializer,
SessionRecordingPreparationManager,
SessionStore,
recover_stale_active_session_marker,
resolve_missioncore_evidence_dir,
)
from k1link.web.camera_archive import recover_incomplete_camera_archives
from k1link.web.device_plugin_composition import load_installed_device_plugins
from k1link.web.plugin_catalog import DevicePluginCatalog, PluginCatalogError
from k1link.web.plugin_runtime import (
@@ -42,13 +39,14 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
INVALID_REQUEST_DETAIL = "Некорректные параметры запроса."
legacy_observation_sessions_root = REPOSITORY_ROOT / "sessions"
observation_sessions_root = resolve_missioncore_evidence_dir(REPOSITORY_ROOT)
plugin_environment = load_installed_device_plugins(REPOSITORY_ROOT)
plugin_catalog: DevicePluginCatalog = plugin_environment.catalog
plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher
session_store = SessionStore(REPOSITORY_ROOT)
session_recording_materializer = SessionRecordingMaterializer(session_store.data_dir)
session_recording_materializer = SessionRecordingMaterializer(
session_store.data_dir,
exporters=plugin_environment.recording_exporters,
)
session_recorded_media_inspector = RecordedMediaInspector(
session_store.data_dir / "recorded-media-preparations"
)
@@ -76,8 +74,9 @@ def refresh_observation_catalog() -> tuple[str, ...]:
"""Discover completed or recoverable local evidence without copying payloads."""
imported = [
*session_store.import_legacy_viewer_live(legacy_observation_sessions_root),
*session_store.import_legacy_viewer_live(observation_sessions_root),
session_id
for archive in plugin_environment.observation_archives
for session_id in session_store.reconcile_archive(archive)
]
return tuple(dict.fromkeys(imported))
@@ -138,12 +137,8 @@ async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
# Recovery is intentionally a one-shot startup phase. The archive
# helper owns a cross-process lease, while ordinary catalog requests
# only perform discovery and therefore never touch a live writer.
await asyncio.to_thread(recover_stale_active_session_marker, observation_sessions_root)
for sessions_root in (
legacy_observation_sessions_root,
observation_sessions_root,
):
await asyncio.to_thread(recover_incomplete_camera_archives, sessions_root)
for archive in plugin_environment.observation_archives:
await asyncio.to_thread(archive.recover, archive.root)
# Full evidence discovery, hashing and RRD queue reconciliation can be
# expensive on field captures. Start it immediately in the background
# instead of holding the ASGI startup gate.
@@ -7,6 +7,7 @@ from typing import Any
from fastapi import APIRouter
from k1link.sessions.plugin_contract import ObservationArchiveSource, RecordingExporter
from k1link.web.plugin_catalog import DevicePluginCatalog, DevicePluginManifest
from k1link.web.plugin_runtime import (
DevicePluginDispatcher,
@@ -25,6 +26,23 @@ class InstalledDevicePluginEnvironment:
legacy_routers: tuple[APIRouter, ...]
_contributions: tuple[DevicePluginRuntimeContribution, ...]
@property
def observation_archives(self) -> tuple[ObservationArchiveSource, ...]:
return tuple(
archive
for contribution in self._contributions
if contribution.observation is not None
for archive in contribution.observation.archives
)
@property
def recording_exporters(self) -> dict[str, RecordingExporter]:
return {
contribution.adapter.plugin_id: contribution.observation.recording_exporter
for contribution in self._contributions
if contribution.observation is not None
}
def close(self) -> None:
cleanup_errors = _close_contributions(self._contributions)
if cleanup_errors:
@@ -103,6 +121,31 @@ def _load_contribution(
raise DevicePluginCompositionError(
f"Device-plugin manifest/runtime actions mismatch for {adapter.plugin_id}"
)
observation = contribution.observation
if observation is not None:
if not observation.archives:
raise DevicePluginCompositionError(
f"Device-plugin observation contribution is empty for {adapter.plugin_id}"
)
archive_ids: set[str] = set()
archive_roots: set[Path] = set()
for archive in observation.archives:
if archive.plugin_id != adapter.plugin_id:
raise DevicePluginCompositionError(
"Device-plugin observation/runtime id mismatch: "
f"{archive.plugin_id} != {adapter.plugin_id}"
)
if archive.archive_id in archive_ids:
raise DevicePluginCompositionError(
f"Duplicate observation archive id for {adapter.plugin_id}"
)
resolved_root = archive.root.expanduser().resolve()
if resolved_root in archive_roots:
raise DevicePluginCompositionError(
f"Duplicate observation archive root for {adapter.plugin_id}"
)
archive_ids.add(archive.archive_id)
archive_roots.add(resolved_root)
except Exception as exc:
_add_cleanup_notes(exc, _close_contributions((contribution,)))
raise
+37 -6
View File
@@ -2,11 +2,16 @@ from __future__ import annotations
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import Any, Protocol
from uuid import uuid4
from fastapi import APIRouter
from missioncore_plugin_sdk.v0alpha2 import RuntimeActionInvocation, RuntimeActionResult
from pydantic import BaseModel, ConfigDict, Field
from k1link.sessions.plugin_contract import ObservationRuntimeContribution
STATE_READ_ACTION_ID = "state.read"
@@ -20,7 +25,7 @@ class DevicePluginActionAdapter(Protocol):
plugin_id: str
action_ids: frozenset[str]
async def invoke(self, action_id: str, payload: Mapping[str, Any]) -> dict[str, Any]: ...
async def invoke(self, invocation: RuntimeActionInvocation) -> Mapping[str, Any]: ...
def _noop() -> None:
@@ -33,6 +38,7 @@ class DevicePluginRuntimeContribution:
adapter: DevicePluginActionAdapter
legacy_routers: tuple[APIRouter, ...] = ()
observation: ObservationRuntimeContribution | None = None
close: Callable[[], None] = _noop
@@ -71,8 +77,33 @@ class DevicePluginDispatcher:
adapter = self._adapters.get(plugin_id)
if adapter is None:
raise PluginNotFoundError(f"Device plugin is not installed: {plugin_id}")
if action_id not in adapter.action_ids:
raise PluginActionNotFoundError(
f"Device plugin {plugin_id} does not declare action {action_id}"
)
return await adapter.invoke(action_id, payload)
return await invoke_device_plugin_adapter(adapter, action_id, payload)
async def invoke_device_plugin_adapter(
adapter: DevicePluginActionAdapter,
action_id: str,
payload: Mapping[str, Any],
) -> dict[str, Any]:
"""Validate SDK request/result envelopes around one plugin action call."""
if action_id not in adapter.action_ids:
raise PluginActionNotFoundError(
f"Device plugin {adapter.plugin_id} does not declare action {action_id}"
)
invocation = RuntimeActionInvocation(
invocation_id=uuid4().hex,
plugin_id=adapter.plugin_id,
action_id=action_id,
requested_at=datetime.now(UTC),
parameters=dict(payload),
)
output = await adapter.invoke(invocation)
result = RuntimeActionResult(
invocation_id=invocation.invocation_id,
plugin_id=invocation.plugin_id,
action_id=invocation.action_id,
completed_at=datetime.now(UTC),
output=dict(output),
)
return dict(result.output)
+6 -8
View File
@@ -31,16 +31,15 @@ from k1link.sessions import (
SessionStore,
validate_recorded_media_timeline,
)
from k1link.viewer.rerun_bridge import RerunSceneSettings
from k1link.viewer.rrd_export import (
from k1link.viewer.recorded import (
APPLICATION_ID as RECORDED_APPLICATION_ID,
)
from k1link.viewer.rrd_export import (
RrdExportError,
from k1link.viewer.recorded import (
RecordedBlueprintError,
recorded_blueprint_rrd,
)
from k1link.viewer.rerun_bridge import RerunSceneSettings
DEFAULT_REPLAY_PLUGIN_ID = "nodedc.device.xgrids-lixelkity-k1"
DEFAULT_REPLAY_ACTION_ID = "stream.start-replay"
SAFE_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$")
SAFE_SHA256 = re.compile(r"^[a-f0-9]{64}$")
@@ -234,7 +233,6 @@ def build_session_router(
recording_preparation_manager: SessionRecordingPreparationManager | None = None,
media_inspector: RecordedMediaInspector | None = None,
allow_synchronous_recording_fallback: bool = False,
replay_plugin_id: str = DEFAULT_REPLAY_PLUGIN_ID,
replay_action_id: str = DEFAULT_REPLAY_ACTION_ID,
) -> APIRouter:
"""Build generic host APIs without exposing filesystem locators to clients."""
@@ -388,7 +386,7 @@ def build_session_router(
"schema_version": "missioncore.observation-session-replay/v1",
"launch": {
"kind": "plugin-action",
"plugin_id": replay_plugin_id,
"plugin_id": command.plugin_id,
"action_id": replay_action_id,
"session_id": command.session_id,
"speed": command.speed,
@@ -671,7 +669,7 @@ def build_session_router(
raise HTTPException(status_code=404, detail=str(exc)) from exc
except (SessionNotReplayableError, SessionIntegrityError) as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
except RrdExportError as exc:
except RecordedBlueprintError as exc:
raise HTTPException(
status_code=500,
detail="Не удалось подготовить настройки визуализатора.",