feat(plugins): isolate device integrations
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Concrete device integrations loaded only through reviewed plugin manifests."""
|
||||
@@ -0,0 +1,5 @@
|
||||
"""XGRIDS/LixelKity K1 compatibility plugin implementation."""
|
||||
|
||||
from .observation import build_xgrids_k1_observation, xgrids_k1_archive_source
|
||||
|
||||
__all__ = ["build_xgrids_k1_observation", "xgrids_k1_archive_source"]
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
"""Bounded, offline analysis of sensitive K1 evidence artifacts."""
|
||||
|
||||
from k1link.analyze.stream_summary import (
|
||||
from k1link.device_plugins.xgrids_k1.analyze.stream_summary import (
|
||||
DEFAULT_STREAM_SUMMARY_MAX_PAYLOAD_BYTES,
|
||||
MAX_STREAM_SUMMARY_PAYLOAD_BYTES,
|
||||
StreamSummary,
|
||||
+2
-2
@@ -9,8 +9,8 @@ from pathlib import Path
|
||||
from typing import TypedDict
|
||||
|
||||
from k1link.artifacts import utc_now_iso
|
||||
from k1link.mqtt import DEFAULT_MAX_MESSAGE_BYTES, iter_capture_frames
|
||||
from k1link.protocol import (
|
||||
from k1link.device_plugins.xgrids_k1.mqtt import DEFAULT_MAX_MESSAGE_BYTES, iter_capture_frames
|
||||
from k1link.device_plugins.xgrids_k1.protocol import (
|
||||
DecodeLimits,
|
||||
StreamDecodeError,
|
||||
decode_lio_pcl,
|
||||
@@ -13,7 +13,7 @@ from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import IO, Any
|
||||
|
||||
from k1link.mqtt.capture import (
|
||||
from k1link.device_plugins.xgrids_k1.mqtt.capture import (
|
||||
FRAME_HEADER,
|
||||
GROUP_COMMIT_MAX_BYTES,
|
||||
GROUP_COMMIT_MAX_MESSAGES,
|
||||
@@ -21,10 +21,8 @@ from k1link.mqtt.capture import (
|
||||
MAX_TOPIC_BYTES,
|
||||
RAW_MAGIC,
|
||||
)
|
||||
|
||||
from .models import (
|
||||
from k1link.sessions.models import (
|
||||
LegacyMediaSourceCandidate,
|
||||
LegacySessionCandidate,
|
||||
SessionModality,
|
||||
SessionStatus,
|
||||
)
|
||||
@@ -41,6 +39,30 @@ MAX_RECOVERY_METADATA_LINE_BYTES = 64 * 1024
|
||||
MAX_RECOVERY_MESSAGES = 500_000
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LegacySessionCandidate:
|
||||
"""K1 archive discovery result retained inside the compatibility adapter."""
|
||||
|
||||
session_id: str
|
||||
display_name: str
|
||||
status: SessionStatus
|
||||
started_at_utc: str | None
|
||||
completed_at_utc: str | None
|
||||
duration_seconds: float | None
|
||||
modalities: tuple[SessionModality, ...]
|
||||
replayable: bool
|
||||
total_bytes: int
|
||||
allowed_root: Path
|
||||
session_root: Path
|
||||
raw_path: Path
|
||||
raw_byte_length: int
|
||||
replay_raw_byte_length: int
|
||||
replay_metadata_byte_length: int
|
||||
raw_sha256: str | None
|
||||
raw_integrity_status: str
|
||||
media_sources: tuple[LegacyMediaSourceCandidate, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _RecoveredCapture:
|
||||
message_count: int
|
||||
@@ -14,7 +14,7 @@ from typing import IO, Any, Final, Literal
|
||||
|
||||
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
|
||||
|
||||
from k1link.mqtt import validate_private_ipv4
|
||||
from k1link.device_plugins.xgrids_k1.mqtt import validate_private_ipv4
|
||||
from k1link.web.camera_archive import (
|
||||
CameraArchiveError,
|
||||
CameraArchiveKind,
|
||||
@@ -16,28 +16,28 @@ from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from k1link import __version__
|
||||
from k1link.analyze import (
|
||||
from k1link.artifacts import write_json_atomic
|
||||
from k1link.device_plugins.xgrids_k1.analyze import (
|
||||
DEFAULT_STREAM_SUMMARY_MAX_PAYLOAD_BYTES,
|
||||
MAX_STREAM_SUMMARY_PAYLOAD_BYTES,
|
||||
summarize_mqtt_streams,
|
||||
)
|
||||
from k1link.artifacts import write_json_atomic
|
||||
from k1link.ble.gatt import dump_metadata
|
||||
from k1link.ble.scanner import scan
|
||||
from k1link.ble.wifi_provisioning import (
|
||||
from k1link.device_plugins.xgrids_k1.ble.gatt import dump_metadata
|
||||
from k1link.device_plugins.xgrids_k1.ble.scanner import scan
|
||||
from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import (
|
||||
PROFILE_ID,
|
||||
WriteMode,
|
||||
provision_wifi_once,
|
||||
)
|
||||
from k1link.macos_credentials import CredentialDialogError, prompt_wifi_credentials
|
||||
from k1link.mqtt import (
|
||||
from k1link.device_plugins.xgrids_k1.mqtt import (
|
||||
DEFAULT_MAX_MESSAGE_BYTES,
|
||||
MAX_CONFIGURABLE_MESSAGE_BYTES,
|
||||
CaptureError,
|
||||
capture_mqtt,
|
||||
)
|
||||
from k1link.net.snapshot import snapshot
|
||||
from k1link.usb.snapshot import snapshot as usb_snapshot
|
||||
from k1link.device_plugins.xgrids_k1.net.snapshot import snapshot
|
||||
from k1link.device_plugins.xgrids_k1.usb.snapshot import snapshot as usb_snapshot
|
||||
from k1link.macos_credentials import CredentialDialogError, prompt_wifi_credentials
|
||||
|
||||
app = typer.Typer(
|
||||
name="k1link",
|
||||
+30
-16
@@ -15,16 +15,31 @@ from pathlib import Path
|
||||
from typing import Any, Literal, Protocol, cast
|
||||
|
||||
from bleak.exc import BleakError
|
||||
from missioncore_plugin_sdk.v0alpha2 import RuntimeActionInvocation
|
||||
from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
|
||||
|
||||
from k1link.artifacts import write_json_atomic
|
||||
from k1link.ble.scanner import scan
|
||||
from k1link.ble.wifi_provisioning import AP_FALLBACK_IPV4, provision_wifi_once
|
||||
from k1link.mqtt import validate_private_ipv4
|
||||
from k1link.protocol.normalizer import normalize_k1_message
|
||||
from k1link.device_plugins.xgrids_k1.ble.scanner import scan
|
||||
from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import (
|
||||
AP_FALLBACK_IPV4,
|
||||
provision_wifi_once,
|
||||
)
|
||||
from k1link.device_plugins.xgrids_k1.camera import (
|
||||
CAMERA_EXCLUSIVE_GROUP,
|
||||
CAMERA_SOURCE_LABELS,
|
||||
CAMERA_SOURCE_PATHS,
|
||||
CameraSourceId,
|
||||
XgridsK1CameraGateway,
|
||||
build_xgrids_k1_camera_router,
|
||||
)
|
||||
from k1link.device_plugins.xgrids_k1.mqtt import validate_private_ipv4
|
||||
from k1link.device_plugins.xgrids_k1.protocol.normalizer import normalize_k1_message
|
||||
from k1link.device_plugins.xgrids_k1.viewer.runtime import (
|
||||
VisualizationRuntime,
|
||||
new_live_session_dir,
|
||||
)
|
||||
from k1link.sessions import ActiveSessionLease, resolve_missioncore_evidence_dir
|
||||
from k1link.viewer.rerun_bridge import RerunSceneSettings
|
||||
from k1link.viewer.runtime import VisualizationRuntime, new_live_session_dir
|
||||
from k1link.web.device_lifecycle import (
|
||||
TERMINAL_ACQUISITION_STATES,
|
||||
AcquisitionRecord,
|
||||
@@ -38,14 +53,6 @@ from k1link.web.plugin_runtime import (
|
||||
PluginActionNotFoundError,
|
||||
PluginExecutionError,
|
||||
)
|
||||
from k1link.web.xgrids_k1_camera import (
|
||||
CAMERA_EXCLUSIVE_GROUP,
|
||||
CAMERA_SOURCE_LABELS,
|
||||
CAMERA_SOURCE_PATHS,
|
||||
CameraSourceId,
|
||||
XgridsK1CameraGateway,
|
||||
build_xgrids_k1_camera_router,
|
||||
)
|
||||
|
||||
XGRIDS_K1_PLUGIN_ID = "nodedc.device.xgrids-lixelkity-k1"
|
||||
XGRIDS_K1_MODEL_ID = "xgrids.lixelkity-k1"
|
||||
@@ -1497,9 +1504,14 @@ class XgridsK1PluginFacade:
|
||||
def __init__(self, service: XgridsK1ServicePort) -> None:
|
||||
self.service = service
|
||||
|
||||
async def invoke(self, action_id: str, payload: Mapping[str, Any]) -> dict[str, Any]:
|
||||
async def invoke(self, invocation: RuntimeActionInvocation) -> dict[str, Any]:
|
||||
if invocation.plugin_id != self.plugin_id:
|
||||
raise PluginActionNotFoundError("runtime invocation targets another plugin")
|
||||
try:
|
||||
return await self._invoke_validated(action_id, payload)
|
||||
return await self._invoke_validated(
|
||||
invocation.action_id,
|
||||
invocation.parameters,
|
||||
)
|
||||
except (ValidationError, ValueError, PluginActionNotFoundError):
|
||||
raise
|
||||
except (BleakError, OSError, TimeoutError, RuntimeError) as exc:
|
||||
@@ -1803,7 +1815,8 @@ def _validate_installed_compatibility_profile(repository_root: Path) -> None:
|
||||
def build_xgrids_k1_plugin(repository_root: Path) -> DevicePluginRuntimeContribution:
|
||||
"""Manifest entrypoint for the reviewed XGRIDS compatibility adapter."""
|
||||
|
||||
from k1link.web.xgrids_k1_legacy_api import build_xgrids_k1_legacy_router
|
||||
from k1link.device_plugins.xgrids_k1 import build_xgrids_k1_observation
|
||||
from k1link.device_plugins.xgrids_k1.legacy_api import build_xgrids_k1_legacy_router
|
||||
|
||||
_validate_installed_compatibility_profile(repository_root)
|
||||
service = XgridsK1CompatibilityService(repository_root)
|
||||
@@ -1817,5 +1830,6 @@ def build_xgrids_k1_plugin(repository_root: Path) -> DevicePluginRuntimeContribu
|
||||
XGRIDS_K1_PLUGIN_ID,
|
||||
),
|
||||
),
|
||||
observation=build_xgrids_k1_observation(repository_root),
|
||||
close=service.close,
|
||||
)
|
||||
+38
-10
@@ -5,8 +5,7 @@ from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, WebSocket, WebSocketDisconnect
|
||||
|
||||
from k1link.web.plugin_runtime import PluginExecutionError
|
||||
from k1link.web.xgrids_k1_facade import (
|
||||
from k1link.device_plugins.xgrids_k1.facade import (
|
||||
ACTION_DISCOVERY_SCAN,
|
||||
ACTION_NETWORK_PROVISION,
|
||||
ACTION_STATE_READ,
|
||||
@@ -21,6 +20,7 @@ from k1link.web.xgrids_k1_facade import (
|
||||
ViewerSettingsRequest,
|
||||
XgridsK1PluginFacade,
|
||||
)
|
||||
from k1link.web.plugin_runtime import PluginExecutionError, invoke_device_plugin_adapter
|
||||
|
||||
|
||||
def build_xgrids_k1_legacy_router(adapter: XgridsK1PluginFacade) -> APIRouter:
|
||||
@@ -30,19 +30,27 @@ def build_xgrids_k1_legacy_router(adapter: XgridsK1PluginFacade) -> APIRouter:
|
||||
|
||||
@router.get("/api/state", deprecated=True)
|
||||
async def get_state() -> dict[str, Any]:
|
||||
return await adapter.invoke(ACTION_STATE_READ, {})
|
||||
return await invoke_device_plugin_adapter(adapter, ACTION_STATE_READ, {})
|
||||
|
||||
@router.post("/api/ble/scan", deprecated=True)
|
||||
async def scan_ble(request: BleScanRequest) -> dict[str, Any]:
|
||||
try:
|
||||
return await adapter.invoke(ACTION_DISCOVERY_SCAN, request.model_dump())
|
||||
return await invoke_device_plugin_adapter(
|
||||
adapter,
|
||||
ACTION_DISCOVERY_SCAN,
|
||||
request.model_dump(),
|
||||
)
|
||||
except (PluginExecutionError, ValueError) as exc:
|
||||
raise HTTPException(status_code=502, detail=f"Ошибка поиска BLE: {exc}") from exc
|
||||
|
||||
@router.post("/api/connect", deprecated=True)
|
||||
async def connect(request: ConnectRequest) -> dict[str, Any]:
|
||||
try:
|
||||
return await adapter.invoke(ACTION_NETWORK_PROVISION, request.model_dump())
|
||||
return await invoke_device_plugin_adapter(
|
||||
adapter,
|
||||
ACTION_NETWORK_PROVISION,
|
||||
request.model_dump(),
|
||||
)
|
||||
except (PluginExecutionError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
@@ -52,31 +60,51 @@ def build_xgrids_k1_legacy_router(adapter: XgridsK1PluginFacade) -> APIRouter:
|
||||
@router.post("/api/session/live", deprecated=True)
|
||||
async def start_live(request: LiveRequest) -> dict[str, Any]:
|
||||
try:
|
||||
return await adapter.invoke(ACTION_STREAM_START_LIVE, request.model_dump())
|
||||
return await invoke_device_plugin_adapter(
|
||||
adapter,
|
||||
ACTION_STREAM_START_LIVE,
|
||||
request.model_dump(),
|
||||
)
|
||||
except (PluginExecutionError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@router.post("/api/session/replay", deprecated=True)
|
||||
async def start_replay(request: ReplayRequest) -> dict[str, Any]:
|
||||
try:
|
||||
return await adapter.invoke(ACTION_STREAM_START_REPLAY, request.model_dump())
|
||||
return await invoke_device_plugin_adapter(
|
||||
adapter,
|
||||
ACTION_STREAM_START_REPLAY,
|
||||
request.model_dump(),
|
||||
)
|
||||
except (PluginExecutionError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@router.post("/api/session/stop", deprecated=True)
|
||||
async def stop_session() -> dict[str, Any]:
|
||||
return await adapter.invoke(ACTION_STREAM_STOP, {})
|
||||
return await invoke_device_plugin_adapter(adapter, ACTION_STREAM_STOP, {})
|
||||
|
||||
@router.post("/api/viewer/settings", deprecated=True)
|
||||
async def update_viewer_settings(request: ViewerSettingsRequest) -> dict[str, Any]:
|
||||
return await adapter.invoke(ACTION_VIEWER_SETTINGS_UPDATE, request.model_dump())
|
||||
return await invoke_device_plugin_adapter(
|
||||
adapter,
|
||||
ACTION_VIEWER_SETTINGS_UPDATE,
|
||||
request.model_dump(),
|
||||
)
|
||||
|
||||
@router.websocket("/api/events")
|
||||
async def events(websocket: WebSocket) -> None:
|
||||
await websocket.accept()
|
||||
try:
|
||||
while True:
|
||||
await websocket.send_json({"state": await adapter.invoke(ACTION_STATE_READ, {})})
|
||||
await websocket.send_json(
|
||||
{
|
||||
"state": await invoke_device_plugin_adapter(
|
||||
adapter,
|
||||
ACTION_STATE_READ,
|
||||
{},
|
||||
)
|
||||
}
|
||||
)
|
||||
await asyncio.sleep(0.5)
|
||||
except WebSocketDisconnect:
|
||||
return
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Read-only MQTT evidence capture for an owner-controlled K1."""
|
||||
|
||||
from k1link.mqtt.capture import (
|
||||
from k1link.device_plugins.xgrids_k1.mqtt.capture import (
|
||||
DEFAULT_MAX_MESSAGE_BYTES,
|
||||
MAX_CONFIGURABLE_MESSAGE_BYTES,
|
||||
REPORT_TOPICS,
|
||||
@@ -0,0 +1,260 @@
|
||||
"""K1-native observation archive and Rerun preparation adapter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.device_plugins.xgrids_k1.archive import (
|
||||
LegacySessionCandidate,
|
||||
discover_legacy_viewer_sessions,
|
||||
)
|
||||
from k1link.device_plugins.xgrids_k1.rrd_export import (
|
||||
RrdExportCancelled,
|
||||
RrdExportError,
|
||||
export_k1mqtt_to_rrd,
|
||||
)
|
||||
from k1link.sessions.active import recover_stale_active_session_marker
|
||||
from k1link.sessions.models import (
|
||||
ObservationArtifactCandidate,
|
||||
ObservationSessionCandidate,
|
||||
SessionIntegrityError,
|
||||
SessionSource,
|
||||
)
|
||||
from k1link.sessions.plugin_contract import (
|
||||
ObservationArchiveSource,
|
||||
ObservationRuntimeContribution,
|
||||
PluginRecordingExportCancelled,
|
||||
PluginRecordingExportError,
|
||||
)
|
||||
from k1link.sessions.store import resolve_missioncore_evidence_dir
|
||||
from k1link.web.camera_archive import recover_incomplete_camera_archives
|
||||
|
||||
XGRIDS_K1_PLUGIN_ID = "nodedc.device.xgrids-lixelkity-k1"
|
||||
MAX_TIMELINE_ORIGIN_LINE_BYTES = 64 * 1024
|
||||
|
||||
|
||||
def build_xgrids_k1_observation(repository_root: Path) -> ObservationRuntimeContribution:
|
||||
"""Compose every K1 evidence root behind the generic observation ABI."""
|
||||
|
||||
roots = (
|
||||
("xgrids-k1.viewer-live.repository", repository_root.resolve() / "sessions"),
|
||||
(
|
||||
"xgrids-k1.viewer-live.evidence",
|
||||
resolve_missioncore_evidence_dir(repository_root),
|
||||
),
|
||||
)
|
||||
return ObservationRuntimeContribution(
|
||||
archives=tuple(
|
||||
ObservationArchiveSource(
|
||||
plugin_id=XGRIDS_K1_PLUGIN_ID,
|
||||
archive_id=archive_id,
|
||||
root=root,
|
||||
discover=_discover_archive,
|
||||
recover=_recover_archive,
|
||||
)
|
||||
for archive_id, root in roots
|
||||
),
|
||||
recording_exporter=_export_recording,
|
||||
)
|
||||
|
||||
|
||||
def xgrids_k1_archive_source(
|
||||
root: Path,
|
||||
*,
|
||||
archive_id: str = "xgrids-k1.viewer-live",
|
||||
) -> ObservationArchiveSource:
|
||||
"""Build one explicit archive source for tests, tools, or migration jobs."""
|
||||
|
||||
return ObservationArchiveSource(
|
||||
plugin_id=XGRIDS_K1_PLUGIN_ID,
|
||||
archive_id=archive_id,
|
||||
root=root,
|
||||
discover=_discover_archive,
|
||||
recover=_recover_archive,
|
||||
)
|
||||
|
||||
|
||||
def _recover_archive(root: Path) -> None:
|
||||
recover_stale_active_session_marker(root)
|
||||
recover_incomplete_camera_archives(root)
|
||||
|
||||
|
||||
def _discover_archive(root: Path) -> tuple[ObservationSessionCandidate, ...]:
|
||||
return tuple(
|
||||
_to_host_candidate(candidate) for candidate in discover_legacy_viewer_sessions(root)
|
||||
)
|
||||
|
||||
|
||||
def _to_host_candidate(candidate: LegacySessionCandidate) -> ObservationSessionCandidate:
|
||||
session_id = candidate.session_id
|
||||
raw_path = candidate.raw_path
|
||||
raw_bytes = candidate.raw_byte_length
|
||||
replay_raw_bytes = candidate.replay_raw_byte_length
|
||||
replay_metadata_bytes = candidate.replay_metadata_byte_length
|
||||
replayable = candidate.replayable
|
||||
metadata_path = raw_path.with_name("mqtt.metadata.jsonl")
|
||||
|
||||
artifacts = [
|
||||
ObservationArtifactCandidate(
|
||||
artifact_id="raw-transport-primary",
|
||||
kind="raw-transport",
|
||||
media_type="application/x-nodedc-k1mqtt",
|
||||
locator=raw_path,
|
||||
byte_length=raw_bytes,
|
||||
replay_byte_length=replay_raw_bytes if replayable else 0,
|
||||
sha256=candidate.raw_sha256,
|
||||
integrity_status=candidate.raw_integrity_status,
|
||||
)
|
||||
]
|
||||
if replay_metadata_bytes > 0:
|
||||
artifacts.append(
|
||||
ObservationArtifactCandidate(
|
||||
artifact_id="raw-transport-index",
|
||||
kind="raw-transport-index",
|
||||
media_type="application/x-ndjson",
|
||||
locator=metadata_path,
|
||||
byte_length=_regular_file_size(metadata_path),
|
||||
replay_byte_length=replay_metadata_bytes if replayable else 0,
|
||||
sha256=None,
|
||||
integrity_status=candidate.raw_integrity_status,
|
||||
)
|
||||
)
|
||||
|
||||
media_sources = candidate.media_sources
|
||||
artifacts.extend(
|
||||
ObservationArtifactCandidate(
|
||||
artifact_id=media.artifact_id,
|
||||
kind="recorded-video",
|
||||
media_type="video/mp4",
|
||||
locator=media.locator,
|
||||
byte_length=media.byte_length,
|
||||
replay_byte_length=0,
|
||||
sha256=None,
|
||||
integrity_status="validated-structure",
|
||||
)
|
||||
for media in media_sources
|
||||
)
|
||||
|
||||
sources: list[SessionSource] = []
|
||||
modalities = candidate.modalities
|
||||
if "point-cloud" in modalities:
|
||||
sources.append(
|
||||
SessionSource(
|
||||
source_id="sensor.lidar.primary",
|
||||
semantic_channel_id="spatial.point-cloud.recorded",
|
||||
modality="point-cloud",
|
||||
status="recorded",
|
||||
seekable=replayable,
|
||||
artifact_id="raw-transport-primary",
|
||||
)
|
||||
)
|
||||
if "trajectory" in modalities:
|
||||
sources.append(
|
||||
SessionSource(
|
||||
source_id="spatial.trajectory",
|
||||
semantic_channel_id="spatial.pose.recorded",
|
||||
modality="trajectory",
|
||||
status="recorded",
|
||||
seekable=replayable,
|
||||
artifact_id="raw-transport-primary",
|
||||
)
|
||||
)
|
||||
sources.extend(
|
||||
SessionSource(
|
||||
source_id=media.source_id,
|
||||
semantic_channel_id="camera.video.recorded",
|
||||
modality="video",
|
||||
status="recorded",
|
||||
seekable=True,
|
||||
artifact_id=media.artifact_id,
|
||||
)
|
||||
for media in media_sources
|
||||
)
|
||||
|
||||
timeline_origin = _timeline_origin(metadata_path) if replayable else None
|
||||
return ObservationSessionCandidate(
|
||||
session_id=session_id,
|
||||
display_name=candidate.display_name,
|
||||
status=candidate.status,
|
||||
started_at_utc=candidate.started_at_utc,
|
||||
completed_at_utc=candidate.completed_at_utc,
|
||||
duration_seconds=candidate.duration_seconds,
|
||||
modalities=modalities,
|
||||
replayable=replayable,
|
||||
total_bytes=candidate.total_bytes,
|
||||
allowed_root=candidate.allowed_root,
|
||||
session_root=candidate.session_root,
|
||||
primary_replay_artifact_id="raw-transport-primary" if replayable else None,
|
||||
timeline_origin_epoch_ns=None if timeline_origin is None else timeline_origin[0],
|
||||
timeline_origin_monotonic_ns=None if timeline_origin is None else timeline_origin[1],
|
||||
sources=tuple(sources),
|
||||
artifacts=tuple(artifacts),
|
||||
)
|
||||
|
||||
|
||||
def _regular_file_size(path: Path) -> int:
|
||||
try:
|
||||
value = path.lstat()
|
||||
except OSError as exc:
|
||||
raise SessionIntegrityError("K1 transport index is unavailable") from exc
|
||||
if not stat.S_ISREG(value.st_mode) or stat.S_ISLNK(value.st_mode):
|
||||
raise SessionIntegrityError("K1 transport index is not a regular file")
|
||||
return value.st_size
|
||||
|
||||
|
||||
def _timeline_origin(path: Path) -> tuple[int, int]:
|
||||
flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
|
||||
try:
|
||||
descriptor = os.open(path, flags)
|
||||
try:
|
||||
payload = os.read(descriptor, MAX_TIMELINE_ORIGIN_LINE_BYTES + 1)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
first_line = payload.splitlines(keepends=True)[0]
|
||||
if len(first_line) > MAX_TIMELINE_ORIGIN_LINE_BYTES or not first_line.endswith(
|
||||
(b"\n", b"\r")
|
||||
):
|
||||
raise ValueError
|
||||
document = json.loads(first_line)
|
||||
epoch_ns = document.get("received_at_epoch_ns")
|
||||
monotonic_ns = document.get("received_monotonic_ns")
|
||||
if (
|
||||
document.get("record_type") != "message"
|
||||
or document.get("sequence") != 1
|
||||
or not _non_negative_int(epoch_ns)
|
||||
or not _non_negative_int(monotonic_ns)
|
||||
):
|
||||
raise ValueError
|
||||
return int(epoch_ns), int(monotonic_ns)
|
||||
except (IndexError, OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
|
||||
raise SessionIntegrityError("K1 transport timeline origin is invalid") from exc
|
||||
|
||||
|
||||
def _non_negative_int(value: object) -> bool:
|
||||
return isinstance(value, int) and not isinstance(value, bool) and value >= 0
|
||||
|
||||
|
||||
def _export_recording(
|
||||
source: Path,
|
||||
destination: Path,
|
||||
*,
|
||||
cancel_event: threading.Event | None = None,
|
||||
activity_callback: object | None = None,
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
return dict(
|
||||
export_k1mqtt_to_rrd(
|
||||
source,
|
||||
destination,
|
||||
cancel_event=cancel_event,
|
||||
activity_callback=activity_callback if callable(activity_callback) else None,
|
||||
)
|
||||
)
|
||||
except RrdExportCancelled as exc:
|
||||
raise PluginRecordingExportCancelled("K1 recording export was cancelled") from exc
|
||||
except RrdExportError as exc:
|
||||
raise PluginRecordingExportError("K1 recording export failed") from exc
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
"""Verified protocol decoders for captured K1 application streams."""
|
||||
|
||||
from k1link.protocol.streams import (
|
||||
from k1link.device_plugins.xgrids_k1.protocol.streams import (
|
||||
DecodeLimits,
|
||||
LegacyPoint,
|
||||
LegacyPointCloudFrame,
|
||||
+1
-1
@@ -9,7 +9,7 @@ from k1link.data_plane import (
|
||||
DecodedPoseView,
|
||||
NormalizationError,
|
||||
)
|
||||
from k1link.protocol.streams import (
|
||||
from k1link.device_plugins.xgrids_k1.protocol.streams import (
|
||||
LegacyPointCloudFrame,
|
||||
LegacyPoseFrame,
|
||||
LioPointCloudFrame,
|
||||
+1
-1
@@ -7,7 +7,7 @@ from typing import NamedTuple
|
||||
|
||||
import lz4.block
|
||||
|
||||
from k1link.protocol.protobuf_wire import (
|
||||
from k1link.device_plugins.xgrids_k1.protocol.protobuf_wire import (
|
||||
ProtobufWireError,
|
||||
ProtoField,
|
||||
decode_zigzag64,
|
||||
@@ -16,8 +16,12 @@ import rerun as rr
|
||||
from rerun import blueprint as rrb
|
||||
|
||||
from k1link.data_plane import DecodedPointCloudView, DecodedPoseView, NormalizationError
|
||||
from k1link.protocol.normalizer import normalize_k1_message
|
||||
from k1link.viewer.replay import ReplayFormatError, detect_replay_format, iter_replay_messages
|
||||
from k1link.device_plugins.xgrids_k1.protocol.normalizer import normalize_k1_message
|
||||
from k1link.device_plugins.xgrids_k1.viewer.replay import (
|
||||
ReplayFormatError,
|
||||
detect_replay_format,
|
||||
iter_replay_messages,
|
||||
)
|
||||
from k1link.viewer.rerun_bridge import (
|
||||
MAX_TRAJECTORY_POSES,
|
||||
TRAJECTORY_APPEND_INTERVAL_NS,
|
||||
@@ -0,0 +1,11 @@
|
||||
"""K1-native capture, replay, and compatibility visualization runtime."""
|
||||
|
||||
from .messages import StreamMessage
|
||||
from .replay import ReplayFormatError, detect_replay_format, iter_replay_messages
|
||||
|
||||
__all__ = [
|
||||
"ReplayFormatError",
|
||||
"StreamMessage",
|
||||
"detect_replay_format",
|
||||
"iter_replay_messages",
|
||||
]
|
||||
+2
-2
@@ -29,7 +29,7 @@ from foxglove.messages import (
|
||||
Vector3,
|
||||
)
|
||||
|
||||
from k1link.protocol.streams import (
|
||||
from k1link.device_plugins.xgrids_k1.protocol.streams import (
|
||||
LegacyPointCloudFrame,
|
||||
LegacyPoseFrame,
|
||||
LioPointCloudFrame,
|
||||
@@ -40,7 +40,7 @@ from k1link.protocol.streams import (
|
||||
decode_lio_pcl,
|
||||
decode_lio_pose,
|
||||
)
|
||||
from k1link.viewer.messages import StreamMessage
|
||||
from k1link.device_plugins.xgrids_k1.viewer.messages import StreamMessage
|
||||
from k1link.viewer.metrics import BridgeMetrics
|
||||
|
||||
POINT_STRUCT = struct.Struct("<fffB3x")
|
||||
@@ -7,9 +7,9 @@ from decimal import Decimal, InvalidOperation
|
||||
from pathlib import Path
|
||||
from typing import IO, Literal
|
||||
|
||||
from k1link.mqtt import CaptureFormatError, iter_capture_frames
|
||||
from k1link.mqtt.capture import RAW_MAGIC
|
||||
from k1link.viewer.messages import StreamMessage
|
||||
from k1link.device_plugins.xgrids_k1.mqtt import CaptureFormatError, iter_capture_frames
|
||||
from k1link.device_plugins.xgrids_k1.mqtt.capture import RAW_MAGIC
|
||||
from k1link.device_plugins.xgrids_k1.viewer.messages import StreamMessage
|
||||
|
||||
MAX_REPLAY_PAYLOAD_BYTES = 2 * 1024 * 1024
|
||||
MAX_LEGACY_LINE_BYTES = MAX_REPLAY_PAYLOAD_BYTES * 2 + 64 * 1024
|
||||
+3
-3
@@ -11,10 +11,10 @@ from typing import Literal, Protocol, TypedDict
|
||||
|
||||
from k1link.artifacts import utc_now_iso, write_json_atomic
|
||||
from k1link.data_plane import DecodedDataPlaneView, NormalizationError
|
||||
from k1link.mqtt import CapturedMqttMessage, CaptureError, capture_mqtt
|
||||
from k1link.viewer.messages import StreamMessage
|
||||
from k1link.device_plugins.xgrids_k1.mqtt import CapturedMqttMessage, CaptureError, capture_mqtt
|
||||
from k1link.device_plugins.xgrids_k1.viewer.messages import StreamMessage
|
||||
from k1link.device_plugins.xgrids_k1.viewer.replay import iter_replay_messages
|
||||
from k1link.viewer.metrics import BridgeMetrics, MetricsSnapshot
|
||||
from k1link.viewer.replay import iter_replay_messages
|
||||
from k1link.viewer.rerun_bridge import DEFAULT_GRPC_PORT, RerunBridge, RerunSceneSettings
|
||||
|
||||
RuntimePhase = Literal[
|
||||
@@ -14,12 +14,22 @@ from .media import (
|
||||
)
|
||||
from .models import (
|
||||
LayoutConflictError,
|
||||
ObservationArtifactCandidate,
|
||||
ObservationSessionCandidate,
|
||||
RecordedMediaArtifact,
|
||||
ReplayArtifact,
|
||||
ReplayCommand,
|
||||
SessionIntegrityError,
|
||||
SessionNotFoundError,
|
||||
SessionNotReplayableError,
|
||||
)
|
||||
from .plugin_contract import (
|
||||
ObservationArchiveSource,
|
||||
ObservationRuntimeContribution,
|
||||
PluginRecordingExportCancelled,
|
||||
PluginRecordingExportError,
|
||||
RecordingExporter,
|
||||
)
|
||||
from .preparation import (
|
||||
RecordingPreparationQueueFull,
|
||||
RecordingPreparationSnapshot,
|
||||
@@ -42,6 +52,12 @@ __all__ = [
|
||||
"ActiveSessionLease",
|
||||
"ActiveSessionLeaseError",
|
||||
"MaterializedRecording",
|
||||
"ObservationArchiveSource",
|
||||
"ObservationArtifactCandidate",
|
||||
"ObservationRuntimeContribution",
|
||||
"ObservationSessionCandidate",
|
||||
"PluginRecordingExportCancelled",
|
||||
"PluginRecordingExportError",
|
||||
"RecordingMaterializationCancelled",
|
||||
"RecordedMediaArtifact",
|
||||
"RECORDED_MEDIA_MANIFEST_SCHEMA",
|
||||
@@ -49,6 +65,8 @@ __all__ = [
|
||||
"RecordedMediaInspector",
|
||||
"RecordedMediaManifest",
|
||||
"ReplayCommand",
|
||||
"ReplayArtifact",
|
||||
"RecordingExporter",
|
||||
"RecordingMaterializationError",
|
||||
"RecordingPreparationQueueFull",
|
||||
"RecordingPreparationSnapshot",
|
||||
|
||||
@@ -14,8 +14,6 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, TypeGuard
|
||||
|
||||
from k1link.viewer.replay import MAX_METADATA_LINE_CHARS
|
||||
|
||||
from .models import RecordedMediaArtifact, ReplayCommand, SessionIntegrityError
|
||||
|
||||
CAMERA_ARCHIVE_SCHEMA = "missioncore.camera-recording/v1"
|
||||
@@ -139,7 +137,7 @@ class RecordedMediaInspector:
|
||||
cached = self._cache.get(key)
|
||||
if cached is not None:
|
||||
identity = _prepared_source_identity(
|
||||
replay.source_path,
|
||||
replay,
|
||||
epoch_paths,
|
||||
cached.manifest.epochs,
|
||||
)
|
||||
@@ -151,15 +149,14 @@ class RecordedMediaInspector:
|
||||
with self._lock:
|
||||
self._cache[key] = _CachedManifest(identity=identity, manifest=manifest)
|
||||
return manifest
|
||||
origin_epoch_ns, origin_monotonic_ns = _raw_timeline_origin(replay.source_path)
|
||||
manifest = _read_manifest(
|
||||
artifact,
|
||||
epoch_paths,
|
||||
origin_epoch_ns=origin_epoch_ns,
|
||||
origin_monotonic_ns=origin_monotonic_ns,
|
||||
origin_epoch_ns=replay.timeline_origin_epoch_ns,
|
||||
origin_monotonic_ns=replay.timeline_origin_monotonic_ns,
|
||||
)
|
||||
identity = _prepared_source_identity(
|
||||
replay.source_path,
|
||||
replay,
|
||||
epoch_paths,
|
||||
manifest.epochs,
|
||||
)
|
||||
@@ -186,7 +183,7 @@ class RecordedMediaInspector:
|
||||
document = _decode_prepared_sidecar(payload)
|
||||
manifest = _manifest_from_sidecar(document, artifact, epoch_paths)
|
||||
identity = _prepared_source_identity(
|
||||
replay.source_path,
|
||||
replay,
|
||||
epoch_paths,
|
||||
manifest.epochs,
|
||||
)
|
||||
@@ -631,15 +628,15 @@ def _epoch_paths(source_path: Path) -> tuple[Path, ...]:
|
||||
|
||||
|
||||
def _prepared_source_identity(
|
||||
raw_path: Path,
|
||||
replay: ReplayCommand,
|
||||
epoch_paths: tuple[Path, ...],
|
||||
epochs: tuple[RecordedMediaEpoch, ...],
|
||||
) -> tuple[tuple[int, int, int, int], ...]:
|
||||
if len(epoch_paths) != len(epochs):
|
||||
raise SessionIntegrityError("recorded media epoch identity is inconsistent")
|
||||
identity: list[tuple[int, int, int, int]] = []
|
||||
for source_file in (raw_path, raw_path.with_name("mqtt.metadata.jsonl")):
|
||||
metadata = _confined_file_stat(source_file, raw_path.parent)
|
||||
for artifact in replay.artifacts:
|
||||
metadata = _session_artifact_stat(artifact.path, replay.session_root)
|
||||
identity.append(
|
||||
(metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mtime_ns)
|
||||
)
|
||||
@@ -671,32 +668,15 @@ def _prepared_source_identity(
|
||||
return tuple(identity)
|
||||
|
||||
|
||||
def _raw_timeline_origin(raw_path: Path) -> tuple[int, int]:
|
||||
metadata_path = raw_path.with_name("mqtt.metadata.jsonl")
|
||||
def _session_artifact_stat(path: Path, session_root: Path) -> os.stat_result:
|
||||
try:
|
||||
line = _read_first_confined_line(
|
||||
metadata_path,
|
||||
raw_path.parent,
|
||||
MAX_METADATA_LINE_CHARS,
|
||||
).decode("utf-8")
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
raise SessionIntegrityError("native capture timing metadata is unavailable") from exc
|
||||
if not line or len(line) > MAX_METADATA_LINE_CHARS or not line.endswith(("\n", "\r")):
|
||||
raise SessionIntegrityError("native capture timing origin is incomplete")
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise SessionIntegrityError("native capture timing origin is invalid") from exc
|
||||
epoch_ns = record.get("received_at_epoch_ns") if isinstance(record, dict) else None
|
||||
monotonic_ns = record.get("received_monotonic_ns") if isinstance(record, dict) else None
|
||||
if (
|
||||
record.get("record_type") != "message"
|
||||
or record.get("sequence") != 1
|
||||
or not _non_negative_int(epoch_ns)
|
||||
or not _non_negative_int(monotonic_ns)
|
||||
):
|
||||
raise SessionIntegrityError("native capture timing origin is invalid")
|
||||
return int(epoch_ns), int(monotonic_ns)
|
||||
session = session_root.resolve(strict=True)
|
||||
parent = path.parent.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise SessionIntegrityError("recording source artifact is missing") from exc
|
||||
if not parent.is_relative_to(session):
|
||||
raise SessionIntegrityError("recording source artifact escapes its session")
|
||||
return _confined_file_stat(path, parent)
|
||||
|
||||
|
||||
def _read_manifest(
|
||||
|
||||
@@ -160,20 +160,44 @@ class WorkspaceLayout:
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReplayArtifact:
|
||||
"""One confined input artifact selected for plugin-owned preparation."""
|
||||
|
||||
artifact_id: str
|
||||
path: Path
|
||||
media_type: str
|
||||
file_byte_length: int
|
||||
replay_byte_length: int
|
||||
expected_sha256: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReplayCommand:
|
||||
"""Internal-only replay command. ``source_path`` never enters an API DTO."""
|
||||
"""Internal replay request containing no vendor format or channel names."""
|
||||
|
||||
session_id: str
|
||||
source_path: Path
|
||||
plugin_id: str
|
||||
allowed_root: Path
|
||||
session_root: Path
|
||||
replay_byte_length: int
|
||||
metadata_byte_length: int
|
||||
expected_source_sha256: str | None
|
||||
primary_artifact_id: str
|
||||
artifacts: tuple[ReplayArtifact, ...]
|
||||
timeline_origin_epoch_ns: int
|
||||
timeline_origin_monotonic_ns: int
|
||||
speed: float
|
||||
loop: bool
|
||||
|
||||
@property
|
||||
def primary_artifact(self) -> ReplayArtifact:
|
||||
matches = tuple(
|
||||
artifact
|
||||
for artifact in self.artifacts
|
||||
if artifact.artifact_id == self.primary_artifact_id
|
||||
)
|
||||
if len(matches) != 1:
|
||||
raise SessionIntegrityError("replay command has no unique primary artifact")
|
||||
return matches[0]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RecordedMediaArtifact:
|
||||
@@ -191,7 +215,19 @@ class RecordedMediaArtifact:
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LegacySessionCandidate:
|
||||
class ObservationArtifactCandidate:
|
||||
artifact_id: str
|
||||
kind: str
|
||||
media_type: str
|
||||
locator: Path
|
||||
byte_length: int
|
||||
replay_byte_length: int
|
||||
sha256: str | None
|
||||
integrity_status: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservationSessionCandidate:
|
||||
session_id: str
|
||||
display_name: str
|
||||
status: SessionStatus
|
||||
@@ -203,13 +239,11 @@ class LegacySessionCandidate:
|
||||
total_bytes: int
|
||||
allowed_root: Path
|
||||
session_root: Path
|
||||
raw_path: Path
|
||||
raw_byte_length: int
|
||||
replay_raw_byte_length: int
|
||||
replay_metadata_byte_length: int
|
||||
raw_sha256: str | None
|
||||
raw_integrity_status: str
|
||||
media_sources: tuple[LegacyMediaSourceCandidate, ...]
|
||||
primary_replay_artifact_id: str | None
|
||||
timeline_origin_epoch_ns: int | None
|
||||
timeline_origin_monotonic_ns: int | None
|
||||
sources: tuple[SessionSource, ...]
|
||||
artifacts: tuple[ObservationArtifactCandidate, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Host-side runtime ABI for observation-capable device plugins.
|
||||
|
||||
The contract deliberately contains no transport name, vendor topic, capture
|
||||
suffix, codec, or viewer implementation. Concrete plugins discover native
|
||||
evidence and export it into the host's canonical recorded-viewer artifact.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Protocol
|
||||
|
||||
from .models import ObservationSessionCandidate
|
||||
|
||||
RecordingProgressPulse = Callable[[], None]
|
||||
RecordingExportResult = Mapping[str, object]
|
||||
|
||||
|
||||
class PluginRecordingExportError(RuntimeError):
|
||||
"""A plugin rejected or failed to convert one native recording."""
|
||||
|
||||
|
||||
class PluginRecordingExportCancelled(PluginRecordingExportError):
|
||||
"""A plugin cooperatively stopped a recording conversion."""
|
||||
|
||||
|
||||
class RecordingExporter(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
source: Path,
|
||||
destination: Path,
|
||||
*,
|
||||
cancel_event: threading.Event | None = None,
|
||||
activity_callback: RecordingProgressPulse | None = None,
|
||||
) -> RecordingExportResult: ...
|
||||
|
||||
|
||||
ObservationArchiveDiscovery = Callable[
|
||||
[Path],
|
||||
tuple[ObservationSessionCandidate, ...],
|
||||
]
|
||||
ObservationArchiveRecovery = Callable[[Path], object]
|
||||
|
||||
|
||||
def _no_recovery(_: Path) -> object:
|
||||
return None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservationArchiveSource:
|
||||
"""One plugin-owned evidence namespace reconciled by the host catalog."""
|
||||
|
||||
plugin_id: str
|
||||
archive_id: str
|
||||
root: Path
|
||||
discover: ObservationArchiveDiscovery
|
||||
recover: ObservationArchiveRecovery = _no_recovery
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservationRuntimeContribution:
|
||||
"""Optional observation capabilities contributed by a device plugin."""
|
||||
|
||||
archives: tuple[ObservationArchiveSource, ...]
|
||||
recording_exporter: RecordingExporter
|
||||
@@ -584,12 +584,20 @@ def _source_identity(command: ReplayCommand) -> tuple[object, ...]:
|
||||
|
||||
identities: list[object] = [
|
||||
command.session_id,
|
||||
str(command.source_path),
|
||||
command.replay_byte_length,
|
||||
command.metadata_byte_length,
|
||||
command.expected_source_sha256,
|
||||
command.plugin_id,
|
||||
command.primary_artifact_id,
|
||||
]
|
||||
for path in (command.source_path, command.source_path.with_name("mqtt.metadata.jsonl")):
|
||||
for artifact in command.artifacts:
|
||||
path = artifact.path
|
||||
identities.extend(
|
||||
(
|
||||
artifact.artifact_id,
|
||||
artifact.media_type,
|
||||
artifact.file_byte_length,
|
||||
artifact.replay_byte_length,
|
||||
artifact.expected_sha256,
|
||||
)
|
||||
)
|
||||
try:
|
||||
value = os.lstat(path)
|
||||
except OSError:
|
||||
|
||||
+276
-171
@@ -16,18 +16,17 @@ from pathlib import Path
|
||||
from typing import Any, cast
|
||||
from uuid import uuid4
|
||||
|
||||
from k1link.viewer.rrd_export import (
|
||||
RrdExportCancelled,
|
||||
RrdExportError,
|
||||
export_k1mqtt_to_rrd,
|
||||
from .models import ReplayArtifact, ReplayCommand
|
||||
from .plugin_contract import (
|
||||
PluginRecordingExportCancelled,
|
||||
PluginRecordingExportError,
|
||||
RecordingExporter,
|
||||
)
|
||||
|
||||
from .models import ReplayCommand
|
||||
|
||||
# v6 adds a real session_time=0 row to the RRD itself. Older sidecars can
|
||||
# declare a zero start while their payload begins at the first decoded sensor
|
||||
# frame, so accepting them would violate the browser playback contract.
|
||||
CACHE_SCHEMA = "missioncore.derived-rerun-recording-cache/v6"
|
||||
CACHE_SCHEMA = "missioncore.derived-rerun-recording-cache/v7"
|
||||
COMPATIBLE_CACHE_SCHEMAS = frozenset({CACHE_SCHEMA})
|
||||
RERUN_RECORDING_MEDIA_TYPE = "application/vnd.rerun.rrd"
|
||||
RERUN_SESSION_TIMELINE = "session_time"
|
||||
@@ -37,7 +36,6 @@ DEFAULT_FREE_SPACE_RESERVE_BYTES = 2 * 1024 * 1024 * 1024
|
||||
|
||||
RrdExporter = Callable[..., Mapping[str, object]]
|
||||
RecordingProgressCallback = Callable[[str, float], None]
|
||||
DEFAULT_RRD_EXPORTER = cast(RrdExporter, export_k1mqtt_to_rrd)
|
||||
|
||||
|
||||
class RecordingMaterializationError(RuntimeError):
|
||||
@@ -71,29 +69,58 @@ class _ValidatedMemoryEntry:
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ValidatedSource:
|
||||
source: Path
|
||||
metadata: Path
|
||||
source_stat: os.stat_result
|
||||
metadata_stat: os.stat_result
|
||||
class _ValidatedArtifact:
|
||||
artifact_id: str
|
||||
path: Path
|
||||
media_type: str
|
||||
file_stat: os.stat_result
|
||||
replay_byte_length: int
|
||||
metadata_byte_length: int
|
||||
expected_source_sha256: str | None
|
||||
expected_sha256: str | None
|
||||
|
||||
@property
|
||||
def identity(self) -> tuple[object, ...]:
|
||||
return (
|
||||
*_stat_identity(self.source_stat),
|
||||
*_stat_identity(self.metadata_stat),
|
||||
self.artifact_id,
|
||||
self.media_type,
|
||||
*_stat_identity(self.file_stat),
|
||||
self.replay_byte_length,
|
||||
self.metadata_byte_length,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ValidatedSource:
|
||||
plugin_id: str
|
||||
primary_artifact_id: str
|
||||
artifacts: tuple[_ValidatedArtifact, ...]
|
||||
|
||||
@property
|
||||
def primary(self) -> _ValidatedArtifact:
|
||||
matches = tuple(
|
||||
artifact
|
||||
for artifact in self.artifacts
|
||||
if artifact.artifact_id == self.primary_artifact_id
|
||||
)
|
||||
if len(matches) != 1:
|
||||
raise RecordingMaterializationError("recording source has no primary artifact")
|
||||
return matches[0]
|
||||
|
||||
@property
|
||||
def identity(self) -> tuple[object, ...]:
|
||||
return (
|
||||
self.plugin_id,
|
||||
self.primary_artifact_id,
|
||||
*(item for artifact in self.artifacts for item in artifact.identity),
|
||||
)
|
||||
|
||||
@property
|
||||
def replay_byte_length(self) -> int:
|
||||
return sum(artifact.replay_byte_length for artifact in self.artifacts)
|
||||
|
||||
|
||||
class SessionRecordingMaterializer:
|
||||
"""Build and validate a per-session seekable RRD under the private data root.
|
||||
|
||||
The native ``.k1mqtt`` capture remains the source of record. Derived RRDs
|
||||
Native plugin evidence remains the source of record. Derived RRDs
|
||||
live below ``data_dir/recordings`` and are reused only when both their
|
||||
source identity and output digest still match an atomically written cache
|
||||
sidecar. Calls for one session are serialized, so concurrent browser
|
||||
@@ -104,7 +131,8 @@ class SessionRecordingMaterializer:
|
||||
self,
|
||||
data_dir: Path,
|
||||
*,
|
||||
exporter: RrdExporter = DEFAULT_RRD_EXPORTER,
|
||||
exporter: RrdExporter | None = None,
|
||||
exporters: Mapping[str, RecordingExporter] | None = None,
|
||||
cache_max_bytes: int | None = None,
|
||||
free_space_reserve_bytes: int | None = None,
|
||||
) -> None:
|
||||
@@ -119,12 +147,12 @@ class SessionRecordingMaterializer:
|
||||
self.recordings_root = recordings_root.resolve()
|
||||
if not self.recordings_root.is_relative_to(private_root):
|
||||
raise RecordingMaterializationError("recording cache escapes the private data root")
|
||||
self._exporter = exporter
|
||||
self._exporter_accepts_cancel = _callable_accepts_keyword(exporter, "cancel_event")
|
||||
self._exporter_accepts_activity = _callable_accepts_keyword(
|
||||
exporter,
|
||||
"activity_callback",
|
||||
)
|
||||
if exporter is not None and exporters is not None:
|
||||
raise ValueError("configure either one test exporter or plugin exporters")
|
||||
self._fallback_exporter = exporter
|
||||
self._exporters = dict(exporters or {})
|
||||
if len(self._exporters) != len(set(self._exporters)):
|
||||
raise ValueError("recording exporter plugin ids must be unique")
|
||||
self.cache_max_bytes = _positive_configuration(
|
||||
cache_max_bytes,
|
||||
environment_name="MISSIONCORE_RRD_CACHE_MAX_BYTES",
|
||||
@@ -152,7 +180,12 @@ class SessionRecordingMaterializer:
|
||||
def supports_cooperative_cancellation(self) -> bool:
|
||||
"""Whether the configured exporter observes a cancellation event."""
|
||||
|
||||
return self._exporter_accepts_cancel
|
||||
exporters: tuple[Callable[..., object], ...] = tuple(self._exporters.values())
|
||||
if self._fallback_exporter is not None:
|
||||
exporters = (*exporters, self._fallback_exporter)
|
||||
return bool(exporters) and all(
|
||||
_callable_accepts_keyword(exporter, "cancel_event") for exporter in exporters
|
||||
)
|
||||
|
||||
def is_recording_available(self, recording: MaterializedRecording) -> bool:
|
||||
"""Cheap no-follow check for a previously validated ready handle."""
|
||||
@@ -463,11 +496,11 @@ class SessionRecordingMaterializer:
|
||||
) from exc
|
||||
|
||||
staged_root: Path | None = None
|
||||
export_source = source.source
|
||||
export_source = source.primary.path
|
||||
try:
|
||||
if (
|
||||
source.replay_byte_length != source.source_stat.st_size
|
||||
or source.metadata_byte_length != source.metadata_stat.st_size
|
||||
if any(
|
||||
artifact.replay_byte_length != artifact.file_stat.st_size
|
||||
for artifact in source.artifacts
|
||||
):
|
||||
staged_root, export_source = _stage_replay_prefix(
|
||||
resolved_session_root,
|
||||
@@ -480,6 +513,7 @@ class SessionRecordingMaterializer:
|
||||
),
|
||||
)
|
||||
summary = self._invoke_exporter(
|
||||
source.plugin_id,
|
||||
export_source,
|
||||
candidate_path,
|
||||
cancel_event=cancel_event,
|
||||
@@ -491,12 +525,12 @@ class SessionRecordingMaterializer:
|
||||
)
|
||||
_raise_if_cancelled(cancel_event)
|
||||
_report_progress(progress_callback, "finalizing", 0.9)
|
||||
except RrdExportCancelled as exc:
|
||||
except PluginRecordingExportCancelled as exc:
|
||||
candidate_path.unlink(missing_ok=True)
|
||||
raise RecordingMaterializationCancelled(
|
||||
"recording preparation was cancelled"
|
||||
) from exc
|
||||
except RrdExportError as exc:
|
||||
except PluginRecordingExportError as exc:
|
||||
candidate_path.unlink(missing_ok=True)
|
||||
raise RecordingMaterializationError("native capture could not be exported") from exc
|
||||
except OSError as exc:
|
||||
@@ -515,13 +549,13 @@ class SessionRecordingMaterializer:
|
||||
raise RecordingMaterializationError("native capture changed during RRD export")
|
||||
_chmod_best_effort(candidate_path, 0o600)
|
||||
source_sha256 = _sha256_prefix_stable(
|
||||
source.source,
|
||||
source.source_stat,
|
||||
source.replay_byte_length,
|
||||
source.primary.path,
|
||||
source.primary.file_stat,
|
||||
source.primary.replay_byte_length,
|
||||
)
|
||||
if (
|
||||
source.expected_source_sha256 is not None
|
||||
and source_sha256 != source.expected_source_sha256
|
||||
source.primary.expected_sha256 is not None
|
||||
and source_sha256 != source.primary.expected_sha256
|
||||
):
|
||||
raise RecordingMaterializationError(
|
||||
"native capture digest no longer matches catalog"
|
||||
@@ -574,18 +608,25 @@ class SessionRecordingMaterializer:
|
||||
|
||||
def _invoke_exporter(
|
||||
self,
|
||||
plugin_id: str,
|
||||
source: Path,
|
||||
destination: Path,
|
||||
*,
|
||||
cancel_event: threading.Event | None,
|
||||
activity_callback: Callable[[], None],
|
||||
) -> Mapping[str, object]:
|
||||
selected = self._fallback_exporter or self._exporters.get(plugin_id)
|
||||
if selected is None:
|
||||
raise PluginRecordingExportError(
|
||||
f"device plugin has no recording exporter: {plugin_id}"
|
||||
)
|
||||
exporter = cast(RrdExporter, selected)
|
||||
kwargs: dict[str, object] = {}
|
||||
if self._exporter_accepts_cancel:
|
||||
if _callable_accepts_keyword(exporter, "cancel_event"):
|
||||
kwargs["cancel_event"] = cancel_event
|
||||
if self._exporter_accepts_activity:
|
||||
if _callable_accepts_keyword(exporter, "activity_callback"):
|
||||
kwargs["activity_callback"] = activity_callback
|
||||
return self._exporter(source, destination, **kwargs)
|
||||
return exporter(source, destination, **kwargs)
|
||||
|
||||
def _load_memory_cache(
|
||||
self,
|
||||
@@ -639,48 +680,53 @@ class SessionRecordingMaterializer:
|
||||
except (OSError, ValueError, json.JSONDecodeError, RecordingMaterializationError):
|
||||
return None
|
||||
|
||||
if document["source_file_byte_length"] != source.source_stat.st_size:
|
||||
if document["plugin_id"] != source.plugin_id:
|
||||
return None
|
||||
if document["source_mtime_ns"] != source.source_stat.st_mtime_ns:
|
||||
if document["primary_artifact_id"] != source.primary_artifact_id:
|
||||
return None
|
||||
if document["source_ctime_ns"] != source.source_stat.st_ctime_ns:
|
||||
return None
|
||||
if document["source_replay_byte_length"] != source.replay_byte_length:
|
||||
return None
|
||||
if document["metadata_file_byte_length"] != source.metadata_stat.st_size:
|
||||
return None
|
||||
if document["metadata_mtime_ns"] != source.metadata_stat.st_mtime_ns:
|
||||
return None
|
||||
if document["metadata_ctime_ns"] != source.metadata_stat.st_ctime_ns:
|
||||
return None
|
||||
if document["metadata_replay_byte_length"] != source.metadata_byte_length:
|
||||
cached_artifacts = document["source_artifacts"]
|
||||
if len(cached_artifacts) != len(source.artifacts):
|
||||
return None
|
||||
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
|
||||
if cached["artifact_id"] != artifact.artifact_id:
|
||||
return None
|
||||
if cached["media_type"] != artifact.media_type:
|
||||
return None
|
||||
if cached["file_byte_length"] != artifact.file_stat.st_size:
|
||||
return None
|
||||
if cached["mtime_ns"] != artifact.file_stat.st_mtime_ns:
|
||||
return None
|
||||
if cached["ctime_ns"] != artifact.file_stat.st_ctime_ns:
|
||||
return None
|
||||
if cached["replay_byte_length"] != artifact.replay_byte_length:
|
||||
return None
|
||||
if document["recording_byte_length"] != recording_stat.st_size:
|
||||
return None
|
||||
if document["recording_mtime_ns"] != recording_stat.st_mtime_ns:
|
||||
return None
|
||||
|
||||
source_sha256 = _sha256_prefix_stable(
|
||||
source.source,
|
||||
source.source_stat,
|
||||
source.replay_byte_length,
|
||||
source.primary.path,
|
||||
source.primary.file_stat,
|
||||
source.primary.replay_byte_length,
|
||||
)
|
||||
if (
|
||||
source.expected_source_sha256 is not None
|
||||
and source_sha256 != source.expected_source_sha256
|
||||
source.primary.expected_sha256 is not None
|
||||
and source_sha256 != source.primary.expected_sha256
|
||||
):
|
||||
raise RecordingMaterializationError(
|
||||
"native capture digest no longer matches catalog"
|
||||
)
|
||||
if source_sha256 != document["source_sha256"]:
|
||||
return None
|
||||
metadata_sha256 = _sha256_prefix_stable(
|
||||
source.metadata,
|
||||
source.metadata_stat,
|
||||
source.metadata_byte_length,
|
||||
)
|
||||
if metadata_sha256 != document["metadata_sha256"]:
|
||||
return None
|
||||
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
|
||||
digest = _sha256_prefix_stable(
|
||||
artifact.path,
|
||||
artifact.file_stat,
|
||||
artifact.replay_byte_length,
|
||||
)
|
||||
if digest != cached["sha256"]:
|
||||
return None
|
||||
recording_sha256 = _sha256_stable(recording_path, recording_stat)
|
||||
if recording_sha256 != document["recording_sha256"]:
|
||||
return None
|
||||
@@ -764,76 +810,103 @@ def _raise_if_cancelled(cancel_event: threading.Event | None) -> None:
|
||||
|
||||
|
||||
def _validate_source(command: ReplayCommand) -> _ValidatedSource:
|
||||
source_path = getattr(command, "source_path", None)
|
||||
plugin_id = getattr(command, "plugin_id", None)
|
||||
allowed_root = getattr(command, "allowed_root", None)
|
||||
session_root = getattr(command, "session_root", None)
|
||||
replay_byte_length = getattr(command, "replay_byte_length", None)
|
||||
metadata_byte_length = getattr(command, "metadata_byte_length", None)
|
||||
expected_source_sha256 = getattr(command, "expected_source_sha256", None)
|
||||
if not isinstance(source_path, Path):
|
||||
raise RecordingMaterializationError("replay command has no native capture")
|
||||
primary_artifact_id = getattr(command, "primary_artifact_id", None)
|
||||
artifacts = getattr(command, "artifacts", None)
|
||||
if not isinstance(plugin_id, str) or SESSION_ID_PATTERN.fullmatch(plugin_id) is None:
|
||||
raise RecordingMaterializationError("replay command has an invalid plugin id")
|
||||
if not isinstance(primary_artifact_id, str) or SESSION_ID_PATTERN.fullmatch(
|
||||
primary_artifact_id
|
||||
) is None:
|
||||
raise RecordingMaterializationError("replay command has an invalid primary artifact")
|
||||
if not isinstance(artifacts, tuple) or not artifacts:
|
||||
raise RecordingMaterializationError("replay command has no source artifacts")
|
||||
try:
|
||||
if not isinstance(allowed_root, Path) or not isinstance(session_root, Path):
|
||||
raise RecordingMaterializationError("replay command has no confinement roots")
|
||||
allowed = allowed_root.expanduser().resolve(strict=True)
|
||||
session = session_root.expanduser().resolve(strict=True)
|
||||
source = source_path.expanduser().absolute()
|
||||
source_parent = source.parent.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise RecordingMaterializationError("native capture is missing") from exc
|
||||
if (
|
||||
not allowed.is_dir()
|
||||
or not session.is_dir()
|
||||
or not session.is_relative_to(allowed)
|
||||
or not source_parent.is_relative_to(session)
|
||||
):
|
||||
raise RecordingMaterializationError("native capture escapes its allowed session root")
|
||||
if source.suffix.casefold() != ".k1mqtt":
|
||||
raise RecordingMaterializationError("native capture has an unsupported format")
|
||||
source_stat = _regular_file_stat_nofollow(source, "native capture")
|
||||
metadata = source.with_name("mqtt.metadata.jsonl")
|
||||
if not metadata.parent.resolve(strict=True).is_relative_to(session):
|
||||
raise RecordingMaterializationError("native metadata escapes its allowed session root")
|
||||
metadata_stat = _regular_file_stat_nofollow(metadata, "native metadata")
|
||||
if (
|
||||
not isinstance(replay_byte_length, int)
|
||||
or isinstance(replay_byte_length, bool)
|
||||
or not 1 <= replay_byte_length <= source_stat.st_size
|
||||
):
|
||||
raise RecordingMaterializationError("native capture replay boundary is invalid")
|
||||
if (
|
||||
not isinstance(metadata_byte_length, int)
|
||||
or isinstance(metadata_byte_length, bool)
|
||||
or not 1 <= metadata_byte_length <= metadata_stat.st_size
|
||||
):
|
||||
raise RecordingMaterializationError("native metadata replay boundary is invalid")
|
||||
if expected_source_sha256 is not None:
|
||||
if not isinstance(expected_source_sha256, str) or not _is_sha256(expected_source_sha256):
|
||||
raise RecordingMaterializationError("native capture expected digest is invalid")
|
||||
if replay_byte_length != source_stat.st_size:
|
||||
raise RecordingMaterializationError(
|
||||
"a full-capture digest cannot describe a replay prefix"
|
||||
raise RecordingMaterializationError("recording confinement root is missing") from exc
|
||||
if not allowed.is_dir() or not session.is_dir() or not session.is_relative_to(allowed):
|
||||
raise RecordingMaterializationError("recording source escapes its allowed session root")
|
||||
|
||||
validated: list[_ValidatedArtifact] = []
|
||||
seen_ids: set[str] = set()
|
||||
seen_names: set[str] = set()
|
||||
for artifact in artifacts:
|
||||
if not isinstance(artifact, ReplayArtifact):
|
||||
raise RecordingMaterializationError("replay command contains an invalid artifact")
|
||||
if (
|
||||
SESSION_ID_PATTERN.fullmatch(artifact.artifact_id) is None
|
||||
or artifact.artifact_id in seen_ids
|
||||
):
|
||||
raise RecordingMaterializationError("replay artifact id is invalid or duplicated")
|
||||
path = artifact.path.expanduser().absolute()
|
||||
try:
|
||||
parent = path.parent.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise RecordingMaterializationError("recording source artifact is missing") from exc
|
||||
if not parent.is_relative_to(session):
|
||||
raise RecordingMaterializationError("recording source artifact escapes its session")
|
||||
file_stat = _regular_file_stat_nofollow(path, "recording source artifact")
|
||||
if artifact.file_byte_length > file_stat.st_size:
|
||||
raise RecordingMaterializationError("recording artifact was truncated after cataloging")
|
||||
if (
|
||||
isinstance(artifact.replay_byte_length, bool)
|
||||
or not 1 <= artifact.replay_byte_length <= file_stat.st_size
|
||||
):
|
||||
raise RecordingMaterializationError("recording artifact replay boundary is invalid")
|
||||
if artifact.expected_sha256 is not None:
|
||||
if not _is_sha256(artifact.expected_sha256):
|
||||
raise RecordingMaterializationError("recording artifact digest is invalid")
|
||||
if artifact.replay_byte_length != file_stat.st_size:
|
||||
raise RecordingMaterializationError(
|
||||
"a full-artifact digest cannot describe a replay prefix"
|
||||
)
|
||||
if path.name in seen_names:
|
||||
raise RecordingMaterializationError("recording artifact filenames are duplicated")
|
||||
seen_ids.add(artifact.artifact_id)
|
||||
seen_names.add(path.name)
|
||||
validated.append(
|
||||
_ValidatedArtifact(
|
||||
artifact_id=artifact.artifact_id,
|
||||
path=path,
|
||||
media_type=artifact.media_type,
|
||||
file_stat=file_stat,
|
||||
replay_byte_length=artifact.replay_byte_length,
|
||||
expected_sha256=artifact.expected_sha256,
|
||||
)
|
||||
)
|
||||
if sum(artifact.artifact_id == primary_artifact_id for artifact in validated) != 1:
|
||||
raise RecordingMaterializationError("recording primary artifact is unavailable")
|
||||
return _ValidatedSource(
|
||||
source=source,
|
||||
metadata=metadata,
|
||||
source_stat=source_stat,
|
||||
metadata_stat=metadata_stat,
|
||||
replay_byte_length=replay_byte_length,
|
||||
metadata_byte_length=metadata_byte_length,
|
||||
expected_source_sha256=expected_source_sha256,
|
||||
plugin_id=plugin_id,
|
||||
primary_artifact_id=primary_artifact_id,
|
||||
artifacts=tuple(validated),
|
||||
)
|
||||
|
||||
|
||||
def _validate_source_state(source: _ValidatedSource) -> _ValidatedSource:
|
||||
return _ValidatedSource(
|
||||
source=source.source,
|
||||
metadata=source.metadata,
|
||||
source_stat=_regular_file_stat_nofollow(source.source, "native capture"),
|
||||
metadata_stat=_regular_file_stat_nofollow(source.metadata, "native metadata"),
|
||||
replay_byte_length=source.replay_byte_length,
|
||||
metadata_byte_length=source.metadata_byte_length,
|
||||
expected_source_sha256=source.expected_source_sha256,
|
||||
plugin_id=source.plugin_id,
|
||||
primary_artifact_id=source.primary_artifact_id,
|
||||
artifacts=tuple(
|
||||
_ValidatedArtifact(
|
||||
artifact_id=artifact.artifact_id,
|
||||
path=artifact.path,
|
||||
media_type=artifact.media_type,
|
||||
file_stat=_regular_file_stat_nofollow(
|
||||
artifact.path,
|
||||
"recording source artifact",
|
||||
),
|
||||
replay_byte_length=artifact.replay_byte_length,
|
||||
expected_sha256=artifact.expected_sha256,
|
||||
)
|
||||
for artifact in source.artifacts
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -880,20 +953,25 @@ def _cache_document(
|
||||
return {
|
||||
"schema_version": CACHE_SCHEMA,
|
||||
"session_id": recording.session_id,
|
||||
"source_file_byte_length": source.source_stat.st_size,
|
||||
"source_replay_byte_length": source.replay_byte_length,
|
||||
"source_mtime_ns": source.source_stat.st_mtime_ns,
|
||||
"source_ctime_ns": source.source_stat.st_ctime_ns,
|
||||
"plugin_id": source.plugin_id,
|
||||
"primary_artifact_id": source.primary_artifact_id,
|
||||
"source_sha256": recording.source_sha256,
|
||||
"metadata_file_byte_length": source.metadata_stat.st_size,
|
||||
"metadata_replay_byte_length": source.metadata_byte_length,
|
||||
"metadata_mtime_ns": source.metadata_stat.st_mtime_ns,
|
||||
"metadata_ctime_ns": source.metadata_stat.st_ctime_ns,
|
||||
"metadata_sha256": _sha256_prefix_stable(
|
||||
source.metadata,
|
||||
source.metadata_stat,
|
||||
source.metadata_byte_length,
|
||||
),
|
||||
"source_artifacts": [
|
||||
{
|
||||
"artifact_id": artifact.artifact_id,
|
||||
"media_type": artifact.media_type,
|
||||
"file_byte_length": artifact.file_stat.st_size,
|
||||
"replay_byte_length": artifact.replay_byte_length,
|
||||
"mtime_ns": artifact.file_stat.st_mtime_ns,
|
||||
"ctime_ns": artifact.file_stat.st_ctime_ns,
|
||||
"sha256": _sha256_prefix_stable(
|
||||
artifact.path,
|
||||
artifact.file_stat,
|
||||
artifact.replay_byte_length,
|
||||
),
|
||||
}
|
||||
for artifact in source.artifacts
|
||||
],
|
||||
"recording_byte_length": recording.byte_length,
|
||||
"recording_mtime_ns": recording_stat.st_mtime_ns,
|
||||
"recording_sha256": recording.sha256,
|
||||
@@ -909,16 +987,10 @@ def _validate_cache_document(value: object, session_id: str) -> dict[str, Any]:
|
||||
expected_keys = {
|
||||
"schema_version",
|
||||
"session_id",
|
||||
"source_file_byte_length",
|
||||
"source_replay_byte_length",
|
||||
"source_mtime_ns",
|
||||
"source_ctime_ns",
|
||||
"plugin_id",
|
||||
"primary_artifact_id",
|
||||
"source_sha256",
|
||||
"metadata_file_byte_length",
|
||||
"metadata_replay_byte_length",
|
||||
"metadata_mtime_ns",
|
||||
"metadata_ctime_ns",
|
||||
"metadata_sha256",
|
||||
"source_artifacts",
|
||||
"recording_byte_length",
|
||||
"recording_mtime_ns",
|
||||
"recording_sha256",
|
||||
@@ -932,15 +1004,10 @@ def _validate_cache_document(value: object, session_id: str) -> dict[str, Any]:
|
||||
raise ValueError("recording cache sidecar identity does not match")
|
||||
if value["timeline"] != RERUN_SESSION_TIMELINE:
|
||||
raise ValueError("recording cache timeline is unsupported")
|
||||
for key in ("plugin_id", "primary_artifact_id"):
|
||||
if not isinstance(value[key], str) or SESSION_ID_PATTERN.fullmatch(value[key]) is None:
|
||||
raise ValueError("recording cache contains an invalid identifier")
|
||||
for key in (
|
||||
"source_file_byte_length",
|
||||
"source_replay_byte_length",
|
||||
"source_mtime_ns",
|
||||
"source_ctime_ns",
|
||||
"metadata_file_byte_length",
|
||||
"metadata_replay_byte_length",
|
||||
"metadata_mtime_ns",
|
||||
"metadata_ctime_ns",
|
||||
"recording_byte_length",
|
||||
"recording_mtime_ns",
|
||||
"timeline_start_ns",
|
||||
@@ -948,10 +1015,51 @@ def _validate_cache_document(value: object, session_id: str) -> dict[str, Any]:
|
||||
):
|
||||
if not isinstance(value[key], int) or isinstance(value[key], bool) or value[key] < 0:
|
||||
raise ValueError("recording cache contains an invalid integer")
|
||||
for key in ("source_sha256", "metadata_sha256", "recording_sha256"):
|
||||
for key in ("source_sha256", "recording_sha256"):
|
||||
digest = value[key]
|
||||
if not isinstance(digest, str) or not _is_sha256(digest):
|
||||
raise ValueError("recording cache contains an invalid digest")
|
||||
source_artifacts = value["source_artifacts"]
|
||||
if not isinstance(source_artifacts, list) or not source_artifacts:
|
||||
raise ValueError("recording cache source artifacts are invalid")
|
||||
artifact_ids: set[str] = set()
|
||||
artifact_keys = {
|
||||
"artifact_id",
|
||||
"media_type",
|
||||
"file_byte_length",
|
||||
"replay_byte_length",
|
||||
"mtime_ns",
|
||||
"ctime_ns",
|
||||
"sha256",
|
||||
}
|
||||
for artifact in source_artifacts:
|
||||
if not isinstance(artifact, dict) or set(artifact) != artifact_keys:
|
||||
raise ValueError("recording cache source artifact is invalid")
|
||||
artifact_id = artifact["artifact_id"]
|
||||
if (
|
||||
not isinstance(artifact_id, str)
|
||||
or SESSION_ID_PATTERN.fullmatch(artifact_id) is None
|
||||
or artifact_id in artifact_ids
|
||||
):
|
||||
raise ValueError("recording cache source artifact id is invalid")
|
||||
artifact_ids.add(artifact_id)
|
||||
if not isinstance(artifact["media_type"], str) or not artifact["media_type"]:
|
||||
raise ValueError("recording cache source media type is invalid")
|
||||
for key in (
|
||||
"file_byte_length",
|
||||
"replay_byte_length",
|
||||
"mtime_ns",
|
||||
"ctime_ns",
|
||||
):
|
||||
item = artifact[key]
|
||||
if not isinstance(item, int) or isinstance(item, bool) or item < 0:
|
||||
raise ValueError("recording cache source artifact boundary is invalid")
|
||||
if not 1 <= artifact["replay_byte_length"] <= artifact["file_byte_length"]:
|
||||
raise ValueError("recording cache source replay boundary is invalid")
|
||||
if not isinstance(artifact["sha256"], str) or not _is_sha256(artifact["sha256"]):
|
||||
raise ValueError("recording cache source digest is invalid")
|
||||
if value["primary_artifact_id"] not in artifact_ids:
|
||||
raise ValueError("recording cache primary artifact is unavailable")
|
||||
if value["timeline_end_ns"] < value["timeline_start_ns"]:
|
||||
raise ValueError("recording cache timeline bounds are invalid")
|
||||
return cast(dict[str, Any], value)
|
||||
@@ -1061,26 +1169,23 @@ def _stage_replay_prefix(
|
||||
staged_root = session_cache_root / f".source.{uuid4().hex}.tmp"
|
||||
try:
|
||||
staged_root.mkdir(mode=0o700)
|
||||
staged_source = staged_root / "mqtt.raw.k1mqtt"
|
||||
staged_metadata = staged_root / "mqtt.metadata.jsonl"
|
||||
_copy_prefix_nofollow(
|
||||
source.source,
|
||||
source.source_stat,
|
||||
staged_source,
|
||||
source.replay_byte_length,
|
||||
cancel_event=cancel_event,
|
||||
activity_callback=activity_callback,
|
||||
)
|
||||
_copy_prefix_nofollow(
|
||||
source.metadata,
|
||||
source.metadata_stat,
|
||||
staged_metadata,
|
||||
source.metadata_byte_length,
|
||||
cancel_event=cancel_event,
|
||||
activity_callback=activity_callback,
|
||||
)
|
||||
staged_primary: Path | None = None
|
||||
for artifact in source.artifacts:
|
||||
staged = staged_root / artifact.path.name
|
||||
_copy_prefix_nofollow(
|
||||
artifact.path,
|
||||
artifact.file_stat,
|
||||
staged,
|
||||
artifact.replay_byte_length,
|
||||
cancel_event=cancel_event,
|
||||
activity_callback=activity_callback,
|
||||
)
|
||||
if artifact.artifact_id == source.primary_artifact_id:
|
||||
staged_primary = staged
|
||||
if staged_primary is None:
|
||||
raise RecordingMaterializationError("staged recording has no primary artifact")
|
||||
_fsync_directory(staged_root)
|
||||
return staged_root, staged_source
|
||||
return staged_root, staged_primary
|
||||
except BaseException:
|
||||
shutil.rmtree(staged_root, ignore_errors=True)
|
||||
raise
|
||||
|
||||
+185
-126
@@ -12,11 +12,12 @@ from typing import Any, cast
|
||||
|
||||
from k1link.artifacts import utc_now_iso
|
||||
|
||||
from .legacy import discover_legacy_viewer_sessions
|
||||
from .models import (
|
||||
LayoutConflictError,
|
||||
LegacySessionCandidate,
|
||||
ObservationArtifactCandidate,
|
||||
ObservationSessionCandidate,
|
||||
RecordedMediaArtifact,
|
||||
ReplayArtifact,
|
||||
ReplayCommand,
|
||||
SessionArtifact,
|
||||
SessionDetail,
|
||||
@@ -30,6 +31,7 @@ from .models import (
|
||||
SessionSummary,
|
||||
WorkspaceLayout,
|
||||
)
|
||||
from .plugin_contract import ObservationArchiveSource
|
||||
|
||||
IDENTIFIER_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
MAX_LAYOUT_BYTES = 256 * 1024
|
||||
@@ -38,6 +40,8 @@ DATABASE_NAME = "mission-core.sqlite3"
|
||||
SCHEMA_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS observation_sessions (
|
||||
session_id TEXT PRIMARY KEY,
|
||||
plugin_id TEXT NOT NULL,
|
||||
archive_id TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL CHECK (status IN ('ready', 'interrupted', 'failed')),
|
||||
started_at_utc TEXT,
|
||||
@@ -48,8 +52,9 @@ CREATE TABLE IF NOT EXISTS observation_sessions (
|
||||
origin TEXT NOT NULL,
|
||||
source_count INTEGER NOT NULL,
|
||||
total_bytes INTEGER NOT NULL,
|
||||
replay_raw_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
replay_metadata_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
primary_replay_artifact_id TEXT,
|
||||
timeline_origin_epoch_ns INTEGER,
|
||||
timeline_origin_monotonic_ns INTEGER,
|
||||
allowed_root TEXT NOT NULL,
|
||||
session_root TEXT NOT NULL,
|
||||
created_at_utc TEXT NOT NULL,
|
||||
@@ -68,6 +73,7 @@ CREATE TABLE IF NOT EXISTS observation_session_artifacts (
|
||||
sha256 TEXT,
|
||||
integrity_status TEXT NOT NULL,
|
||||
locator TEXT NOT NULL,
|
||||
replay_byte_length INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (session_id, artifact_id)
|
||||
);
|
||||
|
||||
@@ -129,20 +135,22 @@ class SessionStore:
|
||||
self._lock = threading.RLock()
|
||||
self._initialize()
|
||||
|
||||
def import_legacy_viewer_live(self, root: Path) -> tuple[str, ...]:
|
||||
allowed_root = root.expanduser().resolve()
|
||||
candidates = discover_legacy_viewer_sessions(allowed_root)
|
||||
def reconcile_archive(self, source: ObservationArchiveSource) -> tuple[str, ...]:
|
||||
"""Reconcile one plugin-owned evidence namespace into the host catalog."""
|
||||
|
||||
allowed_root = source.root.expanduser().resolve()
|
||||
candidates = source.discover(allowed_root)
|
||||
imported: list[str] = []
|
||||
for candidate in candidates:
|
||||
self._upsert_legacy(candidate)
|
||||
self._upsert_candidate(source, candidate)
|
||||
imported.append(candidate.session_id)
|
||||
with self._lock, self._connect() as connection:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
discovered = set(imported)
|
||||
indexed = connection.execute(
|
||||
"SELECT session_id FROM observation_sessions "
|
||||
"WHERE origin = 'legacy-viewer-live' AND allowed_root = ?",
|
||||
(str(allowed_root),),
|
||||
"WHERE plugin_id = ? AND archive_id = ? AND allowed_root = ?",
|
||||
(source.plugin_id, source.archive_id, str(allowed_root)),
|
||||
).fetchall()
|
||||
stale = [row["session_id"] for row in indexed if row["session_id"] not in discovered]
|
||||
connection.executemany(
|
||||
@@ -240,29 +248,57 @@ class SessionStore:
|
||||
if not 0 <= speed <= 100:
|
||||
raise ValueError("speed must be within 0..100")
|
||||
with self._connect() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT s.allowed_root, s.session_root, s.replay_raw_bytes, "
|
||||
"s.replay_metadata_bytes, a.locator, a.sha256 "
|
||||
"FROM observation_sessions AS s "
|
||||
"JOIN observation_session_artifacts AS a ON a.session_id = s.session_id "
|
||||
"WHERE s.session_id = ? AND a.artifact_id = 'raw-mqtt'",
|
||||
session = connection.execute(
|
||||
"SELECT plugin_id, allowed_root, session_root, "
|
||||
"primary_replay_artifact_id, timeline_origin_epoch_ns, "
|
||||
"timeline_origin_monotonic_ns FROM observation_sessions WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
artifact_rows = connection.execute(
|
||||
"SELECT artifact_id, media_type, byte_length, replay_byte_length, locator, sha256 "
|
||||
"FROM observation_session_artifacts WHERE session_id = ? "
|
||||
"AND replay_byte_length > 0 ORDER BY artifact_id",
|
||||
(session_id,),
|
||||
).fetchall()
|
||||
if session is None or not artifact_rows:
|
||||
raise SessionNotReplayableError("observation session replay artifact is unavailable")
|
||||
source_path = _resolve_confined_artifact(
|
||||
Path(row["allowed_root"]),
|
||||
Path(row["session_root"]),
|
||||
Path(row["locator"]),
|
||||
primary_artifact_id = session["primary_replay_artifact_id"]
|
||||
epoch_ns = session["timeline_origin_epoch_ns"]
|
||||
monotonic_ns = session["timeline_origin_monotonic_ns"]
|
||||
if (
|
||||
not isinstance(primary_artifact_id, str)
|
||||
or not isinstance(epoch_ns, int)
|
||||
or not isinstance(monotonic_ns, int)
|
||||
):
|
||||
raise SessionNotReplayableError("observation session replay contract is incomplete")
|
||||
allowed_root = Path(session["allowed_root"])
|
||||
session_root = Path(session["session_root"])
|
||||
artifacts = tuple(
|
||||
ReplayArtifact(
|
||||
artifact_id=row["artifact_id"],
|
||||
path=_resolve_confined_artifact(
|
||||
allowed_root,
|
||||
session_root,
|
||||
Path(row["locator"]),
|
||||
),
|
||||
media_type=row["media_type"],
|
||||
file_byte_length=int(row["byte_length"]),
|
||||
replay_byte_length=int(row["replay_byte_length"]),
|
||||
expected_sha256=row["sha256"],
|
||||
)
|
||||
for row in artifact_rows
|
||||
)
|
||||
if sum(artifact.artifact_id == primary_artifact_id for artifact in artifacts) != 1:
|
||||
raise SessionNotReplayableError("observation session primary artifact is unavailable")
|
||||
return ReplayCommand(
|
||||
session_id=session_id,
|
||||
source_path=source_path,
|
||||
allowed_root=Path(row["allowed_root"]),
|
||||
session_root=Path(row["session_root"]),
|
||||
replay_byte_length=int(row["replay_raw_bytes"]),
|
||||
metadata_byte_length=int(row["replay_metadata_bytes"]),
|
||||
expected_source_sha256=row["sha256"],
|
||||
plugin_id=session["plugin_id"],
|
||||
allowed_root=allowed_root,
|
||||
session_root=session_root,
|
||||
primary_artifact_id=primary_artifact_id,
|
||||
artifacts=artifacts,
|
||||
timeline_origin_epoch_ns=epoch_ns,
|
||||
timeline_origin_monotonic_ns=monotonic_ns,
|
||||
speed=float(speed),
|
||||
loop=loop,
|
||||
)
|
||||
@@ -387,58 +423,92 @@ class SessionStore:
|
||||
def _initialize(self) -> None:
|
||||
with self._connect() as connection:
|
||||
connection.executescript(SCHEMA_SQL)
|
||||
columns = {
|
||||
session_columns = {
|
||||
row["name"]
|
||||
for row in connection.execute("PRAGMA table_info(observation_sessions)")
|
||||
}
|
||||
if "replay_raw_bytes" not in columns:
|
||||
for name, declaration in (
|
||||
("plugin_id", "TEXT NOT NULL DEFAULT ''"),
|
||||
("archive_id", "TEXT NOT NULL DEFAULT ''"),
|
||||
("primary_replay_artifact_id", "TEXT"),
|
||||
("timeline_origin_epoch_ns", "INTEGER"),
|
||||
("timeline_origin_monotonic_ns", "INTEGER"),
|
||||
):
|
||||
if name in session_columns:
|
||||
continue
|
||||
connection.execute(
|
||||
"ALTER TABLE observation_sessions "
|
||||
"ADD COLUMN replay_raw_bytes INTEGER NOT NULL DEFAULT 0"
|
||||
f"ALTER TABLE observation_sessions ADD COLUMN {name} {declaration}" # noqa: S608
|
||||
)
|
||||
if "replay_metadata_bytes" not in columns:
|
||||
artifact_columns = {
|
||||
row["name"]
|
||||
for row in connection.execute(
|
||||
"PRAGMA table_info(observation_session_artifacts)"
|
||||
)
|
||||
}
|
||||
if "replay_byte_length" not in artifact_columns:
|
||||
connection.execute(
|
||||
"ALTER TABLE observation_sessions "
|
||||
"ADD COLUMN replay_metadata_bytes INTEGER NOT NULL DEFAULT 0"
|
||||
"ALTER TABLE observation_session_artifacts "
|
||||
"ADD COLUMN replay_byte_length INTEGER NOT NULL DEFAULT 0"
|
||||
)
|
||||
connection.commit()
|
||||
with _ignore_os_error():
|
||||
self.database_path.chmod(0o600)
|
||||
|
||||
def _upsert_legacy(self, candidate: LegacySessionCandidate) -> None:
|
||||
_validate_identifier(candidate.session_id, "legacy session id")
|
||||
def _upsert_candidate(
|
||||
self,
|
||||
source: ObservationArchiveSource,
|
||||
candidate: ObservationSessionCandidate,
|
||||
) -> None:
|
||||
_validate_identifier(candidate.session_id, "observation session id")
|
||||
_validate_identifier(source.plugin_id, "device plugin id")
|
||||
_validate_identifier(source.archive_id, "observation archive id")
|
||||
allowed_root = candidate.allowed_root.resolve()
|
||||
session_root = candidate.session_root.resolve()
|
||||
if not session_root.is_relative_to(allowed_root):
|
||||
raise SessionIntegrityError("legacy session root escapes its allowed root")
|
||||
sources = _legacy_sources(candidate)
|
||||
artifacts = _legacy_artifacts(candidate, session_root)
|
||||
raise SessionIntegrityError("observation session root escapes its allowed root")
|
||||
if allowed_root != source.root.expanduser().resolve():
|
||||
raise SessionIntegrityError("plugin candidate does not belong to its archive root")
|
||||
sources = candidate.sources
|
||||
artifacts = _validated_candidate_artifacts(candidate, session_root)
|
||||
_validate_candidate_replay(candidate, artifacts)
|
||||
now = utc_now_iso()
|
||||
modalities_json = json.dumps(list(candidate.modalities), separators=(",", ":"))
|
||||
|
||||
with self._lock, self._connect() as connection:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
existing = connection.execute(
|
||||
"SELECT origin, allowed_root, session_root, created_at_utc "
|
||||
"SELECT plugin_id, archive_id, allowed_root, session_root, created_at_utc "
|
||||
"FROM observation_sessions WHERE session_id = ?",
|
||||
(candidate.session_id,),
|
||||
).fetchone()
|
||||
unclaimed_pre_plugin_row = existing is not None and (
|
||||
existing["plugin_id"] == "" and existing["archive_id"] == ""
|
||||
)
|
||||
if existing is not None and (
|
||||
existing["origin"] != "legacy-viewer-live"
|
||||
or Path(existing["allowed_root"]).resolve() != allowed_root
|
||||
Path(existing["allowed_root"]).resolve() != allowed_root
|
||||
or Path(existing["session_root"]).resolve() != session_root
|
||||
or (
|
||||
not unclaimed_pre_plugin_row
|
||||
and (
|
||||
existing["plugin_id"] != source.plugin_id
|
||||
or existing["archive_id"] != source.archive_id
|
||||
)
|
||||
)
|
||||
):
|
||||
connection.rollback()
|
||||
raise SessionIntegrityError("session id is already bound to another origin")
|
||||
raise SessionIntegrityError("session id is already bound to another archive")
|
||||
created_at = existing["created_at_utc"] if existing is not None else now
|
||||
connection.execute(
|
||||
"INSERT INTO observation_sessions "
|
||||
"(session_id, display_name, status, started_at_utc, completed_at_utc, "
|
||||
"(session_id, plugin_id, archive_id, display_name, status, "
|
||||
"started_at_utc, completed_at_utc, "
|
||||
"duration_seconds, modalities_json, replayable, origin, source_count, "
|
||||
"total_bytes, replay_raw_bytes, replay_metadata_bytes, allowed_root, "
|
||||
"total_bytes, primary_replay_artifact_id, timeline_origin_epoch_ns, "
|
||||
"timeline_origin_monotonic_ns, allowed_root, "
|
||||
"session_root, created_at_utc, updated_at_utc) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
|
||||
"ON CONFLICT(session_id) DO UPDATE SET "
|
||||
"plugin_id = excluded.plugin_id, archive_id = excluded.archive_id, "
|
||||
"display_name = excluded.display_name, status = excluded.status, "
|
||||
"started_at_utc = excluded.started_at_utc, "
|
||||
"completed_at_utc = excluded.completed_at_utc, "
|
||||
@@ -446,11 +516,14 @@ class SessionStore:
|
||||
"modalities_json = excluded.modalities_json, "
|
||||
"replayable = excluded.replayable, source_count = excluded.source_count, "
|
||||
"total_bytes = excluded.total_bytes, "
|
||||
"replay_raw_bytes = excluded.replay_raw_bytes, "
|
||||
"replay_metadata_bytes = excluded.replay_metadata_bytes, "
|
||||
"primary_replay_artifact_id = excluded.primary_replay_artifact_id, "
|
||||
"timeline_origin_epoch_ns = excluded.timeline_origin_epoch_ns, "
|
||||
"timeline_origin_monotonic_ns = excluded.timeline_origin_monotonic_ns, "
|
||||
"updated_at_utc = excluded.updated_at_utc",
|
||||
(
|
||||
candidate.session_id,
|
||||
source.plugin_id,
|
||||
source.archive_id,
|
||||
candidate.display_name,
|
||||
candidate.status,
|
||||
candidate.started_at_utc,
|
||||
@@ -458,11 +531,12 @@ class SessionStore:
|
||||
candidate.duration_seconds,
|
||||
modalities_json,
|
||||
int(candidate.replayable),
|
||||
"legacy-viewer-live",
|
||||
source.archive_id,
|
||||
len(sources),
|
||||
candidate.total_bytes,
|
||||
candidate.replay_raw_byte_length,
|
||||
candidate.replay_metadata_byte_length,
|
||||
candidate.primary_replay_artifact_id,
|
||||
candidate.timeline_origin_epoch_ns,
|
||||
candidate.timeline_origin_monotonic_ns,
|
||||
str(allowed_root),
|
||||
str(session_root),
|
||||
created_at,
|
||||
@@ -480,9 +554,20 @@ class SessionStore:
|
||||
connection.executemany(
|
||||
"INSERT INTO observation_session_artifacts "
|
||||
"(session_id, artifact_id, kind, media_type, byte_length, sha256, "
|
||||
"integrity_status, locator) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
"integrity_status, locator, replay_byte_length) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
[
|
||||
(candidate.session_id, *artifact)
|
||||
(
|
||||
candidate.session_id,
|
||||
artifact.artifact_id,
|
||||
artifact.kind,
|
||||
artifact.media_type,
|
||||
artifact.byte_length,
|
||||
artifact.sha256,
|
||||
artifact.integrity_status,
|
||||
str(artifact.locator),
|
||||
artifact.replay_byte_length,
|
||||
)
|
||||
for artifact in artifacts
|
||||
],
|
||||
)
|
||||
@@ -519,82 +604,56 @@ class SessionStore:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _legacy_sources(candidate: LegacySessionCandidate) -> tuple[SessionSource, ...]:
|
||||
rows: list[SessionSource] = []
|
||||
if "point-cloud" in candidate.modalities:
|
||||
rows.append(
|
||||
SessionSource(
|
||||
source_id="sensor.lidar.primary",
|
||||
semantic_channel_id="spatial.point-cloud.recorded",
|
||||
modality="point-cloud",
|
||||
status="recorded",
|
||||
seekable=candidate.replayable,
|
||||
artifact_id="raw-mqtt",
|
||||
)
|
||||
)
|
||||
if "trajectory" in candidate.modalities:
|
||||
rows.append(
|
||||
SessionSource(
|
||||
source_id="spatial.trajectory",
|
||||
semantic_channel_id="spatial.pose.recorded",
|
||||
modality="trajectory",
|
||||
status="recorded",
|
||||
seekable=candidate.replayable,
|
||||
artifact_id="raw-mqtt",
|
||||
)
|
||||
)
|
||||
rows.extend(
|
||||
SessionSource(
|
||||
source_id=media.source_id,
|
||||
semantic_channel_id="camera.video.recorded",
|
||||
modality="video",
|
||||
status="recorded",
|
||||
seekable=True,
|
||||
artifact_id=media.artifact_id,
|
||||
)
|
||||
for media in candidate.media_sources
|
||||
)
|
||||
if len({source.source_id for source in rows}) != len(rows):
|
||||
raise SessionIntegrityError("legacy session contains duplicate source identifiers")
|
||||
return tuple(rows)
|
||||
|
||||
|
||||
def _legacy_artifacts(
|
||||
candidate: LegacySessionCandidate,
|
||||
def _validated_candidate_artifacts(
|
||||
candidate: ObservationSessionCandidate,
|
||||
session_root: Path,
|
||||
) -> tuple[tuple[str, str, str, int, str | None, str, str], ...]:
|
||||
artifacts: list[tuple[str, str, str, int, str | None, str, str]] = [
|
||||
(
|
||||
"raw-mqtt",
|
||||
"raw-transport",
|
||||
"application/x-nodedc-k1mqtt",
|
||||
candidate.raw_byte_length,
|
||||
candidate.raw_sha256,
|
||||
candidate.raw_integrity_status,
|
||||
str(candidate.raw_path),
|
||||
)
|
||||
]
|
||||
for media in candidate.media_sources:
|
||||
) -> tuple[ObservationArtifactCandidate, ...]:
|
||||
artifacts = candidate.artifacts
|
||||
artifact_ids: set[str] = set()
|
||||
for artifact in artifacts:
|
||||
_validate_identifier(artifact.artifact_id, "observation artifact id")
|
||||
if artifact.artifact_id in artifact_ids:
|
||||
raise SessionIntegrityError("observation session contains duplicate artifacts")
|
||||
artifact_ids.add(artifact.artifact_id)
|
||||
if artifact.byte_length < 0 or not 0 <= artifact.replay_byte_length <= artifact.byte_length:
|
||||
raise SessionIntegrityError("observation artifact has invalid byte boundaries")
|
||||
try:
|
||||
locator = media.locator.resolve(strict=True)
|
||||
locator = artifact.locator.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise SessionIntegrityError("legacy video artifact is missing") from exc
|
||||
if not locator.is_dir() or not locator.is_relative_to(session_root):
|
||||
raise SessionIntegrityError("legacy video artifact escapes its session root")
|
||||
artifacts.append(
|
||||
(
|
||||
media.artifact_id,
|
||||
"recorded-video",
|
||||
"video/mp4",
|
||||
media.byte_length,
|
||||
None,
|
||||
"validated-structure",
|
||||
str(locator),
|
||||
)
|
||||
)
|
||||
if len({artifact[0] for artifact in artifacts}) != len(artifacts):
|
||||
raise SessionIntegrityError("legacy session contains duplicate artifact identifiers")
|
||||
return tuple(artifacts)
|
||||
raise SessionIntegrityError("observation artifact is missing") from exc
|
||||
if not locator.is_relative_to(session_root) or not (locator.is_file() or locator.is_dir()):
|
||||
raise SessionIntegrityError("observation artifact escapes its session root")
|
||||
if artifact.replay_byte_length > 0 and not locator.is_file():
|
||||
raise SessionIntegrityError("replay input artifact must be a regular file")
|
||||
source_ids: set[str] = set()
|
||||
for source in candidate.sources:
|
||||
_validate_identifier(source.source_id, "observation source id")
|
||||
if source.source_id in source_ids:
|
||||
raise SessionIntegrityError("observation session contains duplicate sources")
|
||||
if source.artifact_id not in artifact_ids:
|
||||
raise SessionIntegrityError("observation source references an unknown artifact")
|
||||
source_ids.add(source.source_id)
|
||||
return artifacts
|
||||
|
||||
|
||||
def _validate_candidate_replay(
|
||||
candidate: ObservationSessionCandidate,
|
||||
artifacts: tuple[ObservationArtifactCandidate, ...],
|
||||
) -> None:
|
||||
primary_id = candidate.primary_replay_artifact_id
|
||||
replay_artifacts = tuple(artifact for artifact in artifacts if artifact.replay_byte_length > 0)
|
||||
if candidate.replayable:
|
||||
if (
|
||||
primary_id is None
|
||||
or sum(artifact.artifact_id == primary_id for artifact in replay_artifacts) != 1
|
||||
or candidate.timeline_origin_epoch_ns is None
|
||||
or candidate.timeline_origin_monotonic_ns is None
|
||||
):
|
||||
raise SessionIntegrityError("replayable observation contract is incomplete")
|
||||
if candidate.timeline_origin_epoch_ns < 0 or candidate.timeline_origin_monotonic_ns < 0:
|
||||
raise SessionIntegrityError("observation timeline origin is invalid")
|
||||
elif primary_id is not None or replay_artifacts:
|
||||
raise SessionIntegrityError("non-replayable observation declares replay artifacts")
|
||||
|
||||
|
||||
def _summary_from_row(row: sqlite3.Row) -> SessionSummary:
|
||||
|
||||
@@ -1,15 +1,11 @@
|
||||
"""Live/replay visualization bridge for verified K1 MQTT streams."""
|
||||
"""Mission Core viewer consumers and recorded-viewer contracts."""
|
||||
|
||||
from k1link.viewer.messages import StreamMessage
|
||||
from k1link.viewer.replay import (
|
||||
ReplayFormatError,
|
||||
detect_replay_format,
|
||||
iter_replay_messages,
|
||||
)
|
||||
from .recorded import RecordedBlueprintError, recorded_blueprint_rrd
|
||||
from .rerun_bridge import RerunBridge, RerunSceneSettings
|
||||
|
||||
__all__ = [
|
||||
"ReplayFormatError",
|
||||
"StreamMessage",
|
||||
"detect_replay_format",
|
||||
"iter_replay_messages",
|
||||
"RecordedBlueprintError",
|
||||
"RerunBridge",
|
||||
"RerunSceneSettings",
|
||||
"recorded_blueprint_rrd",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Vendor-neutral Rerun blueprint for prepared observation recordings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import suppress
|
||||
from uuid import UUID
|
||||
|
||||
import rerun as rr
|
||||
from rerun import blueprint as rrb
|
||||
|
||||
from k1link.viewer.rerun_bridge import RerunSceneSettings, _parse_hex_color
|
||||
|
||||
APPLICATION_ID = "nodedc_mission_core_recorded"
|
||||
SESSION_TIMELINE = "session_time"
|
||||
|
||||
RECORDED_SPATIAL_VIEW_ID = UUID("5f5f11d5-3b0a-4a81-887b-2be767cba1c0")
|
||||
RECORDED_ROOT_CONTAINER_ID = UUID("b02f2aca-8471-4dcb-b786-53df5a320fc8")
|
||||
RECORDED_POINTS_VISUALIZER_ID = UUID("ca037ec0-8761-4417-86ee-846fa2875303")
|
||||
|
||||
|
||||
class RecordedBlueprintError(RuntimeError):
|
||||
"""A viewer blueprint update could not be serialized safely."""
|
||||
|
||||
|
||||
def recorded_blueprint(
|
||||
settings: RerunSceneSettings,
|
||||
*,
|
||||
include_initial_playback_state: bool = True,
|
||||
) -> rrb.Blueprint:
|
||||
accumulation = max(0.0, settings.accumulation_seconds)
|
||||
time_ranges: list[rr.VisibleTimeRange] | None = None
|
||||
if accumulation > 0:
|
||||
time_ranges = [
|
||||
rr.VisibleTimeRange(
|
||||
SESSION_TIMELINE,
|
||||
start=rr.TimeRangeBoundary.cursor_relative(seconds=-accumulation),
|
||||
end=rr.TimeRangeBoundary.cursor_relative(),
|
||||
)
|
||||
]
|
||||
point_visualizer = rr.Points3D.from_fields(
|
||||
radii=rr.Radius.ui_points(settings.point_size),
|
||||
colors=(
|
||||
[_parse_hex_color(settings.custom_color)]
|
||||
if settings.palette == "custom"
|
||||
else None
|
||||
),
|
||||
).visualizer()
|
||||
point_visualizer.id = RECORDED_POINTS_VISUALIZER_ID
|
||||
spatial_view = rrb.Spatial3DView(
|
||||
origin="/world",
|
||||
name="Пространственная сцена",
|
||||
background=[7, 8, 10, 255],
|
||||
line_grid=rrb.LineGrid3D(
|
||||
visible=settings.show_grid,
|
||||
color=[86, 91, 99, 110],
|
||||
stroke_width=0.75,
|
||||
),
|
||||
overrides={
|
||||
"/world/points": [
|
||||
rrb.EntityBehavior(visible=settings.show_points),
|
||||
point_visualizer,
|
||||
],
|
||||
"/world/trajectory": rrb.EntityBehavior(
|
||||
visible=settings.show_trajectory,
|
||||
),
|
||||
},
|
||||
time_ranges=time_ranges,
|
||||
)
|
||||
spatial_view.id = RECORDED_SPATIAL_VIEW_ID
|
||||
root_container = rrb.Tabs(spatial_view)
|
||||
root_container.id = RECORDED_ROOT_CONTAINER_ID
|
||||
|
||||
if include_initial_playback_state:
|
||||
return rrb.Blueprint(
|
||||
root_container,
|
||||
rrb.TimePanel(
|
||||
timeline=SESSION_TIMELINE,
|
||||
play_state="paused",
|
||||
state="hidden",
|
||||
),
|
||||
auto_layout=False,
|
||||
auto_views=False,
|
||||
collapse_panels=True,
|
||||
)
|
||||
return rrb.Blueprint(
|
||||
root_container,
|
||||
auto_layout=False,
|
||||
auto_views=False,
|
||||
collapse_panels=False,
|
||||
)
|
||||
|
||||
|
||||
def recorded_blueprint_rrd(
|
||||
settings: RerunSceneSettings,
|
||||
*,
|
||||
application_id: str = APPLICATION_ID,
|
||||
recording_id: str,
|
||||
) -> bytes:
|
||||
"""Serialize a bounded active blueprint update without recorded data."""
|
||||
|
||||
recording = rr.RecordingStream(
|
||||
application_id,
|
||||
recording_id=recording_id,
|
||||
send_properties=False,
|
||||
)
|
||||
stream = rr.binary_stream(recording)
|
||||
try:
|
||||
recording.send_blueprint(
|
||||
recorded_blueprint(
|
||||
settings,
|
||||
include_initial_playback_state=False,
|
||||
),
|
||||
make_active=True,
|
||||
make_default=False,
|
||||
)
|
||||
payload = stream.read(flush=True, flush_timeout_sec=5.0)
|
||||
except Exception as exc:
|
||||
raise RecordedBlueprintError("failed to serialize recorded blueprint") from exc
|
||||
finally:
|
||||
with suppress(Exception):
|
||||
recording.disconnect()
|
||||
if not payload or not payload.startswith(b"RRF2") or len(payload) > 1_048_576:
|
||||
raise RecordedBlueprintError("serialized recorded blueprint is invalid")
|
||||
return payload
|
||||
+9
-14
@@ -22,10 +22,7 @@ from k1link.sessions import (
|
||||
SessionRecordingMaterializer,
|
||||
SessionRecordingPreparationManager,
|
||||
SessionStore,
|
||||
recover_stale_active_session_marker,
|
||||
resolve_missioncore_evidence_dir,
|
||||
)
|
||||
from k1link.web.camera_archive import recover_incomplete_camera_archives
|
||||
from k1link.web.device_plugin_composition import load_installed_device_plugins
|
||||
from k1link.web.plugin_catalog import DevicePluginCatalog, PluginCatalogError
|
||||
from k1link.web.plugin_runtime import (
|
||||
@@ -42,13 +39,14 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
|
||||
INVALID_REQUEST_DETAIL = "Некорректные параметры запроса."
|
||||
|
||||
|
||||
legacy_observation_sessions_root = REPOSITORY_ROOT / "sessions"
|
||||
observation_sessions_root = resolve_missioncore_evidence_dir(REPOSITORY_ROOT)
|
||||
plugin_environment = load_installed_device_plugins(REPOSITORY_ROOT)
|
||||
plugin_catalog: DevicePluginCatalog = plugin_environment.catalog
|
||||
plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher
|
||||
session_store = SessionStore(REPOSITORY_ROOT)
|
||||
session_recording_materializer = SessionRecordingMaterializer(session_store.data_dir)
|
||||
session_recording_materializer = SessionRecordingMaterializer(
|
||||
session_store.data_dir,
|
||||
exporters=plugin_environment.recording_exporters,
|
||||
)
|
||||
session_recorded_media_inspector = RecordedMediaInspector(
|
||||
session_store.data_dir / "recorded-media-preparations"
|
||||
)
|
||||
@@ -76,8 +74,9 @@ def refresh_observation_catalog() -> tuple[str, ...]:
|
||||
"""Discover completed or recoverable local evidence without copying payloads."""
|
||||
|
||||
imported = [
|
||||
*session_store.import_legacy_viewer_live(legacy_observation_sessions_root),
|
||||
*session_store.import_legacy_viewer_live(observation_sessions_root),
|
||||
session_id
|
||||
for archive in plugin_environment.observation_archives
|
||||
for session_id in session_store.reconcile_archive(archive)
|
||||
]
|
||||
return tuple(dict.fromkeys(imported))
|
||||
|
||||
@@ -138,12 +137,8 @@ async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
# Recovery is intentionally a one-shot startup phase. The archive
|
||||
# helper owns a cross-process lease, while ordinary catalog requests
|
||||
# only perform discovery and therefore never touch a live writer.
|
||||
await asyncio.to_thread(recover_stale_active_session_marker, observation_sessions_root)
|
||||
for sessions_root in (
|
||||
legacy_observation_sessions_root,
|
||||
observation_sessions_root,
|
||||
):
|
||||
await asyncio.to_thread(recover_incomplete_camera_archives, sessions_root)
|
||||
for archive in plugin_environment.observation_archives:
|
||||
await asyncio.to_thread(archive.recover, archive.root)
|
||||
# Full evidence discovery, hashing and RRD queue reconciliation can be
|
||||
# expensive on field captures. Start it immediately in the background
|
||||
# instead of holding the ASGI startup gate.
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from k1link.sessions.plugin_contract import ObservationArchiveSource, RecordingExporter
|
||||
from k1link.web.plugin_catalog import DevicePluginCatalog, DevicePluginManifest
|
||||
from k1link.web.plugin_runtime import (
|
||||
DevicePluginDispatcher,
|
||||
@@ -25,6 +26,23 @@ class InstalledDevicePluginEnvironment:
|
||||
legacy_routers: tuple[APIRouter, ...]
|
||||
_contributions: tuple[DevicePluginRuntimeContribution, ...]
|
||||
|
||||
@property
|
||||
def observation_archives(self) -> tuple[ObservationArchiveSource, ...]:
|
||||
return tuple(
|
||||
archive
|
||||
for contribution in self._contributions
|
||||
if contribution.observation is not None
|
||||
for archive in contribution.observation.archives
|
||||
)
|
||||
|
||||
@property
|
||||
def recording_exporters(self) -> dict[str, RecordingExporter]:
|
||||
return {
|
||||
contribution.adapter.plugin_id: contribution.observation.recording_exporter
|
||||
for contribution in self._contributions
|
||||
if contribution.observation is not None
|
||||
}
|
||||
|
||||
def close(self) -> None:
|
||||
cleanup_errors = _close_contributions(self._contributions)
|
||||
if cleanup_errors:
|
||||
@@ -103,6 +121,31 @@ def _load_contribution(
|
||||
raise DevicePluginCompositionError(
|
||||
f"Device-plugin manifest/runtime actions mismatch for {adapter.plugin_id}"
|
||||
)
|
||||
observation = contribution.observation
|
||||
if observation is not None:
|
||||
if not observation.archives:
|
||||
raise DevicePluginCompositionError(
|
||||
f"Device-plugin observation contribution is empty for {adapter.plugin_id}"
|
||||
)
|
||||
archive_ids: set[str] = set()
|
||||
archive_roots: set[Path] = set()
|
||||
for archive in observation.archives:
|
||||
if archive.plugin_id != adapter.plugin_id:
|
||||
raise DevicePluginCompositionError(
|
||||
"Device-plugin observation/runtime id mismatch: "
|
||||
f"{archive.plugin_id} != {adapter.plugin_id}"
|
||||
)
|
||||
if archive.archive_id in archive_ids:
|
||||
raise DevicePluginCompositionError(
|
||||
f"Duplicate observation archive id for {adapter.plugin_id}"
|
||||
)
|
||||
resolved_root = archive.root.expanduser().resolve()
|
||||
if resolved_root in archive_roots:
|
||||
raise DevicePluginCompositionError(
|
||||
f"Duplicate observation archive root for {adapter.plugin_id}"
|
||||
)
|
||||
archive_ids.add(archive.archive_id)
|
||||
archive_roots.add(resolved_root)
|
||||
except Exception as exc:
|
||||
_add_cleanup_notes(exc, _close_contributions((contribution,)))
|
||||
raise
|
||||
|
||||
@@ -2,11 +2,16 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Protocol
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter
|
||||
from missioncore_plugin_sdk.v0alpha2 import RuntimeActionInvocation, RuntimeActionResult
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from k1link.sessions.plugin_contract import ObservationRuntimeContribution
|
||||
|
||||
STATE_READ_ACTION_ID = "state.read"
|
||||
|
||||
|
||||
@@ -20,7 +25,7 @@ class DevicePluginActionAdapter(Protocol):
|
||||
plugin_id: str
|
||||
action_ids: frozenset[str]
|
||||
|
||||
async def invoke(self, action_id: str, payload: Mapping[str, Any]) -> dict[str, Any]: ...
|
||||
async def invoke(self, invocation: RuntimeActionInvocation) -> Mapping[str, Any]: ...
|
||||
|
||||
|
||||
def _noop() -> None:
|
||||
@@ -33,6 +38,7 @@ class DevicePluginRuntimeContribution:
|
||||
|
||||
adapter: DevicePluginActionAdapter
|
||||
legacy_routers: tuple[APIRouter, ...] = ()
|
||||
observation: ObservationRuntimeContribution | None = None
|
||||
close: Callable[[], None] = _noop
|
||||
|
||||
|
||||
@@ -71,8 +77,33 @@ class DevicePluginDispatcher:
|
||||
adapter = self._adapters.get(plugin_id)
|
||||
if adapter is None:
|
||||
raise PluginNotFoundError(f"Device plugin is not installed: {plugin_id}")
|
||||
if action_id not in adapter.action_ids:
|
||||
raise PluginActionNotFoundError(
|
||||
f"Device plugin {plugin_id} does not declare action {action_id}"
|
||||
)
|
||||
return await adapter.invoke(action_id, payload)
|
||||
return await invoke_device_plugin_adapter(adapter, action_id, payload)
|
||||
|
||||
|
||||
async def invoke_device_plugin_adapter(
|
||||
adapter: DevicePluginActionAdapter,
|
||||
action_id: str,
|
||||
payload: Mapping[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Validate SDK request/result envelopes around one plugin action call."""
|
||||
|
||||
if action_id not in adapter.action_ids:
|
||||
raise PluginActionNotFoundError(
|
||||
f"Device plugin {adapter.plugin_id} does not declare action {action_id}"
|
||||
)
|
||||
invocation = RuntimeActionInvocation(
|
||||
invocation_id=uuid4().hex,
|
||||
plugin_id=adapter.plugin_id,
|
||||
action_id=action_id,
|
||||
requested_at=datetime.now(UTC),
|
||||
parameters=dict(payload),
|
||||
)
|
||||
output = await adapter.invoke(invocation)
|
||||
result = RuntimeActionResult(
|
||||
invocation_id=invocation.invocation_id,
|
||||
plugin_id=invocation.plugin_id,
|
||||
action_id=invocation.action_id,
|
||||
completed_at=datetime.now(UTC),
|
||||
output=dict(output),
|
||||
)
|
||||
return dict(result.output)
|
||||
|
||||
@@ -31,16 +31,15 @@ from k1link.sessions import (
|
||||
SessionStore,
|
||||
validate_recorded_media_timeline,
|
||||
)
|
||||
from k1link.viewer.rerun_bridge import RerunSceneSettings
|
||||
from k1link.viewer.rrd_export import (
|
||||
from k1link.viewer.recorded import (
|
||||
APPLICATION_ID as RECORDED_APPLICATION_ID,
|
||||
)
|
||||
from k1link.viewer.rrd_export import (
|
||||
RrdExportError,
|
||||
from k1link.viewer.recorded import (
|
||||
RecordedBlueprintError,
|
||||
recorded_blueprint_rrd,
|
||||
)
|
||||
from k1link.viewer.rerun_bridge import RerunSceneSettings
|
||||
|
||||
DEFAULT_REPLAY_PLUGIN_ID = "nodedc.device.xgrids-lixelkity-k1"
|
||||
DEFAULT_REPLAY_ACTION_ID = "stream.start-replay"
|
||||
SAFE_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$")
|
||||
SAFE_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
@@ -234,7 +233,6 @@ def build_session_router(
|
||||
recording_preparation_manager: SessionRecordingPreparationManager | None = None,
|
||||
media_inspector: RecordedMediaInspector | None = None,
|
||||
allow_synchronous_recording_fallback: bool = False,
|
||||
replay_plugin_id: str = DEFAULT_REPLAY_PLUGIN_ID,
|
||||
replay_action_id: str = DEFAULT_REPLAY_ACTION_ID,
|
||||
) -> APIRouter:
|
||||
"""Build generic host APIs without exposing filesystem locators to clients."""
|
||||
@@ -388,7 +386,7 @@ def build_session_router(
|
||||
"schema_version": "missioncore.observation-session-replay/v1",
|
||||
"launch": {
|
||||
"kind": "plugin-action",
|
||||
"plugin_id": replay_plugin_id,
|
||||
"plugin_id": command.plugin_id,
|
||||
"action_id": replay_action_id,
|
||||
"session_id": command.session_id,
|
||||
"speed": command.speed,
|
||||
@@ -671,7 +669,7 @@ def build_session_router(
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except (SessionNotReplayableError, SessionIntegrityError) as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
except RrdExportError as exc:
|
||||
except RecordedBlueprintError as exc:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Не удалось подготовить настройки визуализатора.",
|
||||
|
||||
Reference in New Issue
Block a user