feat(device-plugins): add profiled K1 lifecycle and canonical data plane

This commit is contained in:
DCCONSTRUCTIONS
2026-07-16 19:44:06 +03:00
parent 19ab973110
commit e6f7648b84
45 changed files with 6401 additions and 440 deletions
+1 -2
View File
@@ -234,8 +234,7 @@ def summarize_mqtt_streams(
"""Stream a raw MQTT capture into an aggregate-only, coordinate-free summary."""
if not 1 <= max_payload_bytes <= MAX_STREAM_SUMMARY_PAYLOAD_BYTES:
raise ValueError(
"max_payload_bytes must be between 1 and "
f"{MAX_STREAM_SUMMARY_PAYLOAD_BYTES}"
f"max_payload_bytes must be between 1 and {MAX_STREAM_SUMMARY_PAYLOAD_BYTES}"
)
capture_path = capture.expanduser()
+5 -13
View File
@@ -179,9 +179,7 @@ async def provision_wifi_once(
async with BleakClient(device, timeout=timeout_seconds, pair=False) as client:
device_name = client.name
service = client.services.get_service(SERVICE_UUID)
write_characteristic = client.services.get_characteristic(
WRITE_CHARACTERISTIC_UUID
)
write_characteristic = client.services.get_characteristic(WRITE_CHARACTERISTIC_UUID)
status_characteristic = client.services.get_characteristic(
STATUS_CHARACTERISTIC_UUID
)
@@ -189,13 +187,11 @@ async def provision_wifi_once(
raise ValueError(f"Reviewed K1 service not found: {SERVICE_UUID}")
if write_characteristic is None:
raise ValueError(
"Reviewed K1 write characteristic not found: "
f"{WRITE_CHARACTERISTIC_UUID}"
f"Reviewed K1 write characteristic not found: {WRITE_CHARACTERISTIC_UUID}"
)
if status_characteristic is None:
raise ValueError(
"Reviewed K1 status characteristic not found: "
f"{STATUS_CHARACTERISTIC_UUID}"
f"Reviewed K1 status characteristic not found: {STATUS_CHARACTERISTIC_UUID}"
)
if write_characteristic.service_uuid != service.uuid:
raise ValueError("K1 write characteristic is attached to an unexpected service")
@@ -207,9 +203,7 @@ async def provision_wifi_once(
raise ValueError("Reviewed K1 status characteristic is not readable")
properties = set(write_characteristic.properties)
max_without_response = (
write_characteristic.max_write_without_response_size
)
max_without_response = write_characteristic.max_write_without_response_size
resolved_write_mode: ResolvedWriteMode
if write_mode == "auto":
if "write-without-response" in properties:
@@ -276,9 +270,7 @@ async def provision_wifi_once(
"status_characteristic_uuid": status_characteristic.uuid,
"operation": "single_reviewed_wifi_provisioning_write",
"write_mode": resolved_write_mode,
"write_without_response_advertised": (
"write-without-response" in properties
),
"write_without_response_advertised": ("write-without-response" in properties),
"max_write_without_response_size": max_without_response,
"frame_length": len(frame),
"baseline_status": baseline,
+26
View File
@@ -0,0 +1,26 @@
"""Decoded in-process projections consumed by Mission Core visualizers.
These views are not wire contracts. Portable plugin/host envelopes live in the
versioned Plugin SDK; an extraction boundary can hydrate those envelopes into
these allocation-conscious representations for local consumers.
"""
from k1link.data_plane.views import (
ConsumerFrameContext,
DecodedDataPlaneView,
DecodedDeviceStatusView,
DecodedPointCloudView,
DecodedPoseView,
NormalizationError,
StatusAttribute,
)
__all__ = [
"ConsumerFrameContext",
"DecodedDataPlaneView",
"DecodedDeviceStatusView",
"DecodedPointCloudView",
"DecodedPoseView",
"NormalizationError",
"StatusAttribute",
]
+128
View File
@@ -0,0 +1,128 @@
from __future__ import annotations
import math
from dataclasses import dataclass
class NormalizationError(ValueError):
"""A transport message matched a known channel but could not be normalized."""
@dataclass(frozen=True, slots=True)
class ConsumerFrameContext:
"""Transport-neutral provenance shared by decoded consumer views.
Raw channel names and payloads deliberately do not cross this boundary. The
byte count is retained for operational metrics, while opaque device/session
aliases are optional because older streams do not carry them. They are
deliberately named as source aliases: a vendor header must never become a
Mission Core device identity merely by crossing the decode boundary.
"""
sequence: int
captured_at_epoch_ns: int
received_monotonic_ns: int | None
processing_started_monotonic_ns: int
encoded_size_bytes: int
live: bool
source_device_alias: str | None = None
source_session_alias: str | None = None
def __post_init__(self) -> None:
if self.sequence < 1:
raise ValueError("frame sequence must be positive")
if self.captured_at_epoch_ns < 0:
raise ValueError("capture time must be non-negative")
if self.received_monotonic_ns is not None and self.received_monotonic_ns < 0:
raise ValueError("receive monotonic time must be non-negative")
if self.processing_started_monotonic_ns < 0:
raise ValueError("processing start time must be non-negative")
if self.encoded_size_bytes < 0:
raise ValueError("encoded size must be non-negative")
@dataclass(frozen=True, slots=True)
class DecodedPointCloudView:
"""In-process point-cloud projection in a named Cartesian frame.
This is a consumer view, not the portable SDK ``PointCloudFrame`` contract.
"""
context: ConsumerFrameContext
frame_id: str
positions_xyz: tuple[tuple[float, float, float], ...]
intensities: bytes | None = None
colors_rgb: bytes | None = None
def __post_init__(self) -> None:
if not self.frame_id:
raise ValueError("point-cloud frame_id must not be empty")
point_count = len(self.positions_xyz)
if self.intensities is not None and len(self.intensities) != point_count:
raise ValueError("intensity count must equal point count")
if self.colors_rgb is not None and len(self.colors_rgb) != point_count * 3:
raise ValueError("RGB byte count must equal point count * 3")
if not all(
len(position) == 3 and all(math.isfinite(value) for value in position)
for position in self.positions_xyz
):
raise ValueError("point positions must contain finite xyz triples")
@property
def point_count(self) -> int:
return len(self.positions_xyz)
@dataclass(frozen=True, slots=True)
class DecodedPoseView:
"""In-process pose projection in a named Cartesian coordinate frame."""
context: ConsumerFrameContext
frame_id: str
child_frame_id: str
position_xyz: tuple[float, float, float]
orientation_xyzw: tuple[float, float, float, float]
def __post_init__(self) -> None:
if not self.frame_id or not self.child_frame_id:
raise ValueError("pose frame identifiers must not be empty")
values = (*self.position_xyz, *self.orientation_xyzw)
if not all(math.isfinite(value) for value in values):
raise ValueError("pose values must be finite")
@dataclass(frozen=True, slots=True)
class StatusAttribute:
"""One stable, normalized status attribute."""
name: str
value: bool | int | float | str
def __post_init__(self) -> None:
if not self.name:
raise ValueError("status attribute name must not be empty")
if isinstance(self.value, float) and not math.isfinite(self.value):
raise ValueError("status attribute float must be finite")
@dataclass(frozen=True, slots=True)
class DecodedDeviceStatusView:
"""An in-process device-status projection.
The current verified K1 profile does not yet decode its status topic. The
view exists so future verified status codecs do not alter consumers.
"""
context: ConsumerFrameContext
state: str
attributes: tuple[StatusAttribute, ...] = ()
def __post_init__(self) -> None:
if not self.state:
raise ValueError("device status state must not be empty")
names = [attribute.name for attribute in self.attributes]
if len(names) != len(set(names)):
raise ValueError("device status attribute names must be unique")
DecodedDataPlaneView = DecodedPointCloudView | DecodedPoseView | DecodedDeviceStatusView
+1 -1
View File
@@ -130,7 +130,7 @@ class CaptureFrame:
@dataclass(frozen=True, slots=True)
class CapturedMqttMessage:
"""A message made durable by the raw writer and ready for live preview."""
"""A message flushed to the raw writer before it is exposed to live preview."""
sequence: int
topic: str
+182
View File
@@ -0,0 +1,182 @@
from __future__ import annotations
from typing import Protocol
from k1link.data_plane import (
ConsumerFrameContext,
DecodedDataPlaneView,
DecodedPointCloudView,
DecodedPoseView,
NormalizationError,
)
from k1link.protocol.streams import (
LegacyPointCloudFrame,
LegacyPoseFrame,
LioPointCloudFrame,
LioPoseFrame,
StreamDecodeError,
decode_legacy_pointcloud,
decode_legacy_pose,
decode_lio_pcl,
decode_lio_pose,
)
CANONICAL_MAP_FRAME = "map"
CANONICAL_SENSOR_FRAME = "sensor"
class K1TransportMessage(Protocol):
"""Minimum raw transport shape accepted by the K1 normalizer."""
@property
def sequence(self) -> int: ...
@property
def topic(self) -> str: ...
@property
def payload(self) -> bytes: ...
@property
def received_at_epoch_ns(self) -> int: ...
@property
def received_monotonic_ns(self) -> int | None: ...
@property
def source(self) -> str: ...
def normalize_k1_message(
message: K1TransportMessage,
*,
processing_started_monotonic_ns: int,
) -> DecodedDataPlaneView | None:
"""Normalize one verified K1 transport message.
Unknown channels intentionally return ``None``. A known channel with an
invalid payload raises a transport-neutral ``NormalizationError``. Neither
raw channel names nor raw bytes are retained in the returned consumer view.
"""
try:
if message.topic.endswith("/lio_pcl"):
return _normalize_lio_points(
decode_lio_pcl(message.payload),
_context(message, processing_started_monotonic_ns),
)
if message.topic == "RealtimePointcloud":
return _normalize_legacy_points(
decode_legacy_pointcloud(message.payload),
_context(message, processing_started_monotonic_ns),
)
if message.topic.endswith("/lio_pose"):
return _normalize_lio_pose(
decode_lio_pose(message.payload),
_context(message, processing_started_monotonic_ns),
)
if message.topic == "RealtimePath":
return _normalize_legacy_pose(
decode_legacy_pose(message.payload),
_context(message, processing_started_monotonic_ns),
)
except StreamDecodeError as exc:
raise NormalizationError("known K1 data-plane message failed validation") from exc
return None
def _context(
message: K1TransportMessage,
processing_started_monotonic_ns: int,
) -> ConsumerFrameContext:
return ConsumerFrameContext(
sequence=message.sequence,
captured_at_epoch_ns=message.received_at_epoch_ns,
received_monotonic_ns=message.received_monotonic_ns,
processing_started_monotonic_ns=processing_started_monotonic_ns,
encoded_size_bytes=len(message.payload),
live=message.source == "live_mqtt",
)
def _with_device_context(
context: ConsumerFrameContext,
*,
source_device_alias: str,
source_session_alias: str,
) -> ConsumerFrameContext:
return ConsumerFrameContext(
sequence=context.sequence,
captured_at_epoch_ns=context.captured_at_epoch_ns,
received_monotonic_ns=context.received_monotonic_ns,
processing_started_monotonic_ns=context.processing_started_monotonic_ns,
encoded_size_bytes=context.encoded_size_bytes,
live=context.live,
source_device_alias=source_device_alias or None,
source_session_alias=source_session_alias or None,
)
def _normalize_lio_points(
frame: LioPointCloudFrame,
context: ConsumerFrameContext,
) -> DecodedPointCloudView:
context = _with_device_context(
context,
source_device_alias=frame.header.device_id,
source_session_alias=frame.header.session_id,
)
positions = tuple(point.scaled_xyz(frame.header.scaler) for point in frame.points)
intensities = bytes(point.intensity for point in frame.points)
return DecodedPointCloudView(
context=context,
frame_id=CANONICAL_MAP_FRAME,
positions_xyz=positions,
intensities=intensities,
)
def _normalize_legacy_points(
frame: LegacyPointCloudFrame,
context: ConsumerFrameContext,
) -> DecodedPointCloudView:
return DecodedPointCloudView(
context=context,
frame_id=CANONICAL_MAP_FRAME,
positions_xyz=tuple((point.x, point.y, point.z) for point in frame.points),
intensities=bytes(point.intensity for point in frame.points),
colors_rgb=bytes(
channel for point in frame.points for channel in (point.r, point.g, point.b)
),
)
def _normalize_lio_pose(
frame: LioPoseFrame,
context: ConsumerFrameContext,
) -> DecodedPoseView:
context = _with_device_context(
context,
source_device_alias=frame.header.device_id,
source_session_alias=frame.header.session_id,
)
return DecodedPoseView(
context=context,
frame_id=CANONICAL_MAP_FRAME,
child_frame_id=CANONICAL_SENSOR_FRAME,
position_xyz=frame.position_xyz,
orientation_xyzw=frame.orientation_xyzw,
)
def _normalize_legacy_pose(
frame: LegacyPoseFrame,
context: ConsumerFrameContext,
) -> DecodedPoseView:
return DecodedPoseView(
context=context,
frame_id=CANONICAL_MAP_FRAME,
child_frame_id=CANONICAL_SENSOR_FRAME,
position_xyz=frame.position_xyz,
orientation_xyzw=frame.orientation_xyzw,
)
+1 -130
View File
@@ -1,13 +1,9 @@
from __future__ import annotations
import math
import statistics
import struct
import threading
import time
from collections import deque
from dataclasses import dataclass
from typing import TypedDict
import foxglove
from foxglove import Channel
@@ -45,6 +41,7 @@ from k1link.protocol.streams import (
decode_lio_pose,
)
from k1link.viewer.messages import StreamMessage
from k1link.viewer.metrics import BridgeMetrics
POINT_STRUCT = struct.Struct("<fffB3x")
POINT_STRIDE = POINT_STRUCT.size
@@ -52,113 +49,12 @@ FRAME_ID = "map"
MAX_TRAJECTORY_POSES = 20_000
class MetricsSnapshot(TypedDict):
messages_received: int
payload_bytes: int
pcl_frames: int
pose_frames: int
points_published: int
last_point_count: int
decode_errors: int
preview_dropped: int
pcl_fps: float
pose_fps: float
mqtt_to_publish_ms: float | None
mqtt_to_publish_p50_ms: float | None
mqtt_to_publish_p95_ms: float | None
decode_publish_ms: float | None
trajectory_poses: int
@dataclass(frozen=True, slots=True)
class PackedPointCloud:
data: bytes
point_count: int
class BridgeMetrics:
"""Thread-safe counters shared with the local control API."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._messages_received = 0
self._payload_bytes = 0
self._pcl_frames = 0
self._pose_frames = 0
self._points_published = 0
self._last_point_count = 0
self._decode_errors = 0
self._preview_dropped = 0
self._trajectory_poses = 0
self._pcl_times: deque[int] = deque()
self._pose_times: deque[int] = deque()
self._latencies_ms: deque[float] = deque(maxlen=512)
self._decode_publish_ms: float | None = None
def received(self, payload_bytes: int) -> None:
with self._lock:
self._messages_received += 1
self._payload_bytes += payload_bytes
def published_pcl(self, point_count: int, now_ns: int, decode_publish_ms: float) -> None:
with self._lock:
self._pcl_frames += 1
self._points_published += point_count
self._last_point_count = point_count
self._decode_publish_ms = decode_publish_ms
self._pcl_times.append(now_ns)
_trim_rate_window(self._pcl_times, now_ns)
def published_pose(self, now_ns: int, decode_publish_ms: float, path_size: int) -> None:
with self._lock:
self._pose_frames += 1
self._decode_publish_ms = decode_publish_ms
self._trajectory_poses = path_size
self._pose_times.append(now_ns)
_trim_rate_window(self._pose_times, now_ns)
def record_latency(self, milliseconds: float) -> None:
if not math.isfinite(milliseconds) or milliseconds < 0:
return
with self._lock:
self._latencies_ms.append(milliseconds)
def decode_error(self) -> None:
with self._lock:
self._decode_errors += 1
def preview_dropped(self) -> None:
with self._lock:
self._preview_dropped += 1
def snapshot(self) -> MetricsSnapshot:
now_ns = time.monotonic_ns()
with self._lock:
_trim_rate_window(self._pcl_times, now_ns)
_trim_rate_window(self._pose_times, now_ns)
latencies = list(self._latencies_ms)
last_latency = latencies[-1] if latencies else None
p50 = statistics.median(latencies) if latencies else None
p95 = _percentile(latencies, 0.95) if latencies else None
return {
"messages_received": self._messages_received,
"payload_bytes": self._payload_bytes,
"pcl_frames": self._pcl_frames,
"pose_frames": self._pose_frames,
"points_published": self._points_published,
"last_point_count": self._last_point_count,
"decode_errors": self._decode_errors,
"preview_dropped": self._preview_dropped,
"pcl_fps": _window_rate(self._pcl_times),
"pose_fps": _window_rate(self._pose_times),
"mqtt_to_publish_ms": _rounded(last_latency),
"mqtt_to_publish_p50_ms": _rounded(p50),
"mqtt_to_publish_p95_ms": _rounded(p95),
"decode_publish_ms": _rounded(self._decode_publish_ms),
"trajectory_poses": self._trajectory_poses,
}
class FoxgloveBridge:
"""Decode verified K1 topics and publish Foxglove-native visualization messages."""
@@ -390,28 +286,3 @@ def _identity_pose() -> Pose:
def _timestamp(epoch_ns: int) -> Timestamp:
return Timestamp(epoch_ns // 1_000_000_000, epoch_ns % 1_000_000_000)
def _trim_rate_window(values: deque[int], now_ns: int) -> None:
cutoff = now_ns - 1_000_000_000
while values and values[0] < cutoff:
values.popleft()
def _window_rate(values: deque[int]) -> float:
if len(values) < 2:
return float(len(values))
elapsed = (values[-1] - values[0]) / 1_000_000_000
return len(values) / max(elapsed, 1.0)
def _percentile(values: list[float], fraction: float) -> float:
if not values:
raise ValueError("cannot calculate a percentile of an empty sample")
ordered = sorted(values)
index = min(len(ordered) - 1, math.ceil(len(ordered) * fraction) - 1)
return ordered[index]
def _rounded(value: float | None) -> float | None:
return None if value is None else round(value, 3)
+134
View File
@@ -0,0 +1,134 @@
from __future__ import annotations
import math
import statistics
import threading
import time
from collections import deque
from typing import TypedDict
class MetricsSnapshot(TypedDict):
messages_received: int
payload_bytes: int
pcl_frames: int
pose_frames: int
points_published: int
last_point_count: int
decode_errors: int
preview_dropped: int
pcl_fps: float
pose_fps: float
mqtt_to_publish_ms: float | None
mqtt_to_publish_p50_ms: float | None
mqtt_to_publish_p95_ms: float | None
decode_publish_ms: float | None
trajectory_poses: int
class BridgeMetrics:
"""Thread-safe counters shared by canonical visualization consumers."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._messages_received = 0
self._payload_bytes = 0
self._pcl_frames = 0
self._pose_frames = 0
self._points_published = 0
self._last_point_count = 0
self._decode_errors = 0
self._preview_dropped = 0
self._trajectory_poses = 0
self._pcl_times: deque[int] = deque()
self._pose_times: deque[int] = deque()
self._latencies_ms: deque[float] = deque(maxlen=512)
self._decode_publish_ms: float | None = None
def received(self, payload_bytes: int) -> None:
with self._lock:
self._messages_received += 1
self._payload_bytes += payload_bytes
def published_pcl(self, point_count: int, now_ns: int, decode_publish_ms: float) -> None:
with self._lock:
self._pcl_frames += 1
self._points_published += point_count
self._last_point_count = point_count
self._decode_publish_ms = decode_publish_ms
self._pcl_times.append(now_ns)
_trim_rate_window(self._pcl_times, now_ns)
def published_pose(self, now_ns: int, decode_publish_ms: float, path_size: int) -> None:
with self._lock:
self._pose_frames += 1
self._decode_publish_ms = decode_publish_ms
self._trajectory_poses = path_size
self._pose_times.append(now_ns)
_trim_rate_window(self._pose_times, now_ns)
def record_latency(self, milliseconds: float) -> None:
if not math.isfinite(milliseconds) or milliseconds < 0:
return
with self._lock:
self._latencies_ms.append(milliseconds)
def decode_error(self) -> None:
with self._lock:
self._decode_errors += 1
def preview_dropped(self) -> None:
with self._lock:
self._preview_dropped += 1
def snapshot(self) -> MetricsSnapshot:
now_ns = time.monotonic_ns()
with self._lock:
_trim_rate_window(self._pcl_times, now_ns)
_trim_rate_window(self._pose_times, now_ns)
latencies = list(self._latencies_ms)
last_latency = latencies[-1] if latencies else None
p50 = statistics.median(latencies) if latencies else None
p95 = _percentile(latencies, 0.95) if latencies else None
return {
"messages_received": self._messages_received,
"payload_bytes": self._payload_bytes,
"pcl_frames": self._pcl_frames,
"pose_frames": self._pose_frames,
"points_published": self._points_published,
"last_point_count": self._last_point_count,
"decode_errors": self._decode_errors,
"preview_dropped": self._preview_dropped,
"pcl_fps": _window_rate(self._pcl_times),
"pose_fps": _window_rate(self._pose_times),
"mqtt_to_publish_ms": _rounded(last_latency),
"mqtt_to_publish_p50_ms": _rounded(p50),
"mqtt_to_publish_p95_ms": _rounded(p95),
"decode_publish_ms": _rounded(self._decode_publish_ms),
"trajectory_poses": self._trajectory_poses,
}
def _trim_rate_window(samples: deque[int], now_ns: int) -> None:
cutoff_ns = now_ns - 1_000_000_000
while samples and samples[0] < cutoff_ns:
samples.popleft()
def _window_rate(samples: deque[int]) -> float:
if len(samples) < 2:
return float(len(samples))
elapsed = (samples[-1] - samples[0]) / 1_000_000_000
return len(samples) / max(elapsed, 1.0)
def _percentile(values: list[float], quantile: float) -> float:
if not values:
raise ValueError("values must not be empty")
ordered = sorted(values)
index = max(0, min(len(ordered) - 1, math.ceil(len(ordered) * quantile) - 1))
return ordered[index]
def _rounded(value: float | None) -> float | None:
return None if value is None else round(value, 3)
+43 -90
View File
@@ -11,19 +11,12 @@ import numpy as np
import rerun as rr
from rerun import blueprint as rrb
from k1link.protocol.streams import (
LegacyPointCloudFrame,
LegacyPoseFrame,
LioPointCloudFrame,
LioPoseFrame,
StreamDecodeError,
decode_legacy_pointcloud,
decode_legacy_pose,
decode_lio_pcl,
decode_lio_pose,
from k1link.data_plane import (
DecodedDataPlaneView,
DecodedPointCloudView,
DecodedPoseView,
)
from k1link.viewer.foxglove_bridge import BridgeMetrics
from k1link.viewer.messages import StreamMessage
from k1link.viewer.metrics import BridgeMetrics
PointColorMode = Literal["intensity", "height", "distance", "rgb", "class"]
PointPalette = Literal["turbo", "viridis", "plasma", "grayscale", "custom"]
@@ -59,7 +52,7 @@ SettingsProvider = Callable[[], RerunSceneSettings]
class RerunBridge:
"""Decode verified device topics into a self-hosted Rerun recording stream."""
"""Publish transport-neutral canonical envelopes to a Rerun recording."""
def __init__(
self,
@@ -73,9 +66,7 @@ class RerunBridge:
self.metrics = metrics or BridgeMetrics()
self._settings_provider = settings_provider or RerunSceneSettings
self._settings = self._settings_provider()
self._recording = (recording_factory or rr.RecordingStream)(
"nodedc_mission_core_spatial"
)
self._recording = (recording_factory or rr.RecordingStream)("nodedc_mission_core_spatial")
blueprint = _blueprint(self._settings)
self._url = self._recording.serve_grpc(
grpc_port=grpc_port,
@@ -95,9 +86,7 @@ class RerunBridge:
rr.TransformAxes3D(axis_length=0.45, show_frame=True),
static=True,
)
self._path: deque[tuple[float, float, float]] = deque(
maxlen=MAX_TRAJECTORY_POSES
)
self._path: deque[tuple[float, float, float]] = deque(maxlen=MAX_TRAJECTORY_POSES)
self._last_trajectory_publish_ns = 0
self._last_point_count = 0
self._closed = False
@@ -130,33 +119,23 @@ class RerunBridge:
make_default=True,
)
def process(self, message: StreamMessage) -> None:
started_ns = time.monotonic_ns()
self.metrics.received(len(message.payload))
def process(self, envelope: DecodedDataPlaneView) -> None:
self._apply_latest_settings()
self._set_message_time(message)
self._set_message_time(envelope)
try:
if message.topic.endswith("/lio_pcl"):
self._publish_lio_pcl(decode_lio_pcl(message.payload))
point_frame = True
elif message.topic == "RealtimePointcloud":
self._publish_legacy_pcl(decode_legacy_pointcloud(message.payload))
point_frame = True
elif message.topic.endswith("/lio_pose"):
self._publish_lio_pose(decode_lio_pose(message.payload))
point_frame = False
elif message.topic == "RealtimePath":
self._publish_legacy_pose(decode_legacy_pose(message.payload))
point_frame = False
else:
return
except StreamDecodeError:
self.metrics.decode_error()
if isinstance(envelope, DecodedPointCloudView):
self._publish_points(envelope)
point_frame = True
elif isinstance(envelope, DecodedPoseView):
self._publish_pose(envelope)
point_frame = False
else:
return
published_ns = time.monotonic_ns()
decode_publish_ms = (published_ns - started_ns) / 1_000_000
decode_publish_ms = (
published_ns - envelope.context.processing_started_monotonic_ns
) / 1_000_000
if point_frame:
self.metrics.published_pcl(
self._last_point_count,
@@ -169,10 +148,9 @@ class RerunBridge:
decode_publish_ms,
len(self._path),
)
if message.source == "live_mqtt" and message.received_monotonic_ns is not None:
self.metrics.record_latency(
(published_ns - message.received_monotonic_ns) / 1_000_000
)
context = envelope.context
if context.live and context.received_monotonic_ns is not None:
self.metrics.record_latency((published_ns - context.received_monotonic_ns) / 1_000_000)
def close(self) -> None:
if self._closed:
@@ -187,13 +165,14 @@ class RerunBridge:
# instead of waiting for the publisher thread frame to be collected.
del self._recording
def _set_message_time(self, message: StreamMessage) -> None:
def _set_message_time(self, envelope: DecodedDataPlaneView) -> None:
context = envelope.context
self._recording.set_time("stream_time", timestamp=time.time())
self._recording.set_time(
"capture_time",
timestamp=message.received_at_epoch_ns / 1_000_000_000,
timestamp=context.captured_at_epoch_ns / 1_000_000_000,
)
self._recording.set_time("message_sequence", sequence=message.sequence)
self._recording.set_time("message_sequence", sequence=context.sequence)
def _apply_latest_settings(self) -> None:
settings = self._settings_provider()
@@ -202,34 +181,17 @@ class RerunBridge:
self._settings = settings
self._recording.send_blueprint(_blueprint(settings))
def _publish_lio_pcl(self, frame: LioPointCloudFrame) -> None:
count = len(frame.points)
positions = np.empty((count, 3), dtype=np.float32)
intensities = np.empty(count, dtype=np.uint8)
scaler = frame.header.scaler
for index, point in enumerate(frame.points):
positions[index] = point.scaled_xyz(scaler)
intensities[index] = point.intensity
self._publish_points(positions, intensities, rgb=None)
def _publish_legacy_pcl(self, frame: LegacyPointCloudFrame) -> None:
count = len(frame.points)
positions = np.empty((count, 3), dtype=np.float32)
intensities = np.empty(count, dtype=np.uint8)
rgb = np.empty((count, 3), dtype=np.uint8)
for index, point in enumerate(frame.points):
positions[index] = (point.x, point.y, point.z)
intensities[index] = point.intensity
rgb[index] = (point.r, point.g, point.b)
self._publish_points(positions, intensities, rgb=rgb)
def _publish_points(
self,
positions: np.ndarray,
intensities: np.ndarray,
*,
rgb: np.ndarray | None,
) -> None:
def _publish_points(self, frame: DecodedPointCloudView) -> None:
positions = np.asarray(frame.positions_xyz, dtype=np.float32).reshape((-1, 3))
if frame.intensities is None:
intensities = np.full(frame.point_count, 255, dtype=np.uint8)
else:
intensities = np.frombuffer(frame.intensities, dtype=np.uint8)
rgb = (
None
if frame.colors_rgb is None
else np.frombuffer(frame.colors_rgb, dtype=np.uint8).reshape((-1, 3))
)
self._last_point_count = int(positions.shape[0])
if not self._settings.show_points:
self._recording.log("/world/points", rr.Clear(recursive=False))
@@ -244,25 +206,15 @@ class RerunBridge:
),
)
def _publish_lio_pose(self, frame: LioPoseFrame) -> None:
self._publish_pose(frame.position_xyz, frame.orientation_xyzw)
def _publish_legacy_pose(self, frame: LegacyPoseFrame) -> None:
self._publish_pose(frame.position_xyz, frame.orientation_xyzw)
def _publish_pose(
self,
position_xyz: tuple[float, float, float],
orientation_xyzw: tuple[float, float, float, float],
) -> None:
def _publish_pose(self, frame: DecodedPoseView) -> None:
self._recording.log(
"/world/sensor_pose",
rr.Transform3D(
translation=position_xyz,
quaternion=rr.Quaternion(xyzw=orientation_xyzw),
translation=frame.position_xyz,
quaternion=rr.Quaternion(xyzw=frame.orientation_xyzw),
),
)
self._path.append(position_xyz)
self._path.append(frame.position_xyz)
if not self._settings.show_trajectory:
self._recording.log("/world/trajectory", rr.Clear(recursive=False))
return
@@ -283,6 +235,7 @@ class RerunBridge:
),
)
def _blueprint(settings: RerunSceneSettings) -> rrb.Blueprint:
accumulation = max(0.0, settings.accumulation_seconds)
time_range = rr.VisibleTimeRange(
+54 -5
View File
@@ -7,12 +7,13 @@ import time
from collections.abc import Callable
from datetime import UTC, datetime
from pathlib import Path
from typing import Literal, TypedDict
from typing import Literal, Protocol, TypedDict
from k1link.artifacts import utc_now_iso, write_json_atomic
from k1link.data_plane import DecodedDataPlaneView, NormalizationError
from k1link.mqtt import CapturedMqttMessage, CaptureError, capture_mqtt
from k1link.viewer.foxglove_bridge import BridgeMetrics, MetricsSnapshot
from k1link.viewer.messages import StreamMessage
from k1link.viewer.metrics import BridgeMetrics, MetricsSnapshot
from k1link.viewer.replay import iter_replay_messages
from k1link.viewer.rerun_bridge import DEFAULT_GRPC_PORT, RerunBridge, RerunSceneSettings
@@ -29,10 +30,20 @@ StateCallback = Callable[[], None]
BridgeFactory = Callable[..., RerunBridge]
class CanonicalNormalizer(Protocol):
def __call__(
self,
message: StreamMessage,
*,
processing_started_monotonic_ns: int,
) -> DecodedDataPlaneView | None: ...
class RuntimeSnapshot(TypedDict):
phase: RuntimePhase
message: str
source_mode: SourceMode
source_ready: bool
foxglove_ws_url: str | None
foxglove_viewer_url: str | None
rerun_grpc_url: str | None
@@ -49,6 +60,7 @@ class VisualizationRuntime:
on_state_change: StateCallback | None = None,
grpc_port: int = DEFAULT_GRPC_PORT,
bridge_factory: BridgeFactory | None = None,
normalizer: CanonicalNormalizer,
) -> None:
self._lock = threading.Lock()
self._on_state_change = on_state_change
@@ -57,11 +69,13 @@ class VisualizationRuntime:
self._phase: RuntimePhase = "idle"
self._message = "Готово. Включите устройство и начните с поиска по Bluetooth."
self._source_mode: SourceMode = "idle"
self._source_ready = False
self._foxglove_ws_url: str | None = None
self._foxglove_viewer_url: str | None = None
self._rerun_grpc_url: str | None = None
self._grpc_port = grpc_port
self._bridge_factory = bridge_factory or RerunBridge
self._normalizer = normalizer
self._bridge: RerunBridge | None = None
self._closed = False
self._scene_settings = RerunSceneSettings()
@@ -73,6 +87,7 @@ class VisualizationRuntime:
"phase": self._phase,
"message": self._message,
"source_mode": self._source_mode,
"source_ready": self._source_ready,
"foxglove_ws_url": self._foxglove_ws_url,
"foxglove_viewer_url": self._foxglove_viewer_url,
"rerun_grpc_url": self._rerun_grpc_url,
@@ -135,6 +150,7 @@ class VisualizationRuntime:
if thread is None or not thread.is_alive():
self._phase = "idle"
self._source_mode = "idle"
self._source_ready = False
self._message = "Активного потока нет."
notify_only = True
else:
@@ -161,6 +177,7 @@ class VisualizationRuntime:
else:
self._phase = "idle"
self._source_mode = "idle"
self._source_ready = False
self._message = "Локальный поток завершён."
self._notify()
@@ -194,17 +211,34 @@ class VisualizationRuntime:
self._metrics = BridgeMetrics()
self._phase = phase
self._source_mode = source_mode
self._source_ready = False
self._message = message
self._foxglove_ws_url = None
self._foxglove_viewer_url = None
self._thread = threading.Thread(
target=target,
target=lambda: self._run_target_safely(target, source_mode),
name=f"k1-{source_mode}-session",
daemon=True,
)
self._thread.start()
self._notify()
def _run_target_safely(
self,
target: Callable[[], None],
source_mode: SourceMode,
) -> None:
"""Convert setup failures before the pipeline into observable runtime state."""
try:
target()
except BaseException as exc:
with self._lock:
closed = self._closed
if closed:
return
self._finish_error(f"Ошибка {source_mode}-источника: {type(exc).__name__}: {exc}")
def _run_replay(self, path: Path, *, speed: float, loop: bool) -> None:
def produce(put: Callable[[StreamMessage], None]) -> str:
while not self._stop_event.is_set():
@@ -324,8 +358,9 @@ class VisualizationRuntime:
publisher_aborted.set()
else:
self._rerun_grpc_url = bridge.grpc_url
if not self._closed and self._phase != "stopping":
if running_phase == "replay" and not self._closed and self._phase != "stopping":
self._phase = running_phase
self._source_ready = True
self._message = "Локальный Rerun-мост готов; источник данных запущен."
publisher_ready.set()
self._notify()
@@ -339,7 +374,18 @@ class VisualizationRuntime:
except queue.Empty:
continue
try:
bridge.process(message)
processing_started_ns = time.monotonic_ns()
self._metrics.received(len(message.payload))
try:
envelope = self._normalizer(
message,
processing_started_monotonic_ns=processing_started_ns,
)
except NormalizationError:
self._metrics.decode_error()
else:
if envelope is not None:
bridge.process(envelope)
finally:
messages.task_done()
if self._metrics.snapshot()["messages_received"] % 10 == 0:
@@ -413,6 +459,7 @@ class VisualizationRuntime:
with self._lock:
if self._phase != "stopping":
self._phase = phase
self._source_ready = True
self._message = message
self._notify()
@@ -420,6 +467,7 @@ class VisualizationRuntime:
with self._lock:
self._phase = "idle"
self._source_mode = "idle"
self._source_ready = False
self._message = message
self._foxglove_ws_url = None
self._foxglove_viewer_url = None
@@ -428,6 +476,7 @@ class VisualizationRuntime:
def _finish_error(self, message: str) -> None:
with self._lock:
self._phase = "error"
self._source_ready = False
self._message = message
self._foxglove_ws_url = None
self._foxglove_viewer_url = None
+15 -2
View File
@@ -6,7 +6,9 @@ from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any
from fastapi import FastAPI, HTTPException, WebSocket, WebSocketDisconnect
from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles
from pydantic import ValidationError
@@ -23,6 +25,7 @@ from k1link.web.plugin_runtime import (
)
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
INVALID_REQUEST_DETAIL = "Некорректные параметры запроса."
plugin_environment = load_installed_device_plugins(REPOSITORY_ROOT)
@@ -48,6 +51,16 @@ app = FastAPI(
)
@app.exception_handler(RequestValidationError)
async def request_validation_error_handler(
_: Request,
__: RequestValidationError,
) -> JSONResponse:
"""Return validation failures without reflecting request values or credentials."""
return JSONResponse(status_code=422, content={"detail": INVALID_REQUEST_DETAIL})
@app.get("/api/health")
def health() -> dict[str, Any]:
return {
@@ -86,7 +99,7 @@ async def invoke_device_plugin_action(
except (PluginNotFoundError, PluginActionNotFoundError) as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except ValidationError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
raise HTTPException(status_code=422, detail=INVALID_REQUEST_DETAIL) from exc
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except PluginExecutionError as exc:
+391
View File
@@ -0,0 +1,391 @@
from __future__ import annotations
import threading
from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from typing import Any, Literal
from uuid import UUID, uuid4
OperationStatus = Literal[
"accepted",
"running",
"operator_action_required",
"succeeded",
"failed",
"cancelled",
"timed_out",
"interrupted",
]
AcquisitionState = Literal[
"preparing",
"prepared",
"awaiting_external_start",
"starting",
"acquiring",
"awaiting_external_stop",
"stopping",
"finalizing",
"completed",
"failed",
"aborted",
"interrupted",
]
ControlMode = Literal["operator-manual", "plugin-commanded", "observe-only"]
TERMINAL_OPERATION_STATUSES: frozenset[OperationStatus] = frozenset(
{"succeeded", "failed", "cancelled", "timed_out", "interrupted"}
)
TERMINAL_ACQUISITION_STATES: frozenset[AcquisitionState] = frozenset(
{"completed", "failed", "aborted", "interrupted"}
)
def utc_now() -> datetime:
return datetime.now(UTC)
def _iso(value: datetime | None) -> str | None:
return value.isoformat().replace("+00:00", "Z") if value is not None else None
def _identifier(value: str | None, *, prefix: str) -> str:
if value is None:
return f"{prefix}-{uuid4()}"
candidate = value.strip()
if not candidate or len(candidate) > 128:
raise ValueError(f"{prefix} id must contain 1..128 characters")
try:
UUID(candidate.removeprefix(f"{prefix}-"))
except ValueError as exc:
raise ValueError(f"{prefix} id must be a generated UUID identifier") from exc
return candidate
@dataclass(slots=True)
class OperationRecord:
operation_id: str
action: str
status: OperationStatus
accepted_at: datetime
device_id: str | None = None
device_session_id: str | None = None
idempotency_key: str | None = None
deadline_at: datetime | None = None
stage_code: str = "accepted"
message_code: str = "operation.accepted"
sequence: int = 1
state_revision: int = 1
completed_at: datetime | None = None
cancellable: bool = False
cancel_requested: bool = False
result: dict[str, Any] | None = None
error: dict[str, Any] | None = None
evidence_refs: tuple[str, ...] = ()
# A keyed, non-reversible digest supplied by the service. It is deliberately
# excluded from API snapshots: callers only need mismatch detection, while
# the journal must never retain action inputs or secret material.
request_fingerprint: str | None = field(default=None, repr=False)
def as_dict(self) -> dict[str, Any]:
return {
"schema_version": "missioncore.operation-snapshot/v1alpha2",
"operation_id": self.operation_id,
"action": self.action,
"status": self.status,
"accepted_at": _iso(self.accepted_at),
"completed_at": _iso(self.completed_at),
"deadline_at": _iso(self.deadline_at),
"device_id": self.device_id,
"device_session_id": self.device_session_id,
"idempotency_key": self.idempotency_key,
"stage_code": self.stage_code,
"message_code": self.message_code,
"sequence": self.sequence,
"state_revision": self.state_revision,
"cancellable": self.cancellable,
"cancel_requested": self.cancel_requested,
"result": dict(self.result) if self.result is not None else None,
"error": dict(self.error) if self.error is not None else None,
"evidence_refs": list(self.evidence_refs),
}
class OperationJournal:
"""Bounded, secret-free operation journal for one in-process plugin runtime.
The journal deliberately stores lifecycle metadata only. Action inputs, BLE
frames, MQTT payloads and credentials never enter operation events.
"""
def __init__(
self,
*,
max_records: int = 128,
clock: Callable[[], datetime] = utc_now,
) -> None:
if max_records < 1:
raise ValueError("max_records must be positive")
self._max_records = max_records
self._clock = clock
self._lock = threading.Lock()
self._records: dict[str, OperationRecord] = {}
self._order: list[str] = []
self._idempotency: dict[str, str] = {}
def begin(
self,
action: str,
*,
operation_id: str | None = None,
idempotency_key: str | None = None,
device_id: str | None = None,
device_session_id: str | None = None,
deadline_seconds: float | None = None,
cancellable: bool = False,
request_fingerprint: str | None = None,
) -> tuple[OperationRecord, bool]:
action = action.strip()
if not action:
raise ValueError("operation action cannot be blank")
if idempotency_key is not None:
idempotency_key = idempotency_key.strip()
if not idempotency_key or len(idempotency_key) > 160:
raise ValueError("idempotency key must contain 1..160 characters")
if deadline_seconds is not None and not 0 < deadline_seconds <= 86_400:
raise ValueError("operation deadline must be within 1..86400 seconds")
with self._lock:
if idempotency_key is not None and idempotency_key in self._idempotency:
existing_idempotent = self._records[self._idempotency[idempotency_key]]
if existing_idempotent.action != action:
raise ValueError("idempotency key is already bound to another action")
if existing_idempotent.request_fingerprint != request_fingerprint:
raise ValueError("idempotency key is already bound to a different request")
return existing_idempotent, False
resolved_id = _identifier(operation_id, prefix="op")
existing_by_id = self._records.get(resolved_id)
if existing_by_id is not None:
if existing_by_id.action != action:
raise ValueError("operation id is already bound to another action")
if existing_by_id.request_fingerprint != request_fingerprint:
raise ValueError("operation id is already bound to a different request")
return existing_by_id, False
now = self._clock()
record = OperationRecord(
operation_id=resolved_id,
action=action,
status="accepted",
accepted_at=now,
device_id=device_id,
device_session_id=device_session_id,
idempotency_key=idempotency_key,
deadline_at=(
now + timedelta(seconds=deadline_seconds)
if deadline_seconds is not None
else None
),
cancellable=cancellable,
request_fingerprint=request_fingerprint,
)
self._records[resolved_id] = record
self._order.append(resolved_id)
if idempotency_key is not None:
self._idempotency[idempotency_key] = resolved_id
self._trim_locked()
return record, True
def transition(
self,
operation_id: str,
status: OperationStatus,
*,
stage_code: str,
message_code: str,
result: Mapping[str, Any] | None = None,
error: Mapping[str, Any] | None = None,
evidence_refs: Iterable[str] = (),
) -> OperationRecord:
with self._lock:
record = self._require_locked(operation_id)
if record.status in TERMINAL_OPERATION_STATUSES:
if record.status == status:
return record
raise ValueError(f"operation {operation_id} is already terminal")
record.status = status
record.stage_code = stage_code
record.message_code = message_code
record.sequence += 1
record.state_revision += 1
record.result = dict(result) if result is not None else None
record.error = dict(error) if error is not None else None
record.evidence_refs = tuple(evidence_refs)
if status in TERMINAL_OPERATION_STATUSES:
record.completed_at = self._clock()
self._trim_locked()
return record
def request_cancel(self, operation_id: str) -> OperationRecord:
with self._lock:
record = self._require_locked(operation_id)
if record.status in TERMINAL_OPERATION_STATUSES:
return record
if not record.cancellable:
raise ValueError(f"operation {operation_id} is not cancellable")
record.cancel_requested = True
record.sequence += 1
record.state_revision += 1
record.stage_code = "cancellation-requested"
record.message_code = "operation.cancellation_requested"
return record
def transition_if_pending(
self,
operation_id: str | None,
status: OperationStatus,
*,
stage_code: str,
message_code: str,
result: Mapping[str, Any] | None = None,
error: Mapping[str, Any] | None = None,
evidence_refs: Iterable[str] = (),
) -> OperationRecord | None:
"""Atomically transition an existing non-terminal operation.
Lifecycle reconciliation and explicit stop/abort paths can race. This
helper makes terminalization idempotent without exposing mutable journal
records or turning an already-completed operation into an error.
"""
with self._lock:
if operation_id is None:
return None
record = self._records.get(operation_id)
if record is None or record.status in TERMINAL_OPERATION_STATUSES:
return record
record.status = status
record.stage_code = stage_code
record.message_code = message_code
record.sequence += 1
record.state_revision += 1
record.result = dict(result) if result is not None else None
record.error = dict(error) if error is not None else None
record.evidence_refs = tuple(evidence_refs)
if status in TERMINAL_OPERATION_STATUSES:
record.completed_at = self._clock()
self._trim_locked()
return record
def get(self, operation_id: str) -> OperationRecord:
with self._lock:
return self._require_locked(operation_id)
def latest(self) -> OperationRecord | None:
with self._lock:
return self._records[self._order[-1]] if self._order else None
def snapshot(self, *, limit: int = 20) -> list[dict[str, Any]]:
if limit < 1:
return []
with self._lock:
return [self._records[item].as_dict() for item in self._order[-limit:]]
def _require_locked(self, operation_id: str) -> OperationRecord:
try:
return self._records[operation_id]
except KeyError as exc:
raise KeyError(f"unknown operation: {operation_id}") from exc
def _trim_locked(self) -> None:
while len(self._order) > self._max_records:
oldest_id = next(
(
operation_id
for operation_id in self._order
if self._records[operation_id].status in TERMINAL_OPERATION_STATUSES
),
None,
)
# Never evict an operation that still needs reconciliation. A brief
# overrun is safer than turning a later device observation into an
# unknown-operation failure.
if oldest_id is None:
return
self._order.remove(oldest_id)
oldest = self._records.pop(oldest_id)
if oldest.idempotency_key is not None:
self._idempotency.pop(oldest.idempotency_key, None)
@dataclass(slots=True)
class AcquisitionRecord:
acquisition_id: str
device_id: str
device_session_id: str
compatibility_profile_id: str
control_mode: ControlMode
requested_streams: tuple[str, ...]
target_host: str
duration_seconds: float
evidence_policy: Literal["required", "best-effort", "disabled"]
state: AcquisitionState = "preparing"
state_revision: int = 1
created_at: datetime = field(default_factory=utc_now)
updated_at: datetime = field(default_factory=utc_now)
message_code: str = "acquisition.preparing"
operator_instructions: tuple[str, ...] = ()
result: dict[str, Any] | None = None
def transition(
self,
state: AcquisitionState,
*,
message_code: str,
operator_instructions: Iterable[str] = (),
result: Mapping[str, Any] | None = None,
) -> None:
if self.state in TERMINAL_ACQUISITION_STATES:
if self.state == state:
return
raise ValueError(f"acquisition {self.acquisition_id} is already terminal")
self.state = state
self.state_revision += 1
self.updated_at = utc_now()
self.message_code = message_code
self.operator_instructions = tuple(operator_instructions)
self.result = dict(result) if result is not None else None
def as_dict(self) -> dict[str, Any]:
return {
"schema_version": "missioncore.acquisition-snapshot/v1alpha2",
"acquisition_id": self.acquisition_id,
"device_id": self.device_id,
"device_session_id": self.device_session_id,
"compatibility_profile_id": self.compatibility_profile_id,
"control_mode": self.control_mode,
"requested_streams": list(self.requested_streams),
"target_host": self.target_host,
"duration_seconds": self.duration_seconds,
"evidence_policy": self.evidence_policy,
"state": self.state,
"state_revision": self.state_revision,
"created_at": _iso(self.created_at),
"updated_at": _iso(self.updated_at),
"message_code": self.message_code,
"operator_instructions": list(self.operator_instructions),
"result": dict(self.result) if self.result is not None else None,
}
def new_acquisition_id() -> str:
return f"acq-{uuid4()}"
def new_device_id() -> str:
return f"device-{uuid4()}"
def new_device_session_id() -> str:
return f"device-session-{uuid4()}"
+182 -6
View File
@@ -1,13 +1,29 @@
from __future__ import annotations
import json
import re
from pathlib import Path
from typing import Annotated, Any, Literal
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, ValidationError
from pydantic import (
AfterValidator,
BaseModel,
ConfigDict,
Field,
ValidationError,
model_validator,
)
from k1link.web.plugin_runtime import STATE_READ_ACTION_ID
PLUGIN_API_VERSION = "missioncore.nodedc/v1alpha1"
PLUGIN_API_VERSION = "missioncore.nodedc/v1alpha2"
SUPPORTED_PLUGIN_API_VERSIONS = frozenset(
{
"missioncore.nodedc/v1alpha1",
PLUGIN_API_VERSION,
}
)
_V1ALPHA2_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]*$")
def _reject_blank(value: str) -> str:
@@ -16,6 +32,11 @@ def _reject_blank(value: str) -> str:
return value
def _validate_v1alpha2_identifier(value: str, path: str) -> None:
if _V1ALPHA2_IDENTIFIER.fullmatch(value) is None:
raise ValueError(f"{path} must be a v1alpha2 identifier")
ShortText = Annotated[
str,
Field(min_length=1, max_length=160),
@@ -31,6 +52,11 @@ EntrypointText = Annotated[
Field(min_length=1, max_length=256),
AfterValidator(_reject_blank),
]
ProfilePathText = Annotated[
str,
Field(min_length=1, max_length=512),
AfterValidator(_reject_blank),
]
class CapabilityManifest(BaseModel):
@@ -75,6 +101,14 @@ class PluginActionManifest(BaseModel):
secretFields: list[ShortText]
class CompatibilityProfileLinkManifest(BaseModel):
model_config = ConfigDict(extra="forbid")
profileId: ShortText
path: ProfilePathText
modelId: ShortText
class PluginMetadata(BaseModel):
model_config = ConfigDict(extra="forbid")
@@ -94,26 +128,164 @@ class PluginMetadata(BaseModel):
class PluginSpec(BaseModel):
model_config = ConfigDict(extra="forbid")
hostApiRange: Literal["v1alpha1"]
hostApiRange: Literal["v1alpha1", "v1alpha2"]
runtime: PluginRuntimeManifest
permissions: list[ShortText]
actions: list[PluginActionManifest]
models: list[DeviceModelManifest] = Field(min_length=1, max_length=1)
models: list[DeviceModelManifest] = Field(min_length=1)
compatibilityProfiles: list[CompatibilityProfileLinkManifest] | None = None
class DevicePluginManifest(BaseModel):
model_config = ConfigDict(extra="forbid")
apiVersion: Literal["missioncore.nodedc/v1alpha1"]
apiVersion: Literal[
"missioncore.nodedc/v1alpha1",
"missioncore.nodedc/v1alpha2",
]
kind: Literal["DevicePlugin"]
metadata: PluginMetadata
spec: PluginSpec
@model_validator(mode="after")
def validate_contract_version(self) -> DevicePluginManifest:
expected_host_range = self.apiVersion.rsplit("/", maxsplit=1)[-1]
if self.spec.hostApiRange != expected_host_range:
raise ValueError("apiVersion and hostApiRange must declare the same contract")
if self.apiVersion == "missioncore.nodedc/v1alpha1":
if len(self.spec.models) != 1:
raise ValueError("v1alpha1 must declare exactly one device model")
if self.spec.compatibilityProfiles is not None:
raise ValueError("v1alpha1 must not declare compatibilityProfiles")
else:
if not self.spec.compatibilityProfiles:
raise ValueError("v1alpha2 must declare at least one compatibility profile")
identifiers: list[tuple[str, str]] = [
("metadata.id", self.metadata.id),
*(
(f"spec.permissions[{index}]", permission)
for index, permission in enumerate(self.spec.permissions)
),
]
for action_index, action in enumerate(self.spec.actions):
identifiers.append((f"spec.actions[{action_index}].id", action.id))
identifiers.extend(
(
f"spec.actions[{action_index}].secretFields[{field_index}]",
field,
)
for field_index, field in enumerate(action.secretFields)
)
for model_index, model in enumerate(self.spec.models):
identifiers.append((f"spec.models[{model_index}].id", model.id))
identifiers.extend(
(
f"spec.models[{model_index}].capabilities[{capability_index}].id",
capability.id,
)
for capability_index, capability in enumerate(model.capabilities)
)
for profile_index, profile in enumerate(self.spec.compatibilityProfiles):
identifiers.extend(
(
(
f"spec.compatibilityProfiles[{profile_index}].profileId",
profile.profileId,
),
(
f"spec.compatibilityProfiles[{profile_index}].modelId",
profile.modelId,
),
)
)
for path, value in identifiers:
_validate_v1alpha2_identifier(value, path)
return self
class PluginCatalogError(RuntimeError):
"""An installed manifest is invalid or conflicts with another manifest."""
def _reject_duplicate_profile_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
result: dict[str, Any] = {}
for key, value in pairs:
if key in result:
raise PluginCatalogError(f"Duplicate JSON key in compatibility profile: {key}")
result[key] = value
return result
def _validate_profile_links(
manifest: DevicePluginManifest,
manifest_path: Path,
) -> None:
links = manifest.spec.compatibilityProfiles
if links is None:
return
plugin_directory = manifest_path.parent.resolve()
model_ids = {model.id for model in manifest.spec.models}
linked_model_ids: set[str] = set()
profile_ids: set[str] = set()
profile_paths: set[Path] = set()
for link in links:
if link.profileId in profile_ids:
raise PluginCatalogError(
f"Duplicate compatibility profile id in {manifest.metadata.id}: {link.profileId}"
)
profile_ids.add(link.profileId)
if link.modelId not in model_ids:
raise PluginCatalogError(
f"Compatibility profile {link.profileId} references unknown model {link.modelId}"
)
linked_model_ids.add(link.modelId)
relative_path = Path(link.path)
if relative_path.is_absolute():
raise PluginCatalogError(
f"Compatibility profile path must be plugin-relative: {link.path}"
)
profile_path = (plugin_directory / relative_path).resolve()
try:
profile_path.relative_to(plugin_directory)
except ValueError as exc:
raise PluginCatalogError(
f"Compatibility profile path escapes plugin directory: {link.path}"
) from exc
if profile_path in profile_paths:
raise PluginCatalogError(
f"Duplicate compatibility profile path in {manifest.metadata.id}: {link.path}"
)
profile_paths.add(profile_path)
if not profile_path.is_file():
raise PluginCatalogError(f"Compatibility profile does not exist: {link.path}")
try:
profile_document = json.loads(
profile_path.read_text(encoding="utf-8"),
object_pairs_hook=_reject_duplicate_profile_keys,
)
except (OSError, json.JSONDecodeError) as exc:
raise PluginCatalogError(
f"Invalid compatibility profile {profile_path}: {exc}"
) from exc
if not isinstance(profile_document, dict):
raise PluginCatalogError(
f"Compatibility profile must contain a JSON object: {link.path}"
)
if profile_document.get("profile_id") != link.profileId:
raise PluginCatalogError(
f"Compatibility profile id mismatch for {link.path}: expected {link.profileId}"
)
if linked_model_ids != model_ids:
missing = ", ".join(sorted(model_ids - linked_model_ids))
raise PluginCatalogError(f"Device models without compatibility profiles: {missing}")
class DevicePluginCatalog:
"""Read-only catalog of statically reviewed device-plugin manifests."""
@@ -141,6 +313,8 @@ class DevicePluginCatalog:
raise PluginCatalogError(f"Duplicate device-plugin id: {plugin_id}")
plugin_ids.add(plugin_id)
_validate_profile_links(manifest, path)
action_ids: set[str] = set()
for action in manifest.spec.actions:
if action.id in action_ids:
@@ -178,7 +352,9 @@ class DevicePluginCatalog:
return list(self._validated_manifests)
def plugin_documents(self) -> list[dict[str, Any]]:
return [manifest.model_dump(mode="json") for manifest in self.manifests()]
return [
manifest.model_dump(mode="json", exclude_none=True) for manifest in self.manifests()
]
def model_documents(self) -> list[dict[str, Any]]:
models: list[dict[str, Any]] = []
File diff suppressed because it is too large Load Diff