SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(k1): add live cameras and reliable spatial following

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-17 00:26:03 +03:00
parent a281faf923
commit 2bda1986bd
33 changed files with 2927 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/viewer/rerun_bridge.py
+127 -36
View File
@@ -2,10 +2,11 @@ from __future__ import annotations
import math
import time
from collections import deque
from collections.abc import Callable
from contextlib import suppress
from dataclasses import asdict, dataclass
from typing import Literal
from uuid import uuid4
import numpy as np
import rerun as rr
@@ -21,7 +22,12 @@ from k1link.viewer.metrics import BridgeMetrics
PointColorMode = Literal["intensity", "height", "distance", "rgb", "class"]
PointPalette = Literal["turbo", "viridis", "plasma", "grayscale", "custom"]
MAX_TRAJECTORY_POSES = 20_000
MAX_TRAJECTORY_POSES = 2_000
TRAJECTORY_APPEND_INTERVAL_NS = 500_000_000
TRAJECTORY_FORCE_APPEND_NS = 2_000_000_000
TRAJECTORY_MIN_DISTANCE_METERS = 0.02
TRAJECTORY_PUBLISH_INTERVAL_NS = 500_000_000
LIVE_GRPC_BUFFER_LIMIT = "32MiB"
DEFAULT_GRPC_PORT = 9876
DEFAULT_CORS_ORIGINS = (
"http://127.0.0.1:5173",
@@ -66,27 +72,52 @@ class RerunBridge:
self.metrics = metrics or BridgeMetrics()
self._settings_provider = settings_provider or RerunSceneSettings
self._settings = self._settings_provider()
self._recording = (recording_factory or rr.RecordingStream)("nodedc_mission_core_spatial")
blueprint = _blueprint(self._settings)
self._url = self._recording.serve_grpc(
grpc_port=grpc_port,
default_blueprint=blueprint,
server_memory_limit="512MiB",
newest_first=True,
cors_allow_origin=list(cors_allow_origin),
)
self._recording.send_blueprint(
blueprint,
make_active=True,
make_default=True,
)
self._recording.log("/world", rr.ViewCoordinates.RIGHT_HAND_Z_UP, static=True)
self._recording.log(
"/world/sensor_pose",
rr.TransformAxes3D(axis_length=0.45, show_frame=True),
static=True,
)
self._path: deque[tuple[float, float, float]] = deque(maxlen=MAX_TRAJECTORY_POSES)
if recording_factory is None:
recording = rr.RecordingStream(
"nodedc_mission_core_spatial",
recording_id=uuid4(),
)
else:
recording = recording_factory("nodedc_mission_core_spatial")
try:
blueprint = _blueprint(self._settings)
url = recording.serve_grpc(
grpc_port=grpc_port,
default_blueprint=blueprint,
# This is a reconnect cushion for the live preview, not the source
# of record. Raw MQTT evidence is persisted independently. A large
# late-client backlog can block the native SDK and freeze preview.
server_memory_limit=LIVE_GRPC_BUFFER_LIMIT,
# Rerun 0.34.1 can replay ActivateStore before StoreInfo when an
# evicted buffer is served newest-first, leaving late viewers on the
# welcome screen. Preserve protocol order within the bounded cache.
newest_first=False,
cors_allow_origin=list(cors_allow_origin),
)
recording.send_blueprint(
blueprint,
make_active=True,
make_default=True,
)
recording.log("/world", rr.ViewCoordinates.RIGHT_HAND_Z_UP, static=True)
recording.log(
"/world/sensor_pose",
rr.TransformAxes3D(axis_length=0.45, show_frame=True),
static=True,
)
# Do not expose a URL whose StoreInfo, blueprint and static scene are
# still waiting in the SDK micro-batcher.
recording.flush(timeout_sec=5.0)
except BaseException:
# Preserve the construction failure while still making a best
# effort to release a partially started native listener.
with suppress(BaseException):
recording.disconnect()
raise
self._recording = recording
self._url = url
self._path: list[tuple[float, float, float]] = []
self._last_path_append_source_ns = 0
self._last_trajectory_publish_ns = 0
self._last_point_count = 0
self._closed = False
@@ -96,13 +127,14 @@ class RerunBridge:
return self._url
def begin_session(self, metrics: BridgeMetrics | None = None) -> None:
"""Reset session-local state while keeping the process-wide server alive."""
"""Reset session state while keeping the process-lifetime server alive."""
if self._closed:
raise RuntimeError("Rerun bridge is already closed")
if metrics is not None:
self.metrics = metrics
self._settings = self._settings_provider()
self._path.clear()
self._last_path_append_source_ns = 0
self._last_trajectory_publish_ns = 0
self._last_point_count = 0
self._recording.set_time("stream_time", timestamp=time.time())
@@ -118,6 +150,10 @@ class RerunBridge:
make_active=True,
make_default=True,
)
# VisualizationRuntime publishes grpc_url only after this method
# returns, so a late subscriber cannot race the initial StoreInfo and
# blueprint through the SDK micro-batcher.
self._recording.flush(timeout_sec=5.0)
def process(self, envelope: DecodedDataPlaneView) -> None:
self._apply_latest_settings()
@@ -158,12 +194,14 @@ class RerunBridge:
self._closed = True
recording = self._recording
try:
recording.flush(timeout_sec=5.0)
try:
recording.flush(timeout_sec=5.0)
finally:
recording.disconnect()
finally:
recording.disconnect()
# The Python wrapper owns the native gRPC server. Release it immediately
# instead of waiting for the publisher thread frame to be collected.
del self._recording
# The Python wrapper owns the native gRPC server. Release it even if
# flush or disconnect fails instead of waiting for garbage collection.
del self._recording
def _set_message_time(self, envelope: DecodedDataPlaneView) -> None:
context = envelope.context
@@ -207,25 +245,35 @@ class RerunBridge:
)
def _publish_pose(self, frame: DecodedPoseView) -> None:
publish_now_ns = time.monotonic_ns()
position = (
float(frame.position_xyz[0]),
float(frame.position_xyz[1]),
float(frame.position_xyz[2]),
)
self._recording.log(
"/world/sensor_pose",
rr.Transform3D(
translation=frame.position_xyz,
translation=position,
quaternion=rr.Quaternion(xyzw=frame.orientation_xyzw),
),
)
self._path.append(frame.position_xyz)
appended = self._append_trajectory_pose(
position,
frame.context.captured_at_epoch_ns,
)
if not self._settings.show_trajectory:
self._recording.log("/world/trajectory", rr.Clear(recursive=False))
return
now_ns = time.monotonic_ns()
if not appended:
return
if (
len(self._path) > 2
and len(self._path) % 20 != 0
and now_ns - self._last_trajectory_publish_ns < 200_000_000
publish_now_ns - self._last_trajectory_publish_ns
< TRAJECTORY_PUBLISH_INTERVAL_NS
):
return
self._last_trajectory_publish_ns = now_ns
self._last_trajectory_publish_ns = publish_now_ns
self._recording.log(
"/world/trajectory",
rr.LineStrips3D(
@@ -235,6 +283,39 @@ class RerunBridge:
),
)
def _append_trajectory_pose(
self,
position: tuple[float, float, float],
source_time_ns: int,
) -> bool:
if not self._path:
self._path.append(position)
self._last_path_append_source_ns = source_time_ns
return True
elapsed_ns = source_time_ns - self._last_path_append_source_ns
if elapsed_ns < 0:
# A source clock discontinuity must not freeze trajectory sampling
# until the timestamp catches up again. Preserve message order and
# start a new sampling interval at the discontinuity.
elapsed_ns = TRAJECTORY_FORCE_APPEND_NS
if elapsed_ns < TRAJECTORY_APPEND_INTERVAL_NS:
return False
if (
math.dist(self._path[-1], position) < TRAJECTORY_MIN_DISTANCE_METERS
and elapsed_ns < TRAJECTORY_FORCE_APPEND_NS
):
return False
self._path.append(position)
self._last_path_append_source_ns = source_time_ns
if len(self._path) > MAX_TRAJECTORY_POSES:
last = self._path[-1]
self._path = self._path[::2]
if self._path[-1] != last:
self._path.append(last)
return True
def _blueprint(settings: RerunSceneSettings) -> rrb.Blueprint:
accumulation = max(0.0, settings.accumulation_seconds)
@@ -255,12 +336,22 @@ def _blueprint(settings: RerunSceneSettings) -> rrb.Blueprint:
),
time_ranges=[time_range],
),
_live_time_panel(),
auto_layout=False,
auto_views=False,
collapse_panels=True,
)
def _live_time_panel() -> rrb.TimePanel:
"""Keep the hidden vendor timeline on its native live edge."""
return rrb.TimePanel(
timeline="stream_time",
play_state="following",
state="hidden",
)
def _point_colors(
positions: np.ndarray,
intensities: np.ndarray,
src/k1link/viewer/runtime.py
+33 -11
View File
@@ -28,6 +28,10 @@ RuntimePhase = Literal[
SourceMode = Literal["idle", "live", "replay"]
StateCallback = Callable[[], None]
BridgeFactory = Callable[..., RerunBridge]
# TODO: replace the mixed-modality FIFO with a latest point-cloud slot and a
# bounded pose queue. The compact queue protects acquisition from a slow
# visualizer, but under sustained pressure it can still evict pose messages.
PREVIEW_QUEUE_SIZE = 4
class CanonicalNormalizer(Protocol):
@@ -191,7 +195,13 @@ class VisualizationRuntime:
self._bridge = None
self._rerun_grpc_url = None
if bridge is not None:
bridge.close()
try:
bridge.close()
except BaseException as exc:
self._finish_error(
f"Ошибка завершения визуального моста: {type(exc).__name__}: {exc}"
)
raise
self._notify()
def _start(
@@ -308,7 +318,7 @@ class VisualizationRuntime:
*,
running_phase: RuntimePhase,
) -> None:
messages: queue.Queue[StreamMessage] = queue.Queue(maxsize=32)
messages: queue.Queue[StreamMessage] = queue.Queue(maxsize=PREVIEW_QUEUE_SIZE)
source_done = threading.Event()
publisher_ready = threading.Event()
publisher_aborted = threading.Event()
@@ -334,23 +344,28 @@ class VisualizationRuntime:
def publish() -> None:
bridge: RerunBridge | None = None
try:
bridge = self._bridge
with self._lock:
bridge = self._bridge
if self._closed:
publisher_aborted.set()
if publisher_aborted.is_set():
publisher_ready.set()
return
if bridge is None:
candidate = self._bridge_factory(
grpc_port=self._grpc_port,
metrics=self._metrics,
settings_provider=self._current_scene_settings,
)
bridge = candidate
with self._lock:
if self._closed:
publisher_aborted.set()
else:
self._bridge = candidate
bridge = candidate
if publisher_aborted.is_set():
candidate.close()
publisher_ready.set()
return
if publisher_aborted.is_set():
publisher_ready.set()
return
assert bridge is not None
bridge.begin_session(self._metrics)
with self._lock:
@@ -402,12 +417,19 @@ class VisualizationRuntime:
finally:
if bridge is not None:
with self._lock:
close_bridge = self._closed and self._bridge is bridge
if close_bridge:
close_bridge = self._closed and (
self._bridge is bridge or publisher_aborted.is_set()
)
if close_bridge and self._bridge is bridge:
self._bridge = None
self._rerun_grpc_url = None
if close_bridge:
bridge.close()
try:
bridge.close()
except BaseException as exc:
# Process shutdown must not become a false successful
# idle state when the native bridge failed to close.
publisher_error.append(exc)
def join_publisher() -> None:
# Never orphan a publisher: the session thread remains its owner.
src/k1link/web/xgrids_k1_camera.py
+569
View File
@@ -0,0 +1,569 @@
from __future__ import annotations
import asyncio
import os
import queue
import signal
import subprocess
import threading
from collections import deque
from contextlib import suppress
from dataclasses import dataclass, field
from pathlib import Path
from typing import IO, Any, Final, Literal
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
from k1link.mqtt import validate_private_ipv4
CameraSourceId = Literal["sensor.camera.left", "sensor.camera.right"]
CAMERA_SOURCE_PATHS: Final[dict[CameraSourceId, str]] = {
"sensor.camera.left": "/live/chn_left_main",
"sensor.camera.right": "/live/chn_right_main",
}
CAMERA_SOURCE_LABELS: Final[dict[CameraSourceId, str]] = {
"sensor.camera.left": "K1 · камера слева",
"sensor.camera.right": "K1 · камера справа",
}
CAMERA_MEDIA_TYPE: Final = 'video/mp4; codecs="avc1.641028"'
CAMERA_EXCLUSIVE_GROUP: Final = "camera.preview.decoder"
MAX_FMP4_BOX_BYTES: Final = 8 * 1024 * 1024
MAX_FMP4_SEGMENT_BYTES: Final = 1024 * 1024
MAX_QUEUED_SEGMENTS: Final = 4
@dataclass
class CameraProcessLease:
generation: int
source_id: CameraSourceId
process: subprocess.Popen[bytes]
stderr_tail: deque[str] = field(default_factory=lambda: deque(maxlen=12))
segments: queue.Queue[tuple[str, bytes] | None] = field(
default_factory=lambda: queue.Queue(maxsize=MAX_QUEUED_SEGMENTS)
)
failure_code: str | None = None
class XgridsK1CameraGateway:
"""One fail-closed K1 RTSP owner with a browser-safe fMP4 delivery plane.
Selection and delivery are deliberately separate. A plugin action selects
one allowlisted producer and publishes an opaque generation. Only a matching
WebSocket may then start FFmpeg. This keeps the device address and RTSP path
out of the generic UI and prevents two browser windows from opening two K1
sessions.
"""
def __init__(self, repository_root: Path, plugin_id: str) -> None:
self._repository_root = repository_root.resolve()
self._plugin_id = plugin_id
self._lock = threading.RLock()
self._revision = 0
self._generation = 0
self._phase = "idle"
self._source_id: CameraSourceId | None = None
self._target_host: str | None = None
self._process: subprocess.Popen[bytes] | None = None
self._process_generation: int | None = None
self._error: dict[str, str] | None = None
self._closed = False
self._ffmpeg_path, self._ffmpeg_source = _resolve_ffmpeg(self._repository_root)
def snapshot(self) -> dict[str, Any]:
with self._lock:
delivery = None
if (
self._source_id is not None
and self._ffmpeg_path is not None
and self._phase != "error"
):
delivery = {
"id": f"camera-preview-{self._generation}",
"kind": "mse-fmp4-websocket",
"url": (
f"/api/v1/device-plugins/{self._plugin_id}"
f"/camera-preview/{self._generation}"
),
"media_type": CAMERA_MEDIA_TYPE,
}
return {
"schema_version": "missioncore.camera-preview/v1alpha1",
"phase": self._phase,
"revision": self._revision,
"generation": self._generation if self._source_id is not None else None,
"active_source_id": self._source_id,
"activation": {
"exclusive_group": CAMERA_EXCLUSIVE_GROUP,
"max_active": 1,
},
"delivery": delivery,
"runtime_dependency": {
"kind": "ffmpeg",
"status": "available" if self._ffmpeg_path is not None else "missing",
"source": self._ffmpeg_source,
},
"error": dict(self._error) if self._error is not None else None,
}
def select(self, source_id: CameraSourceId, target_host: str) -> dict[str, Any]:
if source_id not in CAMERA_SOURCE_PATHS:
raise ValueError("неизвестный camera source")
target = validate_private_ipv4(target_host)
old_process: subprocess.Popen[bytes] | None = None
with self._lock:
if self._closed:
raise RuntimeError("camera gateway уже закрыт")
if self._ffmpeg_path is None:
self._generation += 1
self._revision += 1
self._source_id = source_id
self._target_host = target
self._phase = "error"
self._error = {
"code": "ffmpeg-unavailable",
"message": "Локальный camera adapter FFmpeg не найден.",
}
raise RuntimeError("локальный camera adapter FFmpeg не найден")
if (
self._source_id == source_id
and self._target_host == target
and self._phase in {"selected", "connecting", "streaming"}
):
return self.snapshot()
old_process = self._detach_process_locked()
self._generation += 1
self._revision += 1
self._source_id = source_id
self._target_host = target
self._phase = "selected"
self._error = None
_terminate_process(old_process)
return self.snapshot()
def stop(self, generation: int) -> dict[str, Any]:
old_process: subprocess.Popen[bytes] | None
with self._lock:
if self._source_id is None:
return self.snapshot()
if generation != self._generation:
raise ValueError("camera preview generation устарело")
old_process = self._detach_process_locked()
self._revision += 1
self._phase = "idle"
self._source_id = None
self._target_host = None
self._error = None
_terminate_process(old_process)
return self.snapshot()
def stop_current(self) -> dict[str, Any]:
old_process: subprocess.Popen[bytes] | None
with self._lock:
old_process = self._detach_process_locked()
changed = self._source_id is not None or self._phase != "idle"
if changed:
self._revision += 1
self._phase = "idle"
self._source_id = None
self._target_host = None
self._error = None
_terminate_process(old_process)
return self.snapshot()
def open_delivery(self, generation: int) -> CameraProcessLease:
with self._lock:
if self._closed:
raise RuntimeError("camera gateway закрыт")
if generation != self._generation or self._source_id is None:
raise ValueError("camera preview generation не активно")
if self._process is not None:
raise RuntimeError("для camera preview уже открыт browser consumer")
if self._ffmpeg_path is None or self._target_host is None:
raise RuntimeError("camera preview runtime не готов")
argv = _build_ffmpeg_argv(
self._ffmpeg_path,
self._target_host,
self._source_id,
)
try:
process = subprocess.Popen(
argv,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
start_new_session=(os.name == "posix"),
)
except OSError as exc:
self._revision += 1
self._phase = "error"
self._error = {
"code": "ffmpeg-start-failed",
"message": "Не удалось запустить локальный camera adapter.",
}
raise RuntimeError("не удалось запустить camera adapter") from exc
if process.stdout is None or process.stderr is None:
_terminate_process(process)
raise RuntimeError("camera adapter не открыл media pipes")
lease = CameraProcessLease(
generation=generation,
source_id=self._source_id,
process=process,
)
self._process = process
self._process_generation = generation
self._revision += 1
self._phase = "connecting"
self._error = None
threading.Thread(
target=_drain_stderr,
args=(lease,),
name=f"k1-camera-stderr-{generation}",
daemon=True,
).start()
threading.Thread(
target=_read_fmp4_stdout,
args=(lease,),
name=f"k1-camera-fmp4-{generation}",
daemon=True,
).start()
return lease
def mark_streaming(self, lease: CameraProcessLease) -> None:
with self._lock:
if (
lease.generation != self._generation
or self._process is not lease.process
or self._phase == "streaming"
):
return
self._revision += 1
self._phase = "streaming"
def release_delivery(self, lease: CameraProcessLease, *, client_closed: bool) -> None:
_terminate_process(lease.process)
with self._lock:
if self._process is lease.process:
self._process = None
self._process_generation = None
if lease.generation != self._generation or self._source_id is None:
return
self._revision += 1
if client_closed:
self._phase = "selected"
self._error = None
else:
self._phase = "error"
if lease.failure_code == "consumer-too-slow":
message = (
"Browser не успевает принимать camera stream; "
"канал остановлен без накопления задержки."
)
elif lease.failure_code == "invalid-fmp4":
message = "Camera adapter вернул некорректный fMP4 stream."
elif lease.failure_code == "segment-too-large":
message = "Camera adapter отклонил слишком большой video segment."
else:
message = _safe_ffmpeg_message(lease.stderr_tail)
self._error = {
"code": lease.failure_code or "camera-source-ended",
"message": message,
}
def close(self) -> None:
old_process: subprocess.Popen[bytes] | None
with self._lock:
if self._closed:
return
self._closed = True
old_process = self._detach_process_locked()
self._revision += 1
self._phase = "idle"
self._source_id = None
self._target_host = None
self._error = None
_terminate_process(old_process)
def _detach_process_locked(self) -> subprocess.Popen[bytes] | None:
process = self._process
self._process = None
self._process_generation = None
return process
def _resolve_ffmpeg(repository_root: Path) -> tuple[Path | None, str]:
configured = os.environ.get("MISSIONCORE_FFMPEG_BINARY")
candidates: list[tuple[Path, str]] = []
if configured:
candidates.append((Path(configured).expanduser(), "configured"))
candidates.append((repository_root / ".runtime" / "ffmpeg" / "ffmpeg", "bundled-local"))
# Explicit development-only fallbacks. Product packaging must provide the
# repository-local binary or MISSIONCORE_FFMPEG_BINARY instead.
candidates.extend(
(
(Path("/opt/homebrew/bin/ffmpeg"), "development-system"),
(Path("/usr/local/bin/ffmpeg"), "development-system"),
)
)
for candidate, source in candidates:
try:
resolved = candidate.resolve(strict=True)
except OSError:
continue
if resolved.is_file() and os.access(resolved, os.X_OK):
return resolved, source
return None, "missing"
def _build_ffmpeg_argv(
ffmpeg_path: Path,
target_host: str,
source_id: CameraSourceId,
) -> list[str]:
target = validate_private_ipv4(target_host)
path = CAMERA_SOURCE_PATHS.get(source_id)
if path is None:
raise ValueError("неизвестный camera source")
upstream = f"rtsp://{target}:8554{path}"
return [
str(ffmpeg_path),
"-hide_banner",
"-loglevel",
"warning",
"-nostdin",
"-rtsp_transport",
"tcp",
"-allowed_media_types",
"video",
"-timeout",
"5000000",
"-probesize",
"4000000",
"-analyzeduration",
"2000000",
"-fflags",
"nobuffer",
"-i",
upstream,
"-map",
"0:v:0",
"-an",
"-sn",
"-dn",
"-c:v",
"copy",
"-f",
"mp4",
"-movflags",
"+empty_moov+default_base_moof+omit_tfhd_offset+frag_every_frame+skip_trailer",
"-flush_packets",
"1",
"pipe:1",
]
def _read_exact(stream: IO[bytes], size: int) -> bytes:
chunks: list[bytes] = []
remaining = size
while remaining:
chunk = stream.read(remaining)
if not chunk:
raise EOFError
chunks.append(chunk)
remaining -= len(chunk)
return b"".join(chunks)
def _read_mp4_box(stream: IO[bytes]) -> tuple[bytes, bytes]:
header = _read_exact(stream, 8)
size = int.from_bytes(header[:4], "big")
box_type = header[4:8]
if size == 1:
extended = _read_exact(stream, 8)
size = int.from_bytes(extended, "big")
header += extended
if size == 0 or size < len(header) or size > MAX_FMP4_BOX_BYTES:
raise ValueError("invalid or unbounded ISO-BMFF box")
return box_type, header + _read_exact(stream, size - len(header))
def _enqueue_segment(lease: CameraProcessLease, kind: str, payload: bytes) -> bool:
if len(payload) > MAX_FMP4_SEGMENT_BYTES:
lease.failure_code = "segment-too-large"
_finish_segment_queue(lease)
with suppress(OSError):
lease.process.terminate()
return False
try:
lease.segments.put_nowait((kind, payload))
return True
except queue.Full:
lease.failure_code = "consumer-too-slow"
while True:
try:
lease.segments.get_nowait()
except queue.Empty:
break
lease.segments.put_nowait(None)
with suppress(OSError):
lease.process.terminate()
return False
def _finish_segment_queue(lease: CameraProcessLease) -> None:
try:
lease.segments.put_nowait(None)
except queue.Full:
# A full queue is itself a bounded-latency failure. Never leave FFmpeg
# back-pressured while the browser accumulates tens of seconds.
lease.failure_code = lease.failure_code or "consumer-too-slow"
while True:
try:
lease.segments.get_nowait()
except queue.Empty:
break
lease.segments.put_nowait(None)
def _read_fmp4_stdout(lease: CameraProcessLease) -> None:
stream = lease.process.stdout
if stream is None:
lease.failure_code = "invalid-fmp4"
_finish_segment_queue(lease)
return
init_parts: list[bytes] = []
fragment_parts: list[bytes] = []
init_sent = False
try:
while True:
box_type, box = _read_mp4_box(stream)
if not init_sent:
init_parts.append(box)
if box_type == b"moov":
if not _enqueue_segment(lease, "init", b"".join(init_parts)):
return
init_sent = True
continue
if box_type == b"moof":
fragment_parts = [box]
continue
if fragment_parts:
fragment_parts.append(box)
if box_type == b"mdat":
if not _enqueue_segment(lease, "media", b"".join(fragment_parts)):
return
fragment_parts = []
continue
# `styp`/`sidx` are valid media-segment prefixes. Preserve them and
# wait for the following moof+mdat instead of forwarding raw boxes.
if box_type in {b"styp", b"sidx"}:
fragment_parts.append(box)
except EOFError:
if not init_sent:
lease.failure_code = "invalid-fmp4"
except (OSError, ValueError):
lease.failure_code = "invalid-fmp4"
finally:
_finish_segment_queue(lease)
def _drain_stderr(lease: CameraProcessLease) -> None:
stream = lease.process.stderr
if stream is None:
return
try:
while True:
line = stream.readline()
if not line:
return
text = line.decode("utf-8", errors="replace").strip()
if text:
lease.stderr_tail.append(text[-240:])
except (OSError, ValueError):
return
def _safe_ffmpeg_message(stderr_tail: deque[str]) -> str:
# Never reflect the RTSP URL or device address into browser state.
joined = " ".join(stderr_tail).lower()
if "connection refused" in joined:
return "Camera endpoint отклонил локальное соединение."
if "timed out" in joined or "timeout" in joined:
return "Camera endpoint не ответил за отведённое время."
if "invalid data" in joined or "could not find codec" in joined:
return "Camera endpoint вернул неподдерживаемый media stream."
return "Camera stream завершился до первого пригодного видеофрагмента."
def _terminate_process(process: subprocess.Popen[bytes] | None) -> None:
if process is None:
return
try:
if process.poll() is None:
if os.name == "posix":
with suppress(ProcessLookupError):
os.killpg(process.pid, signal.SIGINT)
else:
process.terminate()
try:
process.wait(timeout=2.0)
except subprocess.TimeoutExpired:
process.terminate()
try:
process.wait(timeout=1.0)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=1.0)
finally:
for stream in (process.stdout, process.stderr):
if stream is not None:
with suppress(OSError):
stream.close()
def build_xgrids_k1_camera_router(
gateway: XgridsK1CameraGateway,
plugin_id: str,
) -> APIRouter:
router = APIRouter(include_in_schema=False)
@router.websocket(
f"/api/v1/device-plugins/{plugin_id}/camera-preview/{{generation}}"
)
async def camera_preview(websocket: WebSocket, generation: int) -> None:
await websocket.accept()
try:
lease = gateway.open_delivery(generation)
except (ValueError, RuntimeError):
await websocket.close(code=1008, reason="Camera preview lease is not active")
return
client_closed = False
try:
while True:
segment = await asyncio.to_thread(lease.segments.get)
if segment is None:
break
kind, payload = segment
if kind == "media":
gateway.mark_streaming(lease)
await websocket.send_bytes(payload)
except WebSocketDisconnect:
client_closed = True
except RuntimeError:
client_closed = True
finally:
gateway.release_delivery(lease, client_closed=client_closed)
with suppress(RuntimeError):
await websocket.close()
return router
src/k1link/web/xgrids_k1_facade.py
+173 -18
View File
@@ -35,6 +35,14 @@ from k1link.web.plugin_runtime import (
PluginActionNotFoundError,
PluginExecutionError,
)
from k1link.web.xgrids_k1_camera import (
CAMERA_EXCLUSIVE_GROUP,
CAMERA_SOURCE_LABELS,
CAMERA_SOURCE_PATHS,
CameraSourceId,
XgridsK1CameraGateway,
build_xgrids_k1_camera_router,
)
XGRIDS_K1_PLUGIN_ID = "nodedc.device.xgrids-lixelkity-k1"
XGRIDS_K1_MODEL_ID = "xgrids.lixelkity-k1"
@@ -55,6 +63,8 @@ ACTION_ACQUISITION_STATE_READ = "acquisition.state.read"
ACTION_STREAM_START_LIVE = "stream.start-live"
ACTION_STREAM_START_REPLAY = "stream.start-replay"
ACTION_STREAM_STOP = "stream.stop"
ACTION_CAMERA_PREVIEW_SELECT = "camera.preview.select"
ACTION_CAMERA_PREVIEW_STOP = "camera.preview.stop"
ACTION_VIEWER_SETTINGS_UPDATE = "viewer.settings.update"
RequestedStreamId = Literal[
@@ -145,6 +155,16 @@ class ReplayRequest(StrictRequest):
loop: bool = False
class CameraPreviewSelectRequest(StrictRequest):
source_id: CameraSourceId
device_session_id: str = Field(min_length=1, max_length=128)
class CameraPreviewStopRequest(StrictRequest):
device_session_id: str = Field(min_length=1, max_length=128)
generation: int = Field(ge=1)
class ViewerSettingsRequest(StrictRequest):
point_size: float = Field(default=2.5, ge=0.5, le=12.0)
color_mode: Literal["intensity", "height", "distance", "rgb", "class"] = "intensity"
@@ -189,10 +209,15 @@ class XgridsK1CompatibilityService:
# The host-owned visual runtime receives the vendor normalizer
# explicitly. There is no implicit K1 decoder in the visual layer.
self.runtime = VisualizationRuntime(normalizer=normalize_k1_message)
self.camera_preview = XgridsK1CameraGateway(
self.repository_root,
XGRIDS_K1_PLUGIN_ID,
)
def state(self) -> dict[str, Any]:
runtime = self.runtime.snapshot()
self._reconcile_acquisition(runtime)
camera_preview = self.camera_preview.snapshot()
metrics = runtime["metrics"]
with self._lock:
operation_phase = self._operation_phase
@@ -264,7 +289,11 @@ class XgridsK1CompatibilityService:
"attestation": compatibility_attestation,
"vendor_writes_enabled": False,
"camera_preview": (
"observed-runtime-adapter-pending"
(
"local-browser-adapter-available"
if camera_preview["runtime_dependency"]["status"] == "available"
else "local-runtime-dependency-missing"
)
if active_profile_id is not None
else "unverified"
),
@@ -292,8 +321,13 @@ class XgridsK1CompatibilityService:
else None
),
"connection_verification": connection_verification,
"sensor_catalog": _sensor_catalog(active_profile_id),
"sensor_catalog": _sensor_catalog(
active_profile_id,
device_session_id,
camera_preview,
),
"device_calibration": _device_calibration_snapshot(active_profile_id),
"camera_preview": camera_preview,
"acquisition": acquisition,
"operations": operation_documents,
"last_operation": operation_documents[-1] if operation_documents else None,
@@ -367,6 +401,9 @@ class XgridsK1CompatibilityService:
known_ids = {str(item["device_id"]) for item in self.state()["devices"]}
if request.device_id not in known_ids:
raise ValueError("сначала найдите и выберите устройство через Bluetooth")
# A reprovision can replace both the device session and its address.
# Revoke the previous browser lease before any new device-side write.
self.camera_preview.stop_current()
# Unwrap once at the provisioning service boundary. The plain value is
# kept only in this stack frame, included in a keyed request digest, and
# passed to the reviewed BLE write boundary; it is never journaled.
@@ -808,6 +845,7 @@ class XgridsK1CompatibilityService:
try:
with self._lock:
acquisition.transition("stopping", message_code="acquisition.stopping")
self.camera_preview.stop_current()
self.runtime.stop()
self._cancel_pending_acquisition_operations(
exclude_operation_id=operation.operation_id,
@@ -877,6 +915,7 @@ class XgridsK1CompatibilityService:
return self.state()
try:
if acquisition.state not in TERMINAL_ACQUISITION_STATES:
self.camera_preview.stop_current()
self.runtime.stop()
self._cancel_pending_acquisition_operations(
exclude_operation_id=operation.operation_id,
@@ -958,6 +997,7 @@ class XgridsK1CompatibilityService:
def stop(self) -> dict[str, Any]:
"""Deprecated capture-only shim; it never claims that K1 stopped scanning."""
self.camera_preview.stop_current()
with self._lock:
acquisition = self._acquisition
if acquisition is None or acquisition.state in TERMINAL_ACQUISITION_STATES:
@@ -970,6 +1010,34 @@ class XgridsK1CompatibilityService:
)
)
def select_camera_preview(self, request: CameraPreviewSelectRequest) -> dict[str, Any]:
target = self._camera_target_for_session(request.device_session_id)
self.camera_preview.select(request.source_id, target)
return self.state()
def stop_camera_preview(self, request: CameraPreviewStopRequest) -> dict[str, Any]:
self._camera_target_for_session(request.device_session_id)
self.camera_preview.stop(request.generation)
return self.state()
def close(self) -> None:
camera_error: Exception | None = None
try:
self.camera_preview.close()
except Exception as exc:
camera_error = exc
try:
self.runtime.close()
except Exception as exc:
if camera_error is not None:
exc.add_note(
"camera gateway cleanup also failed: "
f"{type(camera_error).__name__}: {camera_error}"
)
raise
if camera_error is not None:
raise camera_error
def update_viewer_settings(self, request: ViewerSettingsRequest) -> dict[str, Any]:
self.runtime.update_scene_settings(
RerunSceneSettings(
@@ -1034,6 +1102,22 @@ class XgridsK1CompatibilityService:
self._device_session_opened_at = _utc_now_iso()
return self._device_id, self._device_session_id
def _camera_target_for_session(self, device_session_id: str) -> str:
with self._lock:
current_session_id = self._device_session_id
target = self._k1_ip
attestation = self._compatibility_attestation
if current_session_id is None or device_session_id != current_session_id:
raise ValueError("указана неактивная device-сессия")
if attestation is None:
raise ValueError("точный compatibility-профиль K1 не подтверждён")
if target is None:
raise ValueError("у плагина нет подтверждённого локального адреса K1")
target = validate_private_ipv4(target)
if target == AP_FALLBACK_IPV4:
raise ValueError("адрес точки доступа K1 нельзя использовать как camera target")
return target
def _require_acquisition(self, acquisition_id: str | None) -> AcquisitionRecord:
with self._lock:
acquisition = self._acquisition
@@ -1212,6 +1296,10 @@ class XgridsK1ServicePort(Protocol):
def stop(self) -> dict[str, Any]: ...
def select_camera_preview(self, request: CameraPreviewSelectRequest) -> dict[str, Any]: ...
def stop_camera_preview(self, request: CameraPreviewStopRequest) -> dict[str, Any]: ...
def update_viewer_settings(self, request: ViewerSettingsRequest) -> dict[str, Any]: ...
@@ -1236,6 +1324,8 @@ class XgridsK1PluginFacade:
ACTION_STREAM_START_LIVE,
ACTION_STREAM_START_REPLAY,
ACTION_STREAM_STOP,
ACTION_CAMERA_PREVIEW_SELECT,
ACTION_CAMERA_PREVIEW_STOP,
ACTION_VIEWER_SETTINGS_UPDATE,
}
)
@@ -1309,6 +1399,18 @@ class XgridsK1PluginFacade:
if action_id == ACTION_STREAM_STOP:
EmptyRequest.model_validate(payload)
return await asyncio.to_thread(self.service.stop)
if action_id == ACTION_CAMERA_PREVIEW_SELECT:
select_camera_request = CameraPreviewSelectRequest.model_validate(payload)
return await asyncio.to_thread(
self.service.select_camera_preview,
select_camera_request,
)
if action_id == ACTION_CAMERA_PREVIEW_STOP:
stop_camera_request = CameraPreviewStopRequest.model_validate(payload)
return await asyncio.to_thread(
self.service.stop_camera_preview,
stop_camera_request,
)
if action_id == ACTION_VIEWER_SETTINGS_UPDATE:
settings_request = ViewerSettingsRequest.model_validate(payload)
return await asyncio.to_thread(
@@ -1362,8 +1464,67 @@ def _attestation_snapshot(
}
def _sensor_catalog(active_profile_id: str | None) -> dict[str, Any]:
def _sensor_catalog(
active_profile_id: str | None,
device_session_id: str | None,
camera_preview: Mapping[str, Any],
) -> dict[str, Any]:
camera_profile_active = active_profile_id is not None
camera_phase = str(camera_preview.get("phase") or "idle")
active_camera = camera_preview.get("active_source_id")
delivery = camera_preview.get("delivery")
dependency = camera_preview.get("runtime_dependency")
dependency_available = (
isinstance(dependency, Mapping) and dependency.get("status") == "available"
)
camera_streams: list[dict[str, Any]] = []
for source_id in CAMERA_SOURCE_PATHS:
side = "left" if source_id.endswith(".left") else "right"
selected = source_id == active_camera
if not camera_profile_active:
availability = "unverified"
elif not dependency_available:
availability = "unavailable"
elif selected and camera_phase == "streaming":
availability = "streaming"
elif selected and camera_phase == "error":
availability = "error"
elif selected:
availability = "connecting"
else:
availability = "available"
camera_streams.append(
{
"stream_id": f"camera.preview.{side}.live",
"source_id": source_id,
"semantic_channel_id": "camera.preview.live",
"label": CAMERA_SOURCE_LABELS[source_id],
"sensor_kind": "camera",
"modality": "encoded-video",
"availability": availability,
"decode_status": (
"rtsp-h264-observed-browser-remux"
if camera_profile_active and dependency_available
else (
"local-runtime-dependency-missing"
if camera_profile_active
else "profile-not-attested"
)
),
"endpoint_label": f"RTSP · {side}",
"activation": {
"group_id": CAMERA_EXCLUSIVE_GROUP,
"max_active": 1,
"selected": selected,
"controllable": bool(
camera_profile_active and device_session_id and dependency_available
),
},
"delivery": delivery if selected and isinstance(delivery, Mapping) else None,
"frame_id": None,
"coordinate_convention": "unresolved",
}
)
return {
"schema_version": "missioncore.sensor-catalog/v1alpha2",
"revision": active_profile_id or "unprofiled-evidence-only",
@@ -1404,19 +1565,7 @@ def _sensor_catalog(active_profile_id: str | None) -> dict[str, Any]:
"frame_id": None,
"coordinate_convention": None,
},
{
"stream_id": "camera.preview.live",
"sensor_kind": "camera-rig",
"modality": "encoded-video",
"availability": "observed" if camera_profile_active else "unverified",
"decode_status": (
"transport-observed-runtime-adapter-pending"
if camera_profile_active
else "profile-not-attested"
),
"frame_id": None,
"coordinate_convention": None,
},
*camera_streams,
],
}
@@ -1497,6 +1646,12 @@ def build_xgrids_k1_plugin(repository_root: Path) -> DevicePluginRuntimeContribu
adapter = XgridsK1PluginFacade(service)
return DevicePluginRuntimeContribution(
adapter=adapter,
legacy_routers=(build_xgrids_k1_legacy_router(adapter),),
close=service.runtime.close,
legacy_routers=(
build_xgrids_k1_legacy_router(adapter),
build_xgrids_k1_camera_router(
service.camera_preview,
XGRIDS_K1_PLUGIN_ID,
),
),
close=service.close,
)