feat(perception): connect binary ingress to the supervised full graph
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,64 @@
|
||||
"""Fenced decoder RPC; native execution lives in a separately supervised child."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
from .graph_contracts import GraphState
|
||||
from .streaming_lifecycle import StreamingLifecycle
|
||||
from .streaming_pipe_rpc import HEADER_SCRATCH, MAX_OUTPUT, BoundedPipeRpc, PipeRpcError
|
||||
|
||||
|
||||
class StreamingDecoderClient:
|
||||
def __init__(self, runtime: StreamingLifecycle, read_fd: int, write_fd: int) -> None:
|
||||
if runtime.state != GraphState.STARTING:
|
||||
raise PipeRpcError("decoder must be created during warmup")
|
||||
self.runtime = runtime
|
||||
self.reservation = runtime.mailbox.reserve_ingress(HEADER_SCRATCH)
|
||||
self.closed = False
|
||||
self.frames = 0
|
||||
try:
|
||||
self.rpc = BoundedPipeRpc(
|
||||
read_fd, write_fd, lambda: runtime.check_current(runtime.start, starting=True)
|
||||
)
|
||||
self.ready = self.rpc.exchange(None, b"", bytearray(), timeout=10)
|
||||
if self.ready != {"ready": "fragment-h264", "pyav": "18.0.0", "as_limit_mib": 1024}:
|
||||
raise PipeRpcError("decoder startup identity changed")
|
||||
self.rpc.check = lambda: runtime.check_current(runtime.start)
|
||||
except BaseException:
|
||||
runtime.request_stop("failed")
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def configure(self, raw: bytes) -> None:
|
||||
if self.closed or not 0 < len(raw) <= 65536:
|
||||
raise PipeRpcError("decoder init outside bound")
|
||||
if self.rpc.exchange({"op": "init"}, raw, bytearray()) != {"initialized": True}:
|
||||
raise PipeRpcError("decoder init response changed")
|
||||
|
||||
def decode(self, raw: bytes, target: bytearray) -> dict[str, Any]:
|
||||
# The caller has already reserved this exact BGR buffer in a decoded
|
||||
# bundle ticket. RPC fills it directly, without an unaccounted copy.
|
||||
if self.closed or len(target) != MAX_OUTPUT:
|
||||
raise PipeRpcError("decoder output allocation changed")
|
||||
result = self.rpc.exchange({"op": "decode"}, raw, target)
|
||||
duration = result.get("decode_ms")
|
||||
if (
|
||||
set(result) != {"decode_ms", "frame_index"}
|
||||
or type(result["frame_index"]) is not int
|
||||
or result["frame_index"] != self.frames
|
||||
or not isinstance(duration, (int, float))
|
||||
or isinstance(duration, bool)
|
||||
or not math.isfinite(duration)
|
||||
or duration < 0
|
||||
):
|
||||
raise PipeRpcError("decoder frame identity or measurement changed")
|
||||
self.frames += 1
|
||||
return result
|
||||
|
||||
def close(self) -> None:
|
||||
# Caller joins its callback and stops the owned child before closing.
|
||||
if not self.closed:
|
||||
self.closed = True
|
||||
self.reservation.release()
|
||||
@@ -21,7 +21,7 @@ from typing import Any, Literal
|
||||
from .graph_contracts import GraphState
|
||||
from .realtime_contract import StreamStart
|
||||
from .realtime_scene import _wire_integer
|
||||
from .streaming_queue import StreamBundle, StreamMailbox
|
||||
from .streaming_queue import IngressReservation, StreamBundle, StreamMailbox
|
||||
from .worker_lease import WorkerLease, WorkerLeaseError
|
||||
|
||||
|
||||
@@ -116,6 +116,13 @@ class StreamingLifecycle:
|
||||
self._check(start)
|
||||
return self.mailbox.put(bundle)
|
||||
|
||||
def admit_reserved(
|
||||
self, start: StreamStart, bundle: StreamBundle, reservation: IngressReservation
|
||||
) -> bool:
|
||||
with self._lock:
|
||||
self._check(start)
|
||||
return self.mailbox.put_reserved(bundle, reservation)
|
||||
|
||||
@contextmanager
|
||||
def work(self, start: StreamStart, lane: Literal["gpu", "cpu"]) -> Iterator[None]:
|
||||
with self._lock:
|
||||
@@ -133,9 +140,10 @@ class StreamingLifecycle:
|
||||
with self._lock:
|
||||
self._active[lane] -= 1
|
||||
|
||||
def check_current(self, start: StreamStart) -> None:
|
||||
def check_current(self, start: StreamStart, *, starting: bool = False) -> None:
|
||||
"""Only child startup handshakes may opt into the warmup state."""
|
||||
with self._lock:
|
||||
self._check(start)
|
||||
self._check(start, starting=starting)
|
||||
|
||||
def validate_result_binding(self, value: object) -> None:
|
||||
"""Recheck at receipt/use; accepting a hash alone cannot renew authority."""
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Bounded supervised-child RPC with caller-owned output buffers, POSIX only.
|
||||
|
||||
Same small header/binary framing as the existing pilot RPC. No process creation,
|
||||
paths, models, queues or implicit allocation of image payloads. Caller reserves
|
||||
the target buffer and HEADER_SCRATCH before using this transport. Native child
|
||||
memory is separately constrained; timeout poisons this connection, never retries.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import select
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Any
|
||||
|
||||
HEADER_SCRATCH = 65536
|
||||
MAX_HEADER = 4096
|
||||
MAX_INPUT = 1024 * 1024
|
||||
MAX_OUTPUT = 1_440_000
|
||||
|
||||
|
||||
class PipeRpcError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _unique(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise PipeRpcError("duplicate RPC field")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
class BoundedPipeRpc:
|
||||
def __init__(self, read_fd: int, write_fd: int, check: Callable[[], None]) -> None:
|
||||
self.read_fd, self.write_fd, self.check = read_fd, write_fd, check
|
||||
os.set_blocking(read_fd, False)
|
||||
os.set_blocking(write_fd, False)
|
||||
self._lock = threading.Lock()
|
||||
self.failed = False
|
||||
|
||||
def _check(self, deadline: float) -> None:
|
||||
self.check()
|
||||
if time.monotonic() >= deadline:
|
||||
raise PipeRpcError("decoder RPC deadline exceeded")
|
||||
|
||||
def _read(self, target: memoryview, deadline: float) -> None:
|
||||
offset = 0
|
||||
while offset < len(target):
|
||||
self._check(deadline)
|
||||
if not select.select([self.read_fd], [], [], 0.01)[0]:
|
||||
continue
|
||||
try:
|
||||
count = os.readv(self.read_fd, [target[offset:]])
|
||||
except BlockingIOError:
|
||||
continue
|
||||
if not count:
|
||||
raise PipeRpcError("truncated decoder RPC")
|
||||
offset += count
|
||||
self._check(deadline)
|
||||
|
||||
def _write(self, data: bytes | memoryview, deadline: float) -> None:
|
||||
view = memoryview(data)
|
||||
while view:
|
||||
self._check(deadline)
|
||||
if not select.select([], [self.write_fd], [], 0.01)[1]:
|
||||
continue
|
||||
try:
|
||||
count = os.write(self.write_fd, view)
|
||||
except BlockingIOError:
|
||||
continue
|
||||
if not count:
|
||||
raise PipeRpcError("decoder RPC write made no progress")
|
||||
view = view[count:]
|
||||
self._check(deadline)
|
||||
|
||||
def exchange(
|
||||
self,
|
||||
header: Mapping[str, Any] | None,
|
||||
payload: bytes,
|
||||
target: bytearray,
|
||||
*,
|
||||
timeout: float = 0.25,
|
||||
) -> dict[str, Any]:
|
||||
"""None header reads a one-time startup message; no work is submitted."""
|
||||
if not 0 < timeout <= (10 if header is None else 0.25):
|
||||
raise PipeRpcError("RPC timeout outside bound")
|
||||
if len(payload) > MAX_INPUT or len(target) > MAX_OUTPUT:
|
||||
raise PipeRpcError("RPC payload outside bound")
|
||||
if not self._lock.acquire(blocking=False):
|
||||
raise PipeRpcError("concurrent decoder RPC is forbidden")
|
||||
try:
|
||||
if self.failed:
|
||||
raise PipeRpcError("decoder RPC is poisoned")
|
||||
deadline = time.monotonic() + timeout
|
||||
self._check(deadline)
|
||||
if header is not None:
|
||||
raw = json.dumps(
|
||||
{**header, "payload_bytes": len(payload)}, allow_nan=False
|
||||
).encode()
|
||||
if not 0 < len(raw) <= MAX_HEADER:
|
||||
raise PipeRpcError("RPC header outside bound")
|
||||
self._write(struct.pack("<I", len(raw)), deadline)
|
||||
self._write(raw, deadline)
|
||||
self._write(payload, deadline)
|
||||
prefix = bytearray(4)
|
||||
self._read(memoryview(prefix), deadline)
|
||||
size = struct.unpack("<I", prefix)[0]
|
||||
if not 0 < size <= MAX_HEADER:
|
||||
raise PipeRpcError("RPC header outside bound")
|
||||
response = bytearray(size)
|
||||
self._read(memoryview(response), deadline)
|
||||
value = json.loads(response, object_pairs_hook=_unique)
|
||||
if not isinstance(value, dict):
|
||||
raise PipeRpcError("RPC header must be an object")
|
||||
length = value.pop("payload_bytes", None)
|
||||
if type(length) is not int or length != len(target):
|
||||
raise PipeRpcError("decoder response byte count changed")
|
||||
self._read(memoryview(target), deadline)
|
||||
return value
|
||||
except BaseException:
|
||||
self.failed = True
|
||||
raise
|
||||
finally:
|
||||
self._lock.release()
|
||||
@@ -96,6 +96,30 @@ class StreamMailbox:
|
||||
self.drop_counts[reason] += 1
|
||||
self._recent_drops.append({"sequence": bundle["sequence"], "reason": reason})
|
||||
|
||||
def put_reserved(self, bundle: StreamBundle, reservation: IngressReservation) -> bool:
|
||||
"""Atomically transfer preallocated decoded bytes into queue ownership.
|
||||
|
||||
Success consumes the reservation. On rejection/error the caller still
|
||||
owns it and must drop its buffers BEFORE releasing it. No unaccounted
|
||||
interval or transient double charge; active inputs cannot be evicted.
|
||||
"""
|
||||
with self.condition:
|
||||
size = self._ingress.get(reservation)
|
||||
if size is None or type(bundle["payload_bytes"]) is not int:
|
||||
raise ValueError("decoded handoff ownership mismatch")
|
||||
if bundle["payload_bytes"] != size:
|
||||
raise ValueError("decoded handoff size mismatch")
|
||||
del self._ingress[reservation]
|
||||
self.bytes -= size
|
||||
accepted = False
|
||||
try:
|
||||
accepted = self.put(bundle)
|
||||
return accepted
|
||||
finally:
|
||||
if not accepted:
|
||||
self._ingress[reservation] = size
|
||||
self.bytes += size
|
||||
|
||||
def _discard_pending(self, reason: str) -> None:
|
||||
old = self.pending.popleft()
|
||||
_, size = self._owned.pop(id(old))
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
"""Causal input binding with explicit preroll history and per-modality age.
|
||||
|
||||
No new inference or interpolation. The caller retains all preroll points in its
|
||||
bounded rolling window; only the initial current-pair selection is narrowed.
|
||||
After the first camera, the original increment admission rules are unchanged.
|
||||
"""
|
||||
|
||||
from collections import deque
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .streaming_queue import StreamMailbox
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SensorEvent:
|
||||
time_ns: int
|
||||
channel: str
|
||||
sequence: int
|
||||
value: Any
|
||||
|
||||
|
||||
POSE_AGE_NS = 100_000_000
|
||||
NEWEST_POINT_AGE_NS = 100_000_000
|
||||
OLDEST_POINT_AGE_NS = 250_000_000
|
||||
POINT_POSE_SKEW_NS = 100_000_000
|
||||
|
||||
|
||||
def increment_identity(event: SensorEvent) -> dict[str, int]:
|
||||
return {
|
||||
"sequence": event.sequence,
|
||||
"host_monotonic_ns": event.time_ns,
|
||||
"points": len(event.value[0]),
|
||||
}
|
||||
|
||||
|
||||
def milliseconds(value: int | None) -> float | None:
|
||||
return None if value is None else value / 1e6
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SensorBinding:
|
||||
increments: tuple[SensorEvent, ...]
|
||||
history_only: tuple[SensorEvent, ...]
|
||||
pose_age_ns: int | None
|
||||
newest_point_age_ns: int | None
|
||||
oldest_point_age_ns: int | None
|
||||
binding_age_ns: int | None
|
||||
pose_state: str
|
||||
points_state: str
|
||||
reasons: tuple[str, ...]
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return not self.reasons
|
||||
|
||||
def document(self) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": "missioncore.pilot-sensor-binding/v1",
|
||||
"pose_state": self.pose_state,
|
||||
"points_state": self.points_state,
|
||||
"current_pair_available": self.available,
|
||||
"reason_codes": list(self.reasons),
|
||||
"pose_age_ms": milliseconds(self.pose_age_ns),
|
||||
"newest_point_age_ms": milliseconds(self.newest_point_age_ns),
|
||||
"oldest_point_age_ms": milliseconds(self.oldest_point_age_ns),
|
||||
"point_pose_skew_ms": milliseconds(self.binding_age_ns),
|
||||
"preroll_history_only": [increment_identity(e) for e in self.history_only],
|
||||
"preroll_history_disposition": "retained-in-bounded-rolling-window",
|
||||
}
|
||||
|
||||
|
||||
def bind_sensors(
|
||||
camera_time_ns: int,
|
||||
pose: SensorEvent | None,
|
||||
increments: Sequence[SensorEvent],
|
||||
*,
|
||||
previous_camera_time_ns: int | None = None,
|
||||
) -> SensorBinding:
|
||||
increments = tuple(increments)
|
||||
if previous_camera_time_ns is not None and previous_camera_time_ns >= camera_time_ns:
|
||||
raise ValueError("camera binding clock must increase")
|
||||
if pose is not None and pose.time_ns > camera_time_ns:
|
||||
raise ValueError("future pose cannot bind to camera")
|
||||
if any(e.time_ns > camera_time_ns for e in increments):
|
||||
raise ValueError("future points cannot bind to camera")
|
||||
if any(a.time_ns > b.time_ns for a, b in zip(increments, increments[1:], strict=False)):
|
||||
raise ValueError("point binding clock moved backwards")
|
||||
|
||||
history_only: tuple[SensorEvent, ...] = ()
|
||||
if previous_camera_time_ns is None:
|
||||
# History used to warm rolling geometry is not one current increment.
|
||||
# Preserve its identity separately; do not retimestamp or silently drop it.
|
||||
selected: list[SensorEvent] = []
|
||||
history: list[SensorEvent] = []
|
||||
for event in increments:
|
||||
is_current = (
|
||||
pose is not None
|
||||
and camera_time_ns - event.time_ns <= OLDEST_POINT_AGE_NS
|
||||
and abs(event.time_ns - pose.time_ns) <= POINT_POSE_SKEW_NS
|
||||
)
|
||||
(selected if is_current else history).append(event)
|
||||
increments, history_only = tuple(selected), tuple(history)
|
||||
|
||||
pose_age = None if pose is None else camera_time_ns - pose.time_ns
|
||||
newest_age = None if not increments else camera_time_ns - increments[-1].time_ns
|
||||
oldest_age = None if not increments else camera_time_ns - increments[0].time_ns
|
||||
skew = (
|
||||
max(abs(e.time_ns - pose.time_ns) for e in increments)
|
||||
if increments and pose is not None
|
||||
else None
|
||||
)
|
||||
reasons = []
|
||||
if pose is None:
|
||||
pose_state = "unavailable"
|
||||
reasons.append("pose-unavailable")
|
||||
elif pose_age is not None and pose_age > POSE_AGE_NS:
|
||||
pose_state = "stale"
|
||||
reasons.append("pose-too-old")
|
||||
else:
|
||||
pose_state = (
|
||||
"held"
|
||||
if previous_camera_time_ns is not None and pose.time_ns <= previous_camera_time_ns
|
||||
else "current"
|
||||
)
|
||||
if not increments or not any(len(e.value[0]) for e in increments):
|
||||
points_state = "unavailable"
|
||||
reasons.append("point-increment-unavailable")
|
||||
else:
|
||||
points_state = "current"
|
||||
if newest_age is not None and newest_age > NEWEST_POINT_AGE_NS:
|
||||
points_state = "stale"
|
||||
reasons.append("newest-points-too-old")
|
||||
if oldest_age is not None and oldest_age > OLDEST_POINT_AGE_NS:
|
||||
points_state = "stale"
|
||||
reasons.append("oldest-points-too-old")
|
||||
if skew is not None and skew > POINT_POSE_SKEW_NS:
|
||||
reasons.append("point-pose-skew")
|
||||
return SensorBinding(
|
||||
increments,
|
||||
history_only,
|
||||
pose_age,
|
||||
newest_age,
|
||||
oldest_age,
|
||||
skew,
|
||||
pose_state,
|
||||
points_state,
|
||||
tuple(reasons),
|
||||
)
|
||||
|
||||
|
||||
def normalized_sensor(modality: str, sequence: int, time_ns: int, raw: bytes) -> SensorEvent:
|
||||
"""Explicit existing normalized-map wire layout, not a vendor packet parser."""
|
||||
if modality == "lidar":
|
||||
count = int.from_bytes(raw[:4], "little")
|
||||
if len(raw) < 4 or not 0 <= count <= 50000 or len(raw) != 4 + count * 25:
|
||||
raise ValueError("normalized point layout outside bound")
|
||||
xyz = np.frombuffer(raw, "<f8", count=count * 3, offset=4).reshape(count, 3)
|
||||
intensity = np.frombuffer(raw, "u1", count=count, offset=4 + count * 24)
|
||||
# Bound validation temporaries inside the window's metadata/scratch
|
||||
# allowance, even for a maximum-size incoming point increment.
|
||||
flat = xyz.reshape(-1)
|
||||
for offset in range(0, flat.size, 4096):
|
||||
if not np.isfinite(flat[offset : offset + 4096]).all():
|
||||
raise ValueError("nonfinite source cloud")
|
||||
return SensorEvent(time_ns, "points", sequence, (xyz, intensity))
|
||||
if modality != "pose" or len(raw) != 56:
|
||||
raise ValueError("normalized pose layout outside bound")
|
||||
values = np.frombuffer(raw, "<f8")
|
||||
if not np.isfinite(values).all() or abs(float(np.linalg.norm(values[3:])) - 1) > 0.01:
|
||||
raise ValueError("invalid source pose")
|
||||
return SensorEvent(time_ns, "pose", sequence, (values[:3], values[3:]))
|
||||
|
||||
|
||||
class CausalSensorWindow:
|
||||
"""Single-consumer rolling/fresh caches; no async nearest/future lookup.
|
||||
|
||||
Reserve the maximum union of two 64000-point sets before retaining raw
|
||||
views. Metadata and pose allowance is separate; Python objects/native RSS
|
||||
still need the enclosing process limit. Sensor selection rules above are
|
||||
exactly the existing pilot rules, including the initial history-only cut.
|
||||
"""
|
||||
|
||||
CACHE_BYTES = 2 * 64000 * 25 + 16384
|
||||
|
||||
def __init__(self, mailbox: StreamMailbox) -> None:
|
||||
self._reservation = mailbox.reserve_ingress(self.CACHE_BYTES)
|
||||
self.rolling: deque[SensorEvent] = deque()
|
||||
self.fresh: list[SensorEvent] = []
|
||||
self.pose: SensorEvent | None = None
|
||||
self.previous_camera_time: int | None = None
|
||||
self._last_time = -1
|
||||
self.closed = False
|
||||
|
||||
def advance(self, time_ns: int) -> None:
|
||||
if self.closed or time_ns < self._last_time:
|
||||
raise ValueError("sensor stream is closed or moved backwards")
|
||||
self._last_time = time_ns
|
||||
while self.rolling and time_ns - self.rolling[0].time_ns > 1_000_000_000:
|
||||
self.rolling.popleft()
|
||||
|
||||
def append(self, event: SensorEvent) -> None:
|
||||
self.advance(event.time_ns)
|
||||
if event.channel == "pose":
|
||||
self.pose = event
|
||||
return
|
||||
if event.channel != "points":
|
||||
raise ValueError("unsupported normalized sensor channel")
|
||||
for items in (self.rolling, self.fresh):
|
||||
if (
|
||||
len(items) >= 64
|
||||
or sum(len(e.value[0]) for e in items) + len(event.value[0]) > 64000
|
||||
):
|
||||
raise ValueError("causal sensor cache exceeds bound")
|
||||
self.rolling.append(event)
|
||||
self.fresh.append(event)
|
||||
|
||||
def bind(self, time_ns: int) -> SensorBinding:
|
||||
self.advance(time_ns)
|
||||
return bind_sensors(
|
||||
time_ns, self.pose, self.fresh, previous_camera_time_ns=self.previous_camera_time
|
||||
)
|
||||
|
||||
def finish_camera(self, time_ns: int) -> None:
|
||||
if self.closed or time_ns != self._last_time:
|
||||
raise ValueError("sensor window changed during camera binding")
|
||||
self.previous_camera_time = time_ns
|
||||
self.fresh.clear()
|
||||
|
||||
def close(self) -> None:
|
||||
if not self.closed:
|
||||
self.rolling.clear()
|
||||
self.fresh.clear()
|
||||
self.pose = None
|
||||
self.closed = True
|
||||
self._reservation.release()
|
||||
Reference in New Issue
Block a user