feat(perception): add bounded binary stream ingress
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,293 @@
|
||||
"""One controller-supplied IPC socket feeding the common streaming lifecycle.
|
||||
|
||||
No listener, acquisition, archive reader, model RPC, network authentication or
|
||||
automatic reconnect. Reconnect needs a new StreamStart/lease and decoder state.
|
||||
The synchronous consumer is a trusted adapter: raw events are borrowed for that
|
||||
call only. It must reserve decoder scratch before allocation and retain decoded
|
||||
work only through the common mailbox. Slow/failed consumers fail this stream;
|
||||
they do not create another buffering queue.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import select
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import Callable
|
||||
from hashlib import sha256
|
||||
from typing import Any
|
||||
|
||||
from k1link.compute.live_perception import LiveIngressEvent
|
||||
|
||||
from . import streaming_wire as wire
|
||||
from .streaming_lifecycle import StreamingLifecycle
|
||||
|
||||
|
||||
class StreamingIngress:
|
||||
def __init__(
|
||||
self,
|
||||
connection: socket.socket,
|
||||
runtime: StreamingLifecycle,
|
||||
session_id: str,
|
||||
session_generation: int,
|
||||
consume: Callable[[LiveIngressEvent], None],
|
||||
notice: Callable[[dict[str, Any]], None],
|
||||
*,
|
||||
fragment_timeout: float = 0.25,
|
||||
idle_timeout: float = 2.0,
|
||||
) -> None:
|
||||
if not 0 < fragment_timeout <= 2 or not 0 < idle_timeout <= 30:
|
||||
raise ValueError("invalid bounded ingress timeouts")
|
||||
wire.identifier(session_id)
|
||||
wire.decimal(session_generation)
|
||||
if session_generation < 1:
|
||||
raise ValueError("invalid acquisition generation")
|
||||
self.connection, self.runtime = connection, runtime
|
||||
connection.setblocking(False)
|
||||
self.session_id, self.session_generation = session_id, session_generation
|
||||
self.consume, self.notice = consume, notice
|
||||
self.fragment_timeout, self.idle_timeout = fragment_timeout, idle_timeout
|
||||
self.binding = wire.binding(runtime.start)
|
||||
self.opened = False
|
||||
self.error: str | None = None
|
||||
self.terminal: str | None = None
|
||||
self.counts: Counter[str] = Counter()
|
||||
self.last_ingress_sequence = 0
|
||||
self.channel_sequences: dict[str, tuple[str, int, int]] = {}
|
||||
self.camera_initialized = False
|
||||
self.thread = threading.Thread(
|
||||
target=self._serve, name="perception-binary-ingress", daemon=True
|
||||
)
|
||||
runtime.track_thread(self.thread)
|
||||
|
||||
def start(self) -> None:
|
||||
self.runtime.check_current(self.runtime.start)
|
||||
self.thread.start()
|
||||
|
||||
def join(self, timeout: float = 1.0) -> bool:
|
||||
self.thread.join(timeout=timeout)
|
||||
return not self.thread.is_alive()
|
||||
|
||||
def _check(self, deadline: float) -> None:
|
||||
self.runtime.check_current(self.runtime.start)
|
||||
if time.monotonic() >= deadline:
|
||||
raise wire.StreamWireError("ingress deadline exceeded")
|
||||
|
||||
def _read_into(self, target: memoryview, deadline: float) -> None:
|
||||
offset = 0
|
||||
while offset < len(target):
|
||||
self._check(deadline)
|
||||
if not select.select(
|
||||
[self.connection], [], [], min(0.05, max(0, deadline - time.monotonic()))
|
||||
)[0]:
|
||||
continue
|
||||
try:
|
||||
read = self.connection.recv_into(target[offset:])
|
||||
except BlockingIOError:
|
||||
continue
|
||||
if read == 0:
|
||||
raise wire.StreamWireError("unexpected EOF without explicit End")
|
||||
self.counts["wire_bytes"] += read
|
||||
offset += read
|
||||
self._check(deadline)
|
||||
|
||||
def _header(self, deadline: float) -> tuple[int, dict[str, Any]]:
|
||||
prefix = bytearray(wire.PREFIX.size)
|
||||
self._read_into(memoryview(prefix), deadline)
|
||||
magic, kind, size = wire.PREFIX.unpack(prefix)
|
||||
if (
|
||||
magic != wire.MAGIC
|
||||
or kind not in (wire.OPEN, wire.FRAGMENT, wire.END, wire.CANCEL, wire.GAP)
|
||||
or not 0 < size <= wire.MAX_HEADER
|
||||
):
|
||||
raise wire.StreamWireError("invalid frame prefix or header size")
|
||||
raw = bytearray(size)
|
||||
self._read_into(memoryview(raw), deadline)
|
||||
return kind, wire.parse_header(raw)
|
||||
|
||||
def _bound(self, header: dict[str, Any]) -> None:
|
||||
if header.get("binding") != self.binding:
|
||||
raise wire.StreamWireError("frame belongs to another StreamStart")
|
||||
self.runtime.check_current(self.runtime.start)
|
||||
|
||||
def _fragment(self, header: dict[str, Any]) -> tuple[dict[str, Any], int, int, int]:
|
||||
if set(header) != {
|
||||
"binding",
|
||||
"event",
|
||||
"total_bytes",
|
||||
"offset",
|
||||
"fragment_bytes",
|
||||
"payload_sha256",
|
||||
"fragment_sha256",
|
||||
}:
|
||||
raise wire.StreamWireError("fragment fields changed")
|
||||
self._bound(header)
|
||||
event = wire.validate_event_metadata(header["event"])
|
||||
if (
|
||||
event["session_id"] != self.session_id
|
||||
or event["session_generation"] != self.session_generation
|
||||
):
|
||||
raise wire.StreamWireError("event acquisition binding changed")
|
||||
total, offset, size = (header[x] for x in ("total_bytes", "offset", "fragment_bytes"))
|
||||
if any(type(x) is not int for x in (total, offset, size)) or not (
|
||||
0 < total <= wire.PAYLOAD_LIMITS[event["modality"]]
|
||||
and 0 <= offset < total
|
||||
and 0 < size <= wire.MAX_FRAGMENT
|
||||
and size == min(wire.MAX_FRAGMENT, total - offset)
|
||||
):
|
||||
raise wire.StreamWireError("invalid fragment length or offset")
|
||||
wire.digest(header["payload_sha256"])
|
||||
wire.digest(header["fragment_sha256"])
|
||||
return event, total, offset, size
|
||||
|
||||
def _observation(self, first: dict[str, Any], deadline: float) -> None:
|
||||
event, total, offset, size = self._fragment(first)
|
||||
modality = event["modality"]
|
||||
if offset != 0 or event["ingress_sequence"] <= self.last_ingress_sequence:
|
||||
raise wire.StreamWireError("duplicate, out-of-order or incomplete observation")
|
||||
previous = self.channel_sequences.get(modality)
|
||||
if previous and (
|
||||
event["source_id"] != previous[0]
|
||||
or event["source_sequence"] <= previous[1]
|
||||
or event["received_monotonic_ns"] < previous[2]
|
||||
):
|
||||
raise wire.StreamWireError("channel identity, sequence or source clock regressed")
|
||||
if modality == "camera-frame" and not self.camera_initialized:
|
||||
raise wire.StreamWireError("camera frame requires new codec initialization")
|
||||
if modality == "camera-frame" and previous and event["source_sequence"] != previous[1] + 1:
|
||||
raise wire.StreamWireError("camera sequence gap requires new codec initialization")
|
||||
self.counts["observations_started"] += 1
|
||||
reservation = self.runtime.mailbox.reserve_ingress(2 * total)
|
||||
raw = None
|
||||
admitted = None
|
||||
try:
|
||||
raw = bytearray(total)
|
||||
header = first
|
||||
while True:
|
||||
chunk = memoryview(raw)[offset : offset + size]
|
||||
try:
|
||||
self._read_into(chunk, deadline)
|
||||
if sha256(chunk).hexdigest() != header["fragment_sha256"]:
|
||||
raise wire.StreamWireError("fragment integrity mismatch")
|
||||
finally:
|
||||
chunk.release()
|
||||
self.counts["fragments"] += 1
|
||||
offset += size
|
||||
if offset == total:
|
||||
break
|
||||
kind, header = self._header(deadline)
|
||||
if kind != wire.FRAGMENT:
|
||||
raise wire.StreamWireError("incomplete observation before control event")
|
||||
next_event, next_total, next_offset, size = self._fragment(header)
|
||||
if (
|
||||
next_event != event
|
||||
or next_total != total
|
||||
or next_offset != offset
|
||||
or header["payload_sha256"] != first["payload_sha256"]
|
||||
):
|
||||
raise wire.StreamWireError("interleaved or discontinuous fragments")
|
||||
if sha256(raw).hexdigest() != first["payload_sha256"]:
|
||||
raise wire.StreamWireError("observation integrity mismatch")
|
||||
self._check(deadline)
|
||||
admitted = LiveIngressEvent(**event, payload=bytes(raw))
|
||||
raw = None # Immutable callback bytes remain fully reserved.
|
||||
self.consume(admitted)
|
||||
self._check(deadline)
|
||||
self.counts["ingress_sequence_gaps"] += (
|
||||
event["ingress_sequence"] - self.last_ingress_sequence - 1
|
||||
)
|
||||
self.last_ingress_sequence = event["ingress_sequence"]
|
||||
self.channel_sequences[modality] = (
|
||||
event["source_id"],
|
||||
event["source_sequence"],
|
||||
event["received_monotonic_ns"],
|
||||
)
|
||||
if modality == "camera-init":
|
||||
self.camera_initialized = True
|
||||
self.channel_sequences.pop("camera-frame", None)
|
||||
self.counts["observations_completed"] += 1
|
||||
finally:
|
||||
admitted = raw = None
|
||||
reservation.release()
|
||||
|
||||
def _gap(self, header: dict[str, Any]) -> None:
|
||||
if set(header) != {"binding", "modality", "reason", "count"}:
|
||||
raise wire.StreamWireError("gap fields changed")
|
||||
if header["modality"] not in wire.PAYLOAD_LIMITS or header["reason"] not in (
|
||||
"unavailable",
|
||||
"source-gap",
|
||||
"overload",
|
||||
):
|
||||
raise wire.StreamWireError("invalid gap notice")
|
||||
self.counts["declared_gap_observations"] += wire.uint64(header["count"])
|
||||
if header["modality"].startswith("camera"):
|
||||
self.camera_initialized = False
|
||||
self.notice(dict(header))
|
||||
self.runtime.check_current(self.runtime.start)
|
||||
self.counts["gap_notices"] += 1
|
||||
|
||||
def _serve(self) -> None:
|
||||
reservation = None
|
||||
try:
|
||||
# Covers raw header + JSON decoding copies; Python object overhead
|
||||
# belongs to the separately measured RSS envelope, not payload bytes.
|
||||
reservation = self.runtime.mailbox.reserve_ingress(
|
||||
3 * wire.MAX_HEADER + wire.PREFIX.size
|
||||
)
|
||||
kind, header = self._header(time.monotonic() + self.idle_timeout)
|
||||
expected = wire.parse_header(
|
||||
bytearray(
|
||||
wire.open_packet(
|
||||
self.runtime.start,
|
||||
self.session_id,
|
||||
self.session_generation,
|
||||
)[wire.PREFIX.size :]
|
||||
)
|
||||
)
|
||||
if kind != wire.OPEN or header != expected:
|
||||
raise wire.StreamWireError("StreamStart or acquisition handshake mismatch")
|
||||
self.opened = True
|
||||
while True:
|
||||
kind, header = self._header(time.monotonic() + self.idle_timeout)
|
||||
self._bound(header)
|
||||
if kind == wire.FRAGMENT:
|
||||
self._observation(header, time.monotonic() + self.fragment_timeout)
|
||||
elif kind == wire.GAP:
|
||||
self._gap(header)
|
||||
elif kind in (wire.END, wire.CANCEL) and set(header) == {"binding"}:
|
||||
self.terminal = "cancelled" if kind == wire.CANCEL else "end"
|
||||
if kind == wire.CANCEL:
|
||||
self.runtime.request_stop("cancelled")
|
||||
else:
|
||||
self.runtime.mailbox.finish()
|
||||
return
|
||||
else:
|
||||
raise wire.StreamWireError("unexpected stream control event")
|
||||
except Exception as exc:
|
||||
self.error = str(exc)
|
||||
self.terminal = "failed"
|
||||
if self.opened:
|
||||
self.runtime.mailbox.finish(self.error)
|
||||
self.runtime.request_stop("failed")
|
||||
finally:
|
||||
self.connection.close()
|
||||
if reservation is not None:
|
||||
reservation.release()
|
||||
|
||||
def snapshot(self) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": wire.SCHEMA,
|
||||
"opened": self.opened,
|
||||
"terminal": self.terminal,
|
||||
"error": self.error,
|
||||
"counts": dict(self.counts),
|
||||
"incomplete_observations": self.counts["observations_started"]
|
||||
- self.counts["observations_completed"],
|
||||
"source_duration_known": False,
|
||||
"requires_eof_before_delivery": False,
|
||||
"transport": "controller-supplied-ipc-socket",
|
||||
"network_qualified": False,
|
||||
"actuation_allowed": False,
|
||||
}
|
||||
@@ -17,6 +17,21 @@ StreamBundle = Mapping[str, Any]
|
||||
DISCARD_REASONS = frozenset(("cancelled", "gpu-failed", "processing-failed"))
|
||||
|
||||
|
||||
class IngressReservation:
|
||||
"""Raw/reassembly bytes retained outside the decoded work queue.
|
||||
|
||||
Trusted adapters reserve BEFORE allocation and release AFTER borrowed raw
|
||||
buffers/callbacks are gone. Closing ingress never frees a live reservation.
|
||||
This is byte ownership, not another pending camera slot.
|
||||
"""
|
||||
|
||||
def __init__(self, mailbox: StreamMailbox, size: int) -> None:
|
||||
self._mailbox, self.size = mailbox, size
|
||||
|
||||
def release(self) -> None:
|
||||
self._mailbox._release_ingress(self)
|
||||
|
||||
|
||||
class StreamMailbox:
|
||||
def __init__(self, capacity: int = 2, byte_limit: int = 16 * 1024 * 1024) -> None:
|
||||
if type(capacity) is not int or not 1 <= capacity <= 2:
|
||||
@@ -31,6 +46,7 @@ class StreamMailbox:
|
||||
self._recent_drops: deque[dict[str, Any]] = deque(maxlen=256)
|
||||
self._owned: dict[int, tuple[StreamBundle, int]] = {}
|
||||
self._active: set[int] = set()
|
||||
self._ingress: dict[IngressReservation, int] = {}
|
||||
self._last_sequence = -1
|
||||
self.done = False
|
||||
self.error: str | None = None
|
||||
@@ -49,7 +65,32 @@ class StreamMailbox:
|
||||
@property
|
||||
def quiescent(self) -> bool:
|
||||
with self.condition:
|
||||
return self.done and not self._owned and not self.external_pending
|
||||
return self.done and not self._owned and not self.external_pending and not self._ingress
|
||||
|
||||
def reserve_ingress(self, size: int) -> IngressReservation:
|
||||
"""Charge raw buffers to the SAME budget as queued and active inputs.
|
||||
|
||||
Fail closed on exhaustion; do not wait and silently stretch source time.
|
||||
Decoder scratch must also be reserved before allocation; retained decoded
|
||||
work uses the existing bundle ownership. No adapter may hide a buffer.
|
||||
"""
|
||||
if type(size) is not int or size <= 0:
|
||||
raise ValueError("invalid ingress reservation")
|
||||
with self.condition:
|
||||
if self.done or self.bytes + size > self.byte_limit or len(self._ingress) >= 8:
|
||||
raise ValueError("ingress byte budget exhausted or closed")
|
||||
reservation = IngressReservation(self, size)
|
||||
self._ingress[reservation] = size
|
||||
self.bytes += size
|
||||
self.peak_bytes = max(self.peak_bytes, self.bytes)
|
||||
return reservation
|
||||
|
||||
def _release_ingress(self, reservation: IngressReservation) -> None:
|
||||
with self.condition:
|
||||
if reservation not in self._ingress:
|
||||
raise ValueError("ingress release ownership mismatch")
|
||||
self.bytes -= self._ingress.pop(reservation)
|
||||
self.condition.notify_all()
|
||||
|
||||
def _drop(self, bundle: StreamBundle, reason: str) -> None:
|
||||
self.drop_counts[reason] += 1
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Synchronous, deadline-bounded IPC writer for existing raw ingress events.
|
||||
|
||||
The caller owns pacing and already-committed source bytes. No background queue,
|
||||
recording inventory or retry is created; timeout is an explicit failed source
|
||||
outcome. A new connection requires a new controller-approved StreamStart.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import select
|
||||
import socket
|
||||
import time
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
from k1link.compute.live_perception import LiveIngressEvent
|
||||
|
||||
from . import streaming_wire as wire
|
||||
from .realtime_contract import StreamStart
|
||||
|
||||
|
||||
class StreamingSender:
|
||||
def __init__(
|
||||
self,
|
||||
connection: socket.socket,
|
||||
start: StreamStart,
|
||||
session_id: str,
|
||||
session_generation: int,
|
||||
check_current: Callable[[], None],
|
||||
*,
|
||||
timeout: float = 0.25,
|
||||
) -> None:
|
||||
if not 0 < timeout <= 2:
|
||||
raise ValueError("invalid bounded send timeout")
|
||||
self.connection, self.identity, self.check_current = connection, start, check_current
|
||||
self.session_id, self.session_generation = session_id, session_generation
|
||||
self.timeout, self.closed = timeout, False
|
||||
connection.setblocking(False)
|
||||
try:
|
||||
self._send((wire.open_packet(start, session_id, session_generation),))
|
||||
except Exception:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def _send(self, pieces: Iterable[bytes | memoryview]) -> None:
|
||||
if self.closed:
|
||||
raise wire.StreamWireError("sender is closed")
|
||||
deadline = time.monotonic() + self.timeout
|
||||
try:
|
||||
for piece in pieces:
|
||||
view = memoryview(piece)
|
||||
offset = 0
|
||||
while offset < len(view):
|
||||
self.check_current()
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise wire.StreamWireError("send deadline exceeded")
|
||||
if not select.select([], [self.connection], [], min(0.05, remaining))[1]:
|
||||
continue
|
||||
try:
|
||||
count = self.connection.send(view[offset:])
|
||||
except BlockingIOError:
|
||||
continue
|
||||
if count == 0:
|
||||
raise wire.StreamWireError("source connection closed")
|
||||
offset += count
|
||||
self.check_current()
|
||||
if time.monotonic() >= deadline:
|
||||
raise wire.StreamWireError("send deadline exceeded")
|
||||
except Exception:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def send(self, event: LiveIngressEvent) -> None:
|
||||
if (
|
||||
event.session_id != self.session_id
|
||||
or event.session_generation != self.session_generation
|
||||
):
|
||||
self.close()
|
||||
raise wire.StreamWireError("source acquisition binding changed")
|
||||
self._send(piece for pair in wire.event_packets(self.identity, event) for piece in pair)
|
||||
|
||||
def gap(self, *, modality: str, reason: str, count: int) -> None:
|
||||
self._send((wire.gap_packet(self.identity, modality=modality, reason=reason, count=count),))
|
||||
|
||||
def end(self, *, cancel: bool = False) -> None:
|
||||
self._send((wire.terminal_packet(self.identity, cancel=cancel),))
|
||||
self.close()
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
self.connection.close()
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Bounded v2 mapping of existing raw-first LiveIngressEvent semantics.
|
||||
|
||||
This IPC candidate carries binary payloads, not base64/PNG/archive uploads.
|
||||
It is not the selected/authenticated network transport. A trusted controller
|
||||
binds one acquisition session to StreamStart; hashes provide integrity, NOT
|
||||
authentication. No peer-supplied paths, commands, models or calibration loading.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import struct
|
||||
from collections.abc import Iterator
|
||||
from hashlib import sha256
|
||||
from typing import Any, Final
|
||||
|
||||
from k1link.compute.live_perception import LiveIngressEvent
|
||||
|
||||
from .realtime_contract import StreamStart
|
||||
|
||||
SCHEMA: Final = "missioncore.live-perception-wire/v2"
|
||||
MAGIC: Final = b"MCI2"
|
||||
PREFIX: Final = struct.Struct("!4sBI")
|
||||
OPEN, FRAGMENT, END, CANCEL, GAP = range(1, 6)
|
||||
MAX_HEADER: Final = 64 * 1024
|
||||
MAX_FRAGMENT: Final = 1024 * 1024
|
||||
PAYLOAD_LIMITS: Final = {
|
||||
"camera-init": MAX_FRAGMENT,
|
||||
"camera-frame": MAX_FRAGMENT,
|
||||
"lidar": 2 * MAX_FRAGMENT,
|
||||
"pose": 2 * MAX_FRAGMENT,
|
||||
}
|
||||
EVENT_FIELDS: Final = frozenset(
|
||||
(
|
||||
"ingress_sequence",
|
||||
"session_id",
|
||||
"session_generation",
|
||||
"modality",
|
||||
"source_id",
|
||||
"source_sequence",
|
||||
"captured_at_epoch_ns",
|
||||
"received_monotonic_ns",
|
||||
)
|
||||
)
|
||||
INTEGER_FIELDS: Final = EVENT_FIELDS - {"session_id", "modality", "source_id"}
|
||||
_DECIMAL: Final = re.compile(r"^(0|[1-9][0-9]{0,19})$")
|
||||
_DIGEST: Final = re.compile(r"^[a-f0-9]{64}$")
|
||||
|
||||
|
||||
class StreamWireError(ValueError):
|
||||
"""Malformed, stale, oversized or incomplete transport input."""
|
||||
|
||||
|
||||
def uint64(value: object) -> int:
|
||||
if not isinstance(value, str) or not _DECIMAL.fullmatch(value):
|
||||
raise StreamWireError("wire uint64 must be a canonical decimal string")
|
||||
result = int(value)
|
||||
if result > (1 << 64) - 1:
|
||||
raise StreamWireError("wire uint64 overflow")
|
||||
return result
|
||||
|
||||
|
||||
def decimal(value: object) -> str:
|
||||
if type(value) is not int or not 0 <= value < 1 << 64:
|
||||
raise StreamWireError("invalid uint64")
|
||||
return str(value)
|
||||
|
||||
|
||||
def identifier(value: object) -> str:
|
||||
if not isinstance(value, str) or not 1 <= len(value) <= 160:
|
||||
raise StreamWireError("invalid bounded source identity")
|
||||
if not value.isascii() or any(ord(c) < 33 or ord(c) > 126 for c in value):
|
||||
raise StreamWireError("invalid bounded source identity")
|
||||
return value
|
||||
|
||||
|
||||
def digest(value: object) -> str:
|
||||
if not isinstance(value, str) or not _DIGEST.fullmatch(value):
|
||||
raise StreamWireError("invalid payload digest")
|
||||
return value
|
||||
|
||||
|
||||
def canonical(value: dict[str, Any]) -> bytes:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
|
||||
|
||||
|
||||
def start_document(start: StreamStart) -> dict[str, Any]:
|
||||
result = start.to_dict()
|
||||
result["lease_generation"] = decimal(start.lease_generation)
|
||||
return result
|
||||
|
||||
|
||||
def binding(start: StreamStart) -> str:
|
||||
return sha256(canonical(start_document(start))).hexdigest()
|
||||
|
||||
|
||||
def _unique_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise StreamWireError("duplicate metadata field")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def parse_header(raw: bytearray) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(raw, object_pairs_hook=_unique_pairs)
|
||||
except (ValueError, UnicodeError, RecursionError) as exc:
|
||||
raise StreamWireError("invalid bounded metadata") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise StreamWireError("metadata must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def packet(kind: int, header: dict[str, Any]) -> bytes:
|
||||
raw = canonical(header)
|
||||
if kind not in (OPEN, FRAGMENT, END, CANCEL, GAP) or not 0 < len(raw) <= MAX_HEADER:
|
||||
raise StreamWireError("invalid frame header")
|
||||
return PREFIX.pack(MAGIC, kind, len(raw)) + raw
|
||||
|
||||
|
||||
def open_packet(start: StreamStart, session_id: str, session_generation: int) -> bytes:
|
||||
return packet(
|
||||
OPEN,
|
||||
{
|
||||
"schema_version": SCHEMA,
|
||||
"start": start_document(start),
|
||||
"session_id": identifier(session_id),
|
||||
"session_generation": decimal(session_generation),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def event_metadata(event: LiveIngressEvent) -> dict[str, Any]:
|
||||
result = {name: getattr(event, name) for name in EVENT_FIELDS}
|
||||
for name in INTEGER_FIELDS:
|
||||
result[name] = decimal(result[name])
|
||||
validate_event_metadata(result)
|
||||
return result
|
||||
|
||||
|
||||
def validate_event_metadata(value: object) -> dict[str, Any]:
|
||||
if not isinstance(value, dict) or set(value) != EVENT_FIELDS:
|
||||
raise StreamWireError("event fields changed")
|
||||
result = dict(value)
|
||||
for name in INTEGER_FIELDS:
|
||||
result[name] = uint64(value[name])
|
||||
for name in ("session_id", "source_id"):
|
||||
identifier(value[name])
|
||||
if (
|
||||
not isinstance(value["modality"], str)
|
||||
or value["modality"] not in PAYLOAD_LIMITS
|
||||
or result["ingress_sequence"] < 1
|
||||
or result["session_generation"] < 1
|
||||
):
|
||||
raise StreamWireError("invalid raw modality or generation")
|
||||
return result
|
||||
|
||||
|
||||
def event_packets(
|
||||
start: StreamStart,
|
||||
event: LiveIngressEvent,
|
||||
) -> Iterator[tuple[bytes, memoryview]]:
|
||||
"""Iterate one already committed raw event; never assemble a route/archive.
|
||||
|
||||
Source adapter owns the original event bytes. Returned views borrow them;
|
||||
sender must finish each fragment before advancing this iterator.
|
||||
"""
|
||||
metadata = event_metadata(event)
|
||||
total = len(event.payload)
|
||||
if not 0 < total <= PAYLOAD_LIMITS[event.modality]:
|
||||
raise StreamWireError("raw event exceeds modality bound")
|
||||
whole_hash = sha256(event.payload).hexdigest()
|
||||
view = memoryview(event.payload)
|
||||
for offset in range(0, total, MAX_FRAGMENT):
|
||||
chunk = view[offset : offset + MAX_FRAGMENT]
|
||||
yield (
|
||||
packet(
|
||||
FRAGMENT,
|
||||
{
|
||||
"binding": binding(start),
|
||||
"event": metadata,
|
||||
"total_bytes": total,
|
||||
"offset": offset,
|
||||
"fragment_bytes": len(chunk),
|
||||
"payload_sha256": whole_hash,
|
||||
"fragment_sha256": sha256(chunk).hexdigest(),
|
||||
},
|
||||
),
|
||||
chunk,
|
||||
)
|
||||
|
||||
|
||||
def terminal_packet(start: StreamStart, *, cancel: bool = False) -> bytes:
|
||||
return packet(CANCEL if cancel else END, {"binding": binding(start)})
|
||||
|
||||
|
||||
def gap_packet(start: StreamStart, *, modality: str, reason: str, count: int) -> bytes:
|
||||
if modality not in PAYLOAD_LIMITS or reason not in ("unavailable", "source-gap", "overload"):
|
||||
raise StreamWireError("invalid gap notice")
|
||||
return packet(
|
||||
GAP,
|
||||
{
|
||||
"binding": binding(start),
|
||||
"modality": modality,
|
||||
"reason": reason,
|
||||
"count": decimal(count),
|
||||
},
|
||||
)
|
||||
Reference in New Issue
Block a user