feat(perception): add scoped stream control and bounded clock observations
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,225 @@
|
||||
"""Bounded, conditional mapping of two process-scoped monotonic clocks.
|
||||
|
||||
Four timestamps bound remote-minus-local offset WITHOUT symmetric-path claims.
|
||||
Intersect recent intervals, widened for an explicit relative-rate/error budget.
|
||||
Neither OS clock is adjusted. The rate/error envelope is an assumption supplied
|
||||
by the controller, not a hardware accuracy certificate inferred from low RTT.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .streaming_wire import identifier
|
||||
|
||||
|
||||
class ClockMappingError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _stamp(value: int) -> None:
|
||||
if type(value) is not int or not 0 <= value < 2**63:
|
||||
raise ClockMappingError("invalid monotonic timestamp")
|
||||
|
||||
|
||||
def _drift(duration_ns: int, ppm: int) -> int:
|
||||
return (duration_ns * ppm + 999_999) // 1_000_000
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ClockProbe:
|
||||
local_clock_id: str
|
||||
remote_clock_id: str
|
||||
nonce: str
|
||||
local_send_ns: int
|
||||
remote_receive_ns: int
|
||||
remote_send_ns: int
|
||||
local_receive_ns: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for value in (self.local_clock_id, self.remote_clock_id, self.nonce):
|
||||
identifier(value)
|
||||
for stamp in (
|
||||
self.local_send_ns,
|
||||
self.remote_receive_ns,
|
||||
self.remote_send_ns,
|
||||
self.local_receive_ns,
|
||||
):
|
||||
_stamp(stamp)
|
||||
if (
|
||||
not 0 <= self.local_receive_ns - self.local_send_ns <= 2_000_000_000
|
||||
or self.remote_send_ns < self.remote_receive_ns
|
||||
):
|
||||
raise ClockMappingError("clock probe ordering/deadline violated")
|
||||
|
||||
def bounds_at(
|
||||
self, local_ns: int, *, rate_ppm: int, timestamp_error_ns: int
|
||||
) -> tuple[int, int]:
|
||||
_stamp(local_ns)
|
||||
if local_ns < self.local_receive_ns:
|
||||
raise ClockMappingError("clock observation is from the future")
|
||||
# Any outbound/inbound asymmetry is inside these bounds. Remote receive
|
||||
# and send occur between local send and receive; widen across the whole
|
||||
# interval, not just elapsed time after receipt.
|
||||
widen = _drift(local_ns - self.local_send_ns, rate_ppm) + 2 * timestamp_error_ns
|
||||
return (
|
||||
self.remote_send_ns - self.local_receive_ns - widen,
|
||||
self.remote_receive_ns - self.local_send_ns + widen,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ClockBounds:
|
||||
local_clock_id: str
|
||||
remote_clock_id: str
|
||||
measured_at_ns: int
|
||||
expires_at_ns: int
|
||||
offset_lower_ns: int
|
||||
offset_upper_ns: int
|
||||
rate_ppm: int
|
||||
timestamp_error_ns: int
|
||||
samples: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for value in (self.local_clock_id, self.remote_clock_id):
|
||||
identifier(value)
|
||||
_stamp(self.measured_at_ns)
|
||||
_stamp(self.expires_at_ns)
|
||||
if (
|
||||
self.measured_at_ns >= self.expires_at_ns
|
||||
or type(self.offset_lower_ns) is not int
|
||||
or type(self.offset_upper_ns) is not int
|
||||
or self.offset_lower_ns > self.offset_upper_ns
|
||||
or type(self.rate_ppm) is not int
|
||||
or not 1 <= self.rate_ppm <= 1000
|
||||
or type(self.timestamp_error_ns) is not int
|
||||
or not 1 <= self.timestamp_error_ns <= 1_000_000
|
||||
or type(self.samples) is not int
|
||||
or not 1 <= self.samples <= 16
|
||||
):
|
||||
raise ClockMappingError("invalid clock bounds")
|
||||
|
||||
def offset_at(self, local_ns: int) -> tuple[int, int]:
|
||||
_stamp(local_ns)
|
||||
if not self.measured_at_ns <= local_ns < self.expires_at_ns:
|
||||
raise ClockMappingError("clock mapping expired or local clock regressed")
|
||||
extra = _drift(local_ns - self.measured_at_ns, self.rate_ppm)
|
||||
return self.offset_lower_ns - extra, self.offset_upper_ns + extra
|
||||
|
||||
def uncertainty_ns(self, local_ns: int) -> int:
|
||||
lower, upper = self.offset_at(local_ns)
|
||||
return (upper - lower + 1) // 2
|
||||
|
||||
def require(self, local_ns: int, maximum_uncertainty_ns: int = 5_000_000) -> None:
|
||||
if type(maximum_uncertainty_ns) is not int or maximum_uncertainty_ns <= 0:
|
||||
raise ClockMappingError("invalid uncertainty gate")
|
||||
if self.uncertainty_ns(local_ns) > maximum_uncertainty_ns:
|
||||
raise ClockMappingError("clock uncertainty exceeds admission budget")
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": "missioncore.monotonic-clock-bounds/v1",
|
||||
"local_clock_id": self.local_clock_id,
|
||||
"remote_clock_id": self.remote_clock_id,
|
||||
"measured_at_ns": str(self.measured_at_ns),
|
||||
"expires_at_ns": str(self.expires_at_ns),
|
||||
"offset_lower_ns": str(self.offset_lower_ns),
|
||||
"offset_upper_ns": str(self.offset_upper_ns),
|
||||
"relative_rate_budget_ppm": self.rate_ppm,
|
||||
"timestamp_error_budget_ns": str(self.timestamp_error_ns),
|
||||
"samples": self.samples,
|
||||
"conditional_rate_error_envelope": True,
|
||||
"symmetric_network_assumed": False,
|
||||
}
|
||||
|
||||
|
||||
class ClockWindow:
|
||||
def __init__(
|
||||
self,
|
||||
local_clock_id: str,
|
||||
remote_clock_id: str,
|
||||
*,
|
||||
rate_ppm: int,
|
||||
timestamp_error_ns: int,
|
||||
maximum_age_ns: int = 2_000_000_000,
|
||||
) -> None:
|
||||
for value in (local_clock_id, remote_clock_id):
|
||||
identifier(value)
|
||||
if (
|
||||
type(rate_ppm) is not int
|
||||
or not 1 <= rate_ppm <= 1000
|
||||
or type(timestamp_error_ns) is not int
|
||||
or not 1 <= timestamp_error_ns <= 1_000_000
|
||||
or type(maximum_age_ns) is not int
|
||||
or not 0 < maximum_age_ns <= 5_000_000_000
|
||||
):
|
||||
raise ClockMappingError("explicit bounded clock envelope is required")
|
||||
self.local_clock_id, self.remote_clock_id = local_clock_id, remote_clock_id
|
||||
self.rate_ppm, self.timestamp_error_ns, self.maximum_age_ns = (
|
||||
rate_ppm,
|
||||
timestamp_error_ns,
|
||||
maximum_age_ns,
|
||||
)
|
||||
self.samples: deque[ClockProbe] = deque(maxlen=16)
|
||||
self.failed = False
|
||||
self.last_receive_ns = -1
|
||||
|
||||
def add(self, sample: ClockProbe) -> ClockBounds:
|
||||
if self.failed:
|
||||
raise ClockMappingError("clock session quarantined; explicit new session required")
|
||||
if (
|
||||
(sample.local_clock_id, sample.remote_clock_id)
|
||||
!= (self.local_clock_id, self.remote_clock_id)
|
||||
or sample.local_send_ns <= self.last_receive_ns
|
||||
or any(old.nonce == sample.nonce for old in self.samples)
|
||||
):
|
||||
raise ClockMappingError("foreign, overlapping or replayed clock probe")
|
||||
while (
|
||||
self.samples
|
||||
and sample.local_receive_ns - self.samples[0].local_receive_ns >= self.maximum_age_ns
|
||||
):
|
||||
self.samples.popleft()
|
||||
# A new sample must intersect the still-valid old evidence. Do not
|
||||
# silently forget a contradictory sample to manufacture a narrower fit.
|
||||
bounds = [
|
||||
old.bounds_at(
|
||||
sample.local_receive_ns,
|
||||
rate_ppm=self.rate_ppm,
|
||||
timestamp_error_ns=self.timestamp_error_ns,
|
||||
)
|
||||
for old in (*self.samples, sample)
|
||||
]
|
||||
lower, upper = max(x[0] for x in bounds), min(x[1] for x in bounds)
|
||||
if lower > upper:
|
||||
self.failed = True
|
||||
raise ClockMappingError("clock jump or violated rate/error envelope")
|
||||
self.samples.append(sample)
|
||||
self.last_receive_ns = sample.local_receive_ns
|
||||
return self.current(sample.local_receive_ns)
|
||||
|
||||
def current(self, local_ns: int) -> ClockBounds:
|
||||
_stamp(local_ns)
|
||||
if self.failed or not self.samples or local_ns < self.last_receive_ns:
|
||||
raise ClockMappingError("no valid monotonic clock mapping")
|
||||
recent = [x for x in self.samples if local_ns - x.local_receive_ns < self.maximum_age_ns]
|
||||
if not recent:
|
||||
raise ClockMappingError("clock mapping expired")
|
||||
bounds = [
|
||||
x.bounds_at(
|
||||
local_ns, rate_ppm=self.rate_ppm, timestamp_error_ns=self.timestamp_error_ns
|
||||
)
|
||||
for x in recent
|
||||
]
|
||||
lower, upper = max(x[0] for x in bounds), min(x[1] for x in bounds)
|
||||
return ClockBounds(
|
||||
self.local_clock_id,
|
||||
self.remote_clock_id,
|
||||
local_ns,
|
||||
min(x.local_receive_ns for x in recent) + self.maximum_age_ns,
|
||||
lower,
|
||||
upper,
|
||||
self.rate_ppm,
|
||||
self.timestamp_error_ns,
|
||||
len(recent),
|
||||
)
|
||||
@@ -0,0 +1,269 @@
|
||||
"""Capability-scoped clock observation and pending-grant delivery over TLS.
|
||||
|
||||
Poll cannot acquire/renew a GPU lease, pick an epoch, reset models or cancel a
|
||||
run. Only the trusted local controller supplies a pending grant. A request nonce
|
||||
binds each clock response; authenticated polling is rate/buffer bounded. The
|
||||
per-activation bootstrap ticket is provisioned separately, never in CLI/logs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import secrets
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field, replace
|
||||
from hashlib import sha256
|
||||
from typing import Any
|
||||
|
||||
import grpc # type: ignore[import-untyped]
|
||||
|
||||
from . import streaming_wire as wire
|
||||
from .realtime_contract import StreamStart
|
||||
from .streaming_clock import ClockProbe
|
||||
from .streaming_grpc import OPTIONS, StreamAccess
|
||||
|
||||
SERVICE = "missioncore.perception.v1.StreamControl"
|
||||
METHOD = f"/{SERVICE}/Poll"
|
||||
MAX_CONTROL = 8192
|
||||
|
||||
|
||||
def _document(raw: bytes) -> dict[str, Any]:
|
||||
if type(raw) is not bytes or not 0 < len(raw) <= MAX_CONTROL:
|
||||
raise wire.StreamWireError("control message exceeds bound")
|
||||
return wire.parse_header(bytearray(raw))
|
||||
|
||||
|
||||
def _token(value: object) -> None:
|
||||
if (
|
||||
not isinstance(value, str)
|
||||
or len(value) != 64
|
||||
or any(c not in "0123456789abcdef" for c in value)
|
||||
):
|
||||
raise ValueError("invalid capability")
|
||||
|
||||
|
||||
def _offer(access: StreamAccess, activation: StreamStart) -> None:
|
||||
if replace(access.epoch, epoch_id=activation.epoch_id) != activation:
|
||||
raise ValueError("pending grant belongs to another activation")
|
||||
wire.identifier(access.session_id)
|
||||
wire.decimal(access.session_generation)
|
||||
if access.session_generation < 1:
|
||||
raise ValueError("invalid capture generation")
|
||||
_token(access.token)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ControlTicket:
|
||||
activation: StreamStart
|
||||
token: str = field(repr=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_token(self.token)
|
||||
|
||||
def metadata(self) -> tuple[tuple[str, str], ...]:
|
||||
return (
|
||||
("authorization", "Bearer " + self.token),
|
||||
("x-stream-binding", wire.binding(self.activation)),
|
||||
)
|
||||
|
||||
|
||||
class StreamControlEndpoint:
|
||||
def __init__(
|
||||
self,
|
||||
ticket: ControlTicket,
|
||||
pending: Callable[[], StreamAccess | None],
|
||||
*,
|
||||
clock_id: str,
|
||||
clock_ns: Callable[[], int] = time.monotonic_ns,
|
||||
) -> None:
|
||||
wire.identifier(clock_id)
|
||||
self.activation, self.pending, self.clock_id, self.clock_ns = (
|
||||
ticket.activation,
|
||||
pending,
|
||||
clock_id,
|
||||
clock_ns,
|
||||
)
|
||||
self._token_hash = sha256(ticket.token.encode()).digest()
|
||||
self._lock = threading.Lock()
|
||||
self._last_poll_ns = -1
|
||||
self.accepted = self.rejected = 0
|
||||
|
||||
def handler(self) -> Any:
|
||||
return grpc.method_handlers_generic_handler(
|
||||
SERVICE, {"Poll": grpc.unary_unary_rpc_method_handler(self.poll)}
|
||||
)
|
||||
|
||||
def _authenticate(self, context: Any) -> None:
|
||||
if b"ssl" not in context.auth_context().get("transport_security_type", ()):
|
||||
raise wire.StreamWireError("TLS control capability required")
|
||||
metadata = list(context.invocation_metadata())
|
||||
relevant = [(k, v) for k, v in metadata if k in ("authorization", "x-stream-binding")]
|
||||
values = dict(relevant)
|
||||
token = values.get("authorization")
|
||||
if (
|
||||
len(relevant) != 2
|
||||
or len(values) != 2
|
||||
or not isinstance(token, str)
|
||||
or len(token) != 71
|
||||
or not token.startswith("Bearer ")
|
||||
or values.get("x-stream-binding") != wire.binding(self.activation)
|
||||
or not hmac.compare_digest(sha256(token[7:].encode()).digest(), self._token_hash)
|
||||
):
|
||||
raise wire.StreamWireError("control capability rejected")
|
||||
|
||||
async def poll(self, raw: bytes, context: Any) -> bytes:
|
||||
received = self.clock_ns()
|
||||
try:
|
||||
self._authenticate(context)
|
||||
request = _document(raw)
|
||||
if set(request) != {"nonce", "local_clock_id", "local_send_ns"}:
|
||||
raise wire.StreamWireError("unsupported control request")
|
||||
wire.identifier(request["nonce"])
|
||||
wire.identifier(request["local_clock_id"])
|
||||
if wire.uint64(request["local_send_ns"]) >= 2**63:
|
||||
raise ValueError("invalid monotonic timestamp")
|
||||
except (ValueError, TypeError):
|
||||
self.rejected += 1
|
||||
await context.abort(
|
||||
grpc.StatusCode.UNAUTHENTICATED, "control capability/request rejected"
|
||||
)
|
||||
return b""
|
||||
with self._lock:
|
||||
limited = self._last_poll_ns >= 0 and received - self._last_poll_ns < 25_000_000
|
||||
if not limited:
|
||||
self._last_poll_ns = received
|
||||
if limited:
|
||||
self.rejected += 1
|
||||
await context.abort(
|
||||
grpc.StatusCode.RESOURCE_EXHAUSTED, "bounded control poll rate exceeded"
|
||||
)
|
||||
return b""
|
||||
try:
|
||||
offer = self.pending() # Trusted constant-time snapshot; no model/control work.
|
||||
if offer is not None:
|
||||
_offer(offer, self.activation)
|
||||
response = {
|
||||
"schema_version": "missioncore.stream-control-poll/v1",
|
||||
"activation_binding": wire.binding(self.activation),
|
||||
**request,
|
||||
"remote_clock_id": self.clock_id,
|
||||
"remote_receive_ns": str(received),
|
||||
"remote_send_ns": str(self.clock_ns()),
|
||||
"grant": None
|
||||
if offer is None
|
||||
else {
|
||||
"epoch": offer.epoch.to_dict(),
|
||||
"session_id": offer.session_id,
|
||||
"session_generation": str(offer.session_generation),
|
||||
"token": offer.token,
|
||||
},
|
||||
}
|
||||
encoded = wire.canonical(response)
|
||||
if len(encoded) > MAX_CONTROL:
|
||||
raise ValueError("trusted control snapshot exceeds bound")
|
||||
except (ValueError, RuntimeError):
|
||||
await context.abort(
|
||||
grpc.StatusCode.UNAVAILABLE, "local controller has no current offer"
|
||||
)
|
||||
return b""
|
||||
self.accepted += 1
|
||||
return encoded
|
||||
|
||||
|
||||
class StreamControlClient:
|
||||
def __init__(
|
||||
self,
|
||||
target: str,
|
||||
roots: bytes,
|
||||
ticket: ControlTicket,
|
||||
*,
|
||||
clock_id: str,
|
||||
clock_ns: Callable[[], int] = time.monotonic_ns,
|
||||
) -> None:
|
||||
if not roots:
|
||||
raise ValueError("trusted TLS roots required")
|
||||
wire.identifier(clock_id)
|
||||
self.ticket, self.clock_id, self.clock_ns = ticket, clock_id, clock_ns
|
||||
self.channel = grpc.aio.secure_channel(
|
||||
target,
|
||||
grpc.ssl_channel_credentials(root_certificates=roots),
|
||||
options=OPTIONS
|
||||
+ (
|
||||
("grpc.max_send_message_length", MAX_CONTROL),
|
||||
("grpc.max_receive_message_length", MAX_CONTROL),
|
||||
),
|
||||
)
|
||||
self.call = self.channel.unary_unary(METHOD)
|
||||
self.polling = False
|
||||
|
||||
async def poll(self) -> tuple[ClockProbe, StreamAccess | None]:
|
||||
if self.polling:
|
||||
raise ValueError("one outstanding clock/control probe per client")
|
||||
self.polling = True
|
||||
nonce = secrets.token_hex(16)
|
||||
sent = self.clock_ns()
|
||||
try:
|
||||
raw = await self.call(
|
||||
wire.canonical(
|
||||
{"nonce": nonce, "local_clock_id": self.clock_id, "local_send_ns": str(sent)}
|
||||
),
|
||||
metadata=self.ticket.metadata(),
|
||||
timeout=0.5,
|
||||
wait_for_ready=False,
|
||||
)
|
||||
received = self.clock_ns()
|
||||
response = _document(raw)
|
||||
if (
|
||||
set(response)
|
||||
!= {
|
||||
"schema_version",
|
||||
"activation_binding",
|
||||
"nonce",
|
||||
"local_clock_id",
|
||||
"local_send_ns",
|
||||
"remote_clock_id",
|
||||
"remote_receive_ns",
|
||||
"remote_send_ns",
|
||||
"grant",
|
||||
}
|
||||
or response["schema_version"] != "missioncore.stream-control-poll/v1"
|
||||
or response["activation_binding"] != wire.binding(self.ticket.activation)
|
||||
or response["nonce"] != nonce
|
||||
or response["local_clock_id"] != self.clock_id
|
||||
or response["local_send_ns"] != str(sent)
|
||||
):
|
||||
raise wire.StreamWireError("clock/control response binding mismatch")
|
||||
probe = ClockProbe(
|
||||
self.clock_id,
|
||||
response["remote_clock_id"],
|
||||
nonce,
|
||||
sent,
|
||||
wire.uint64(response["remote_receive_ns"]),
|
||||
wire.uint64(response["remote_send_ns"]),
|
||||
received,
|
||||
)
|
||||
grant = response["grant"]
|
||||
access = None
|
||||
if grant is not None:
|
||||
if not isinstance(grant, dict) or set(grant) != {
|
||||
"epoch",
|
||||
"session_id",
|
||||
"session_generation",
|
||||
"token",
|
||||
}:
|
||||
raise wire.StreamWireError("invalid control grant shape")
|
||||
access = StreamAccess(
|
||||
StreamStart.from_dict(grant["epoch"]),
|
||||
grant["session_id"],
|
||||
wire.uint64(grant["session_generation"]),
|
||||
grant["token"],
|
||||
)
|
||||
_offer(access, self.ticket.activation)
|
||||
return probe, access
|
||||
finally:
|
||||
self.polling = False
|
||||
|
||||
async def close(self) -> None:
|
||||
await self.channel.close()
|
||||
@@ -149,7 +149,14 @@ class GrpcStreamEndpoint:
|
||||
if self.active is not None and self.active[0] == epoch:
|
||||
self.active[1].finish()
|
||||
|
||||
async def serve(self, address: str, *, certificate: bytes, private_key: bytes) -> Any:
|
||||
async def serve(
|
||||
self,
|
||||
address: str,
|
||||
*,
|
||||
certificate: bytes,
|
||||
private_key: bytes,
|
||||
control_handlers: tuple[Any, ...] = (),
|
||||
) -> Any:
|
||||
if not certificate or not private_key:
|
||||
raise ValueError("TLS identity is required")
|
||||
server = grpc.aio.server(
|
||||
@@ -158,7 +165,11 @@ class GrpcStreamEndpoint:
|
||||
("grpc.max_receive_message_length", INPUT_CHUNK),
|
||||
("grpc.max_send_message_length", MAX_REPLY),
|
||||
),
|
||||
maximum_concurrent_rpcs=2,
|
||||
# aio reserves its next accept before awaiting a request. Leave
|
||||
# one bounded accept slot beside Exchange + a completing Poll;
|
||||
# otherwise the next Poll can inherit a stale "limit exceeded".
|
||||
# Data ownership still admits exactly one Exchange, never 2 GPUs.
|
||||
maximum_concurrent_rpcs=3 if control_handlers else 2,
|
||||
compression=grpc.Compression.NoCompression,
|
||||
)
|
||||
server.add_generic_rpc_handlers(
|
||||
@@ -166,6 +177,7 @@ class GrpcStreamEndpoint:
|
||||
grpc.method_handlers_generic_handler(
|
||||
SERVICE, {"Exchange": grpc.stream_stream_rpc_method_handler(self.exchange)}
|
||||
),
|
||||
*control_handlers,
|
||||
)
|
||||
)
|
||||
port = server.add_secure_port(
|
||||
|
||||
Reference in New Issue
Block a user