feat(perception): gate cross-host graph with acknowledged source clocks

This commit is contained in:
DCCONSTRUCTIONS
2026-09-02 20:37:21 +03:00
parent d80df61a7e
commit 35b6cd9e9e
17 changed files with 1026 additions and 62 deletions
@@ -126,7 +126,11 @@ class BinaryGraphInput:
length = len(e.value[0]) length = len(e.value[0])
rolling_times[offset : offset + length] = e.time_ns rolling_times[offset : offset + length] = e.time_ns
offset += length offset += length
due = self.wall_zero + stamp - self.source_zero due = (
self.source.due(stamp, time.monotonic_ns())[0]
if hasattr(self.source, "observed")
else self.wall_zero + stamp - self.source_zero
)
bundle = { bundle = {
"input_start": self.epoch, "input_start": self.epoch,
"sequence": event.source_sequence, "sequence": event.source_sequence,
@@ -158,6 +162,8 @@ class BinaryGraphInput:
"enqueued_ns": time.monotonic_ns(), "enqueued_ns": time.monotonic_ns(),
"payload_bytes": size, "payload_bytes": size,
} }
if hasattr(self.source, "observed"):
bundle["clock_observer"] = self.source.observed
self.bgr_hashes.append(hashlib.sha256(image).hexdigest()) self.bgr_hashes.append(hashlib.sha256(image).hexdigest())
bundle["enqueued_ns"] = time.monotonic_ns() bundle["enqueued_ns"] = time.monotonic_ns()
transferred = self.runtime.admit_reserved(self.epoch, bundle, reservation) transferred = self.runtime.admit_reserved(self.epoch, bundle, reservation)
@@ -89,15 +89,25 @@ def build_freshness(scene, bundle, ddr_layer, epoch_id):
return SceneFreshness(epoch_id, CLOCK_DOMAIN, sequence, stamp, tuple(layers.values())) return SceneFreshness(epoch_id, CLOCK_DOMAIN, sequence, stamp, tuple(layers.values()))
def source_observation(bundle, now_ns):
if "clock_observer" in bundle:
stamp, uncertainty_ns = bundle["clock_observer"](now_ns)
return stamp, uncertainty_ns / 1e6
# Legacy same-kernel path, or source/consumer on the originating Mac clock.
return bundle["time_ns"] + now_ns - bundle["due_ns"], 0
def assess(freshness, *, bundle, now_ns): def assess(freshness, *, bundle, now_ns):
# Both sides of this pilot use the SAME monotonic clock. Map it back to observed, uncertainty = source_observation(bundle, now_ns)
# original source arrival time; zero mapping uncertainty is local-only and return assess_observation(freshness, observed, uncertainty)
# is not a claim of hardware camera/LiDAR synchronization or network quality.
def assess_observation(freshness, observed, uncertainty):
return freshness.assess( return freshness.assess(
epoch_id=freshness.epoch_id, epoch_id=freshness.epoch_id,
clock_domain_id=CLOCK_DOMAIN, clock_domain_id=CLOCK_DOMAIN,
observed_source_time_ns=bundle["time_ns"] + now_ns - bundle["due_ns"], observed_source_time_ns=observed,
clock_uncertainty_ms=0, clock_uncertainty_ms=uncertainty,
maximum_clock_uncertainty_ms=5, maximum_clock_uncertainty_ms=5,
maximum_layer_age_ms=250, maximum_layer_age_ms=250,
) )
@@ -141,19 +151,23 @@ def prepare_publication(scene, bundle, ddr_layer, *, epoch_id, now_ns, mode="who
if mode not in ("whole-scene", "per-cell"): if mode not in ("whole-scene", "per-cell"):
raise ValueError("unknown costmap freshness mode") raise ValueError("unknown costmap freshness mode")
scene["costmap_freshness_mode"] = mode scene["costmap_freshness_mode"] = mode
observed, uncertainty = source_observation(bundle, now_ns)
if mode == "per-cell": if mode == "per-cell":
apply_cell_expiry( apply_cell_expiry(
scene, scene,
source_time_ns=bundle["time_ns"], source_time_ns=bundle["time_ns"],
observed_source_time_ns=bundle["time_ns"] + now_ns - bundle["due_ns"], observed_source_time_ns=observed,
uncertainty_ms=uncertainty,
) )
freshness = build_freshness(scene, bundle, ddr_layer, epoch_id) freshness = build_freshness(scene, bundle, ddr_layer, epoch_id)
checked = assess(freshness, bundle=bundle, now_ns=now_ns) # One immutable clock observation per boundary, including cell and policy
# reassessment. A concurrent refresh must not reinterpret this same instant.
checked = assess_observation(freshness, observed, uncertainty)
if not checked.fresh_complete: if not checked.fresh_complete:
suppress_policy(scene) suppress_policy(scene)
# The envelope hashes the actual guarded policy, not the discarded one. # The envelope hashes the actual guarded policy, not the discarded one.
freshness = build_freshness(scene, bundle, ddr_layer, epoch_id) freshness = build_freshness(scene, bundle, ddr_layer, epoch_id)
checked = assess(freshness, bundle=bundle, now_ns=now_ns) checked = assess_observation(freshness, observed, uncertainty)
scene["freshness"] = freshness.to_dict() scene["freshness"] = freshness.to_dict()
scene["freshness_at_publication"] = checked.to_dict() scene["freshness_at_publication"] = checked.to_dict()
scene["stale_at_publication"] = any(item.state == "stale" for item in checked.layers) scene["stale_at_publication"] = any(item.state == "stale" for item in checked.layers)
@@ -202,9 +216,14 @@ def assess_receipt(scene, freshness, *, bundle, now_ns):
their original support time. Layer identity/source timestamps do not change; their original support time. Layer identity/source timestamps do not change;
updated costmap/policy hashes belong to the derived view, not the wire bytes. updated costmap/policy hashes belong to the derived view, not the wire bytes.
""" """
observed = bundle["time_ns"] + now_ns - bundle["due_ns"] observed, uncertainty = source_observation(bundle, now_ns)
previous = scene.get("freshness_at_receipt") or scene["freshness_at_publication"] previous = scene.get("freshness_at_receipt") or scene["freshness_at_publication"]
if observed < _read_wire_integer(previous["checked_at_source_time_ns"]): # Independent clocks can have overlapping intervals even though receipt is
# later. Reject a provably reversed interval, not a shifted midpoint alone.
if observed + int(uncertainty * 1e6) < (
_read_wire_integer(previous["checked_at_source_time_ns"])
- int(previous["clock_uncertainty_ms"] * 1e6)
):
raise ValueError("consumer clock moved backwards") raise ValueError("consumer clock moved backwards")
view = dict(scene) view = dict(scene)
if scene.get("costmap_freshness_mode") == "per-cell": if scene.get("costmap_freshness_mode") == "per-cell":
@@ -212,6 +231,7 @@ def assess_receipt(scene, freshness, *, bundle, now_ns):
view, view,
source_time_ns=freshness.source_time_ns, source_time_ns=freshness.source_time_ns,
observed_source_time_ns=observed, observed_source_time_ns=observed,
uncertainty_ms=uncertainty,
) )
layers = {item.layer: item for item in freshness.layers[:4]} layers = {item.layer: item for item in freshness.layers[:4]}
for name in ("costmap", "policy"): for name in ("costmap", "policy"):
@@ -230,7 +250,7 @@ def assess_receipt(scene, freshness, *, bundle, now_ns):
oldest_required_input_time_ns=oldest, oldest_required_input_time_ns=oldest,
) )
freshness = replace(freshness, layers=tuple(layers.values())) freshness = replace(freshness, layers=tuple(layers.values()))
checked = assess(freshness, bundle=bundle, now_ns=now_ns) checked = assess_observation(freshness, observed, uncertainty)
view = receipt_view(view, checked) view = receipt_view(view, checked)
# Global suppression also changes the policy payload. A derived view must # Global suppression also changes the policy payload. A derived view must
# remain internally verifiable, including on a second consumer boundary. # remain internally verifiable, including on a second consumer boundary.
@@ -238,7 +258,7 @@ def assess_receipt(scene, freshness, *, bundle, now_ns):
if policy.payload_sha256 is not None: if policy.payload_sha256 is not None:
policy = replace(policy, payload_sha256=payload_digest(view, "policy")) policy = replace(policy, payload_sha256=payload_digest(view, "policy"))
freshness = replace(freshness, layers=(*freshness.layers[:-1], policy)) freshness = replace(freshness, layers=(*freshness.layers[:-1], policy))
checked = assess(freshness, bundle=bundle, now_ns=now_ns) checked = assess_observation(freshness, observed, uncertainty)
view["freshness"] = freshness.to_dict() view["freshness"] = freshness.to_dict()
view["freshness_at_receipt"] = checked.to_dict() view["freshness_at_receipt"] = checked.to_dict()
return view, checked return view, checked
@@ -5,6 +5,7 @@ accepted from the network. Grant rotation stays in this trusted local adapter.
""" """
import asyncio import asyncio
import secrets
import threading import threading
import time import time
import traceback import traceback
@@ -15,7 +16,9 @@ from pilot_binary_bridge import BinaryGraphInput
from pilot_network_control import kernel_clock, read_control, write_control from pilot_network_control import kernel_clock, read_control, write_control
from k1link.perception.streaming_continuity import StreamSuspended from k1link.perception.streaming_continuity import StreamSuspended
from k1link.perception.streaming_control_grpc import ControlTicket, StreamControlEndpoint
from k1link.perception.streaming_grpc import GrpcStreamEndpoint from k1link.perception.streaming_grpc import GrpcStreamEndpoint
from k1link.perception.streaming_source_clock import SourceClockMonitor
class NetworkGraphBridge(BinaryGraphInput): class NetworkGraphBridge(BinaryGraphInput):
@@ -33,16 +36,40 @@ class NetworkGraphBridge(BinaryGraphInput):
address, address,
reset_temporal, reset_temporal,
start_delay=2, start_delay=2,
cross_host=False,
): ):
self.clock = kernel_clock() self.cross_host = cross_host
source = SimpleNamespace( self.clock_id = "worker-clock-" + secrets.token_hex(12)
source_zero=source_zero, wall_zero=time.monotonic_ns() + int(start_delay * 1e9) self.clock = (
{"scope": "cross-host-conditional", "clock_id": self.clock_id}
if cross_host
else kernel_clock()
)
source = (
SourceClockMonitor(self.clock_id, source_zero)
if cross_host
else SimpleNamespace(
source_zero=source_zero, wall_zero=time.monotonic_ns() + int(start_delay * 1e9)
)
) )
super().__init__(runtime, decoder, source, report, reset_temporal=reset_temporal) super().__init__(runtime, decoder, source, report, reset_temporal=reset_temporal)
self.control = Path(control) self.control = Path(control)
self.source_status = Path(source_status) self.source_status = Path(source_status)
self.certificate, self.private_key, self.address = certificate, private_key, address self.certificate, self.private_key, self.address = certificate, private_key, address
self.endpoint = GrpcStreamEndpoint(runtime, self.consume, self.notice) self.endpoint = GrpcStreamEndpoint(runtime, self.consume, self.notice)
self.access = None
self.ticket = ControlTicket(runtime.start, secrets.token_hex(32)) if cross_host else None
self.control_endpoint = (
StreamControlEndpoint(
self.ticket,
self._pending,
clock_id=self.clock_id,
observe=source.observe,
)
if cross_host
else None
)
self.clock_states = []
self.stopping, self.ready = threading.Event(), threading.Event() self.stopping, self.ready = threading.Event(), threading.Event()
self.failure = None self.failure = None
self.published = [] self.published = []
@@ -64,6 +91,9 @@ class NetworkGraphBridge(BinaryGraphInput):
def _grant(self): def _grant(self):
access = self.endpoint.issue(self.epoch, "recorded-acquisition", 1) access = self.endpoint.issue(self.epoch, "recorded-acquisition", 1)
if self.cross_host:
self.access = access
return
write_control( write_control(
self.control / "grant.json", self.control / "grant.json",
{ {
@@ -76,6 +106,13 @@ class NetworkGraphBridge(BinaryGraphInput):
}, },
) )
def _pending(self):
try:
self.source.observed(time.monotonic_ns())
except StreamSuspended:
return None
return self.access if self.endpoint.grant is not None else None
def _run(self): def _run(self):
try: try:
asyncio.run(self._serve()) asyncio.run(self._serve())
@@ -90,21 +127,55 @@ class NetworkGraphBridge(BinaryGraphInput):
self.address, self.address,
certificate=Path(self.certificate).read_bytes(), certificate=Path(self.certificate).read_bytes(),
private_key=Path(self.private_key).read_bytes(), private_key=Path(self.private_key).read_bytes(),
control_handlers=(self.control_endpoint.handler(),) if self.cross_host else (),
) )
try: try:
self._grant() if self.cross_host:
self.runtime.pause_input(self.epoch, "source-clock")
write_control(
self.control / "bootstrap.json",
{
"activation": self.runtime.start.to_dict(),
"token": self.ticket.token,
"remote_clock_id": self.clock_id,
"source_zero_ns": str(self.source_zero),
},
)
else:
self._grant()
self.ready.set() self.ready.set()
while not self.stopping.is_set() and not self.runtime.stop_event.is_set(): while not self.stopping.is_set() and not self.runtime.stop_event.is_set():
if self.cross_host:
try:
self.source.observed(time.monotonic_ns())
clock_ready = True
if not self.report["wall_zero_ns"]:
self.report["wall_zero_ns"] = self.source.due(
self.source_zero, time.monotonic_ns()
)[0]
except StreamSuspended:
clock_ready = False
self.runtime.pause_input(self.epoch, "source-clock")
if not self.clock_states or self.clock_states[-1]["ready"] != clock_ready:
self.clock_states.append(
{"at_ns": time.monotonic_ns(), "ready": clock_ready}
)
if len(self.clock_states) > 128:
raise ValueError("bounded clock transition count exceeded")
if self.endpoint.active is None: if self.endpoint.active is None:
ended = self.source_status / "source-end.json" ended = self.source_status / "source-end.json"
if ended.exists(): if self.cross_host and self.source.ended:
self.runtime.mailbox.finish()
elif not self.cross_host and ended.exists():
terminal = read_control(ended) terminal = read_control(ended)
if terminal.get("run_id") != self.runtime.start.run_id: if terminal.get("run_id") != self.runtime.start.run_id:
raise ValueError("external source completion binding mismatch") raise ValueError("external source completion binding mismatch")
if terminal.get("error"): if terminal.get("error"):
raise ValueError("external source failed; inspect its bounded report") raise ValueError("external source failed; inspect its bounded report")
self.runtime.mailbox.finish() self.runtime.mailbox.finish()
elif self.runtime.continuity.phase == "waiting": elif self.runtime.continuity.phase == "waiting" and (
not self.cross_host or clock_ready
):
try: try:
epoch = self.runtime.begin_input(self.runtime.start) epoch = self.runtime.begin_input(self.runtime.start)
except StreamSuspended: except StreamSuspended:
@@ -118,6 +189,8 @@ class NetworkGraphBridge(BinaryGraphInput):
await asyncio.sleep(0.01) await asyncio.sleep(0.01)
finally: finally:
await server.stop(0) await server.stop(0)
if self.cross_host:
(self.control / "bootstrap.json").unlink(missing_ok=True)
deadline = time.monotonic() + 2 deadline = time.monotonic() + 2
while self.endpoint.active is not None and time.monotonic() < deadline: while self.endpoint.active is not None and time.monotonic() < deadline:
await asyncio.sleep(0.01) await asyncio.sleep(0.01)
@@ -157,6 +230,7 @@ class NetworkGraphBridge(BinaryGraphInput):
accepted_camera_sequences=self.accepted, accepted_camera_sequences=self.accepted,
published=self.published, published=self.published,
adapter_failure=self.failure, adapter_failure=self.failure,
clock_states=self.clock_states,
) )
if not self.thread.is_alive() and self.endpoint.active is None: if not self.thread.is_alive() and self.endpoint.active is None:
self.window.close() self.window.close()
@@ -18,8 +18,9 @@ from pathlib import Path
import grpc import grpc
from pilot_binary_source import event_bytes, input_gaps, read_member from pilot_binary_source import event_bytes, input_gaps, read_member
from pilot_freshness import assess_receipt, validate_receipt from pilot_freshness import assess_receipt, validate_receipt
from pilot_network_control import read_grant, write_control from pilot_network_control import read_control, read_grant, write_control
from pilot_source import SensorArchive, camera_events, merged_events from pilot_source import SensorArchive, camera_events, merged_events
from pilot_source_control import SourceControl
from k1link.compute.live_perception import LiveIngressEvent from k1link.compute.live_perception import LiveIngressEvent
from k1link.perception.streaming_grpc import GrpcStreamClient from k1link.perception.streaming_grpc import GrpcStreamClient
@@ -32,6 +33,16 @@ def connection_delay(wall_zero_ns, now_ns):
return max(0, (wall_zero_ns - now_ns - 100_000_000) / 1e9) return max(0, (wall_zero_ns - now_ns - 100_000_000) / 1e9)
async def finish_source(client, reader, clock_control):
# EOF on Exchange can finish the graph and retire its control endpoint.
# Confirm source completion first while that data stream is still alive.
if clock_control:
await clock_control.finish()
if client is not None:
await client.end()
await asyncio.wait_for(reader, timeout=5)
async def run(args): async def run(args):
report = { report = {
"started_utc": datetime.now(UTC).isoformat(), "started_utc": datetime.now(UTC).isoformat(),
@@ -43,7 +54,9 @@ async def run(args):
"models": 0, "models": 0,
"source_clock_speed": 1.0, "source_clock_speed": 1.0,
"full_source_prepass": False, "full_source_prepass": False,
"clock_scope": "two-containers-same-unshifted-Linux-kernel", "clock_scope": "cross-host-conditional"
if args.cross_host
else "two-containers-same-unshifted-Linux-kernel",
"error": None, "error": None,
"actuation_allowed": False, "actuation_allowed": False,
"commands_enabled": False, "commands_enabled": False,
@@ -59,6 +72,7 @@ async def run(args):
gaps, next_gap, outage_until = input_gaps(args.input_gap), 0, 0 gaps, next_gap, outage_until = input_gaps(args.input_gap), 0, 0
previous_epoch = None previous_epoch = None
clock = None clock = None
clock_control = None
async def disconnect(): async def disconnect():
nonlocal client, reader nonlocal client, reader
@@ -71,11 +85,25 @@ async def run(args):
try: try:
deadline = time.monotonic() + 120 deadline = time.monotonic() + 120
while not grant_path.exists(): startup_path = control / "bootstrap.json" if args.cross_host else grant_path
while not startup_path.exists():
if time.monotonic() > deadline: if time.monotonic() > deadline:
raise TimeoutError("trusted graph grant was not issued after warmup") raise TimeoutError("trusted graph grant was not issued after warmup")
await asyncio.sleep(0.02) await asyncio.sleep(0.02)
clock, _ = read_grant(grant_path) if args.cross_host:
bootstrap = read_control(startup_path)
clock_control = SourceControl(
args.target, Path(args.certificate).read_bytes(), bootstrap
)
clock_control.start()
anchor = await clock_control.wait_anchor()
clock = {
"source_zero_ns": anchor.source_zero_ns,
"wall_zero_ns": anchor.local_zero_ns,
"clock": {"scope": "source-and-consumer-same-Mac-clock"},
}
else:
clock, _ = read_grant(grant_path)
first = next(camera_events(args.camera_index, 1)).time_ns first = next(camera_events(args.camera_index, 1)).time_ns
if clock["source_zero_ns"] != first - 500_000_000: if clock["source_zero_ns"] != first - 500_000_000:
raise ValueError("recording prefix does not match admitted source clock") raise ValueError("recording prefix does not match admitted source clock")
@@ -104,6 +132,20 @@ async def run(args):
"due_ns": clock["wall_zero_ns"] + source_stamp - clock["source_zero_ns"], "due_ns": clock["wall_zero_ns"] + source_stamp - clock["source_zero_ns"],
} }
freshness = validate_receipt(scene, bundle, epoch_id=epoch.epoch_id) freshness = validate_receipt(scene, bundle, epoch_id=epoch.epoch_id)
if clock_control and not clock_control.ready():
rejected = report.setdefault("clock_rejected_results", [])
if len(rejected) >= args.frames:
raise ValueError("bounded rejected receipt ledger exceeded")
rejected.append(
{
"sequence": sequence,
"epoch_id": epoch.epoch_id,
"payload_sha256": hashlib.sha256(payload).hexdigest(),
"reason": "source-clock-unavailable",
}
)
# No fresh scene/policy authority while clock admission is lost.
continue
checked_at = time.monotonic_ns() checked_at = time.monotonic_ns()
view, checked = assess_receipt( view, checked = assess_receipt(
scene, freshness, bundle=bundle, now_ns=checked_at scene, freshness, bundle=bundle, now_ns=checked_at
@@ -140,7 +182,18 @@ async def run(args):
async def connect(): async def connect():
nonlocal client, reader, access, previous_epoch nonlocal client, reader, access, previous_epoch
value, candidate = read_grant(grant_path) if clock_control:
if not clock_control.ready() or clock_control.access is None:
return False
candidate = clock_control.access
value = {
**clock,
"cutoff_ns": clock["source_zero_ns"]
+ time.monotonic_ns()
- clock["wall_zero_ns"],
}
else:
value, candidate = read_grant(grant_path)
if candidate.epoch.epoch_id == previous_epoch: if candidate.epoch.epoch_id == previous_epoch:
return False return False
if any( if any(
@@ -185,6 +238,8 @@ async def run(args):
if event.channel == "camera": if event.channel == "camera":
released.append(event.sequence) released.append(event.sequence)
try: try:
if clock_control and not clock_control.ready():
await disconnect()
if reader is not None and reader.done(): if reader is not None and reader.done():
reader.result() # Invalid payload is fatal; transport interruption is not. reader.result() # Invalid payload is fatal; transport interruption is not.
raise ConnectionError("result stream ended before source EOF") raise ConnectionError("result stream ended before source EOF")
@@ -240,20 +295,23 @@ async def run(args):
break break
source_end = True source_end = True
report["end_sent_monotonic_ns"] = time.monotonic_ns() report["end_sent_monotonic_ns"] = time.monotonic_ns()
if client is not None: await finish_source(client, reader, clock_control)
await client.end() if not clock_control:
await asyncio.wait_for(reader, timeout=5) write_control(
write_control( status / "source-end.json", {"run_id": access.epoch.run_id, "error": False}
status / "source-end.json", {"run_id": access.epoch.run_id, "error": False} )
)
except Exception: except Exception:
report["error"] = traceback.format_exc() report["error"] = traceback.format_exc()
if access is not None: if access is not None and not args.cross_host:
write_control( write_control(
status / "source-end.json", {"run_id": access.epoch.run_id, "error": True} status / "source-end.json", {"run_id": access.epoch.run_id, "error": True}
) )
finally: finally:
await disconnect() await disconnect()
if clock_control:
await clock_control.close()
report["clock_samples"] = clock_control.samples
report["clock_errors"] = clock_control.errors
from run_joint_pilot import distribution from run_joint_pilot import distribution
report.update( report.update(
@@ -291,6 +349,7 @@ async def run(args):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("--target", required=True) parser.add_argument("--target", required=True)
parser.add_argument("--cross-host", action="store_true")
parser.add_argument("--control", required=True) parser.add_argument("--control", required=True)
parser.add_argument("--source-status", required=True) parser.add_argument("--source-status", required=True)
parser.add_argument("--certificate", required=True) parser.add_argument("--certificate", required=True)
@@ -166,6 +166,8 @@ class PilotController:
def source_now(self): def source_now(self):
if self.source_clock is None: if self.source_clock is None:
raise WorkerLeaseError("source clock mapping not configured") raise WorkerLeaseError("source clock mapping not configured")
if hasattr(self.source_clock, "source_now"):
return self.source_clock.source_now(time.monotonic_ns())
return self.source_clock.source_zero + time.monotonic_ns() - self.source_clock.wall_zero return self.source_clock.source_zero + time.monotonic_ns() - self.source_clock.wall_zero
def _renew(self): def _renew(self):
@@ -0,0 +1,113 @@
"""Bounded application-side clock loop; no lease or model control."""
import asyncio
import secrets
import time
import grpc
from k1link.perception.realtime_contract import StreamStart
from k1link.perception.streaming_clock import ClockMappingError, ClockWindow
from k1link.perception.streaming_control_grpc import ControlTicket, StreamControlClient
from k1link.perception.streaming_source_clock import SourceAnchor
class SourceControl:
def __init__(self, target, roots, bootstrap):
ticket = ControlTicket(StreamStart.from_dict(bootstrap["activation"]), bootstrap["token"])
clock_id = "source-clock-" + secrets.token_hex(12)
self.client = StreamControlClient(target, roots, ticket, clock_id=clock_id)
self.window = ClockWindow(
clock_id, bootstrap["remote_clock_id"], rate_ppm=500, timestamp_error_ns=50_000
)
self.source_zero_ns = int(bootstrap["source_zero_ns"])
self.anchor = None # Latch once after the initial clock handshake, never on resume.
self.access = None
self.confirmed = False
self.failure = None
self.stopping = self.ending = False
self.samples, self.errors = [], []
self.task = None
async def update(self):
probe, access = await self.client.poll()
bounds = self.window.add(probe)
now = time.monotonic_ns()
if self.anchor is None:
try:
self.window.current(now).require(now)
except ClockMappingError:
pass # Pre-start clock warmup, not a running source pause or retimestamp.
else:
self.anchor = SourceAnchor(self.source_zero_ns, now + 2_000_000_000)
if self.anchor is not None:
await self.client.acknowledge(probe, self.anchor, ended=self.ending)
self.confirmed = True
self.access = access
now = time.monotonic_ns()
self.samples.append(
{
"at_ns": now,
"bounds": bounds.to_dict(),
"uncertainty_ns": bounds.uncertainty_ns(now),
"ready": self.ready(),
"acknowledged": self.anchor is not None,
}
)
if len(self.samples) > 2048:
raise ValueError("bounded clock diagnostic count exceeded")
def ready(self):
if self.failure:
raise RuntimeError("source clock task failed") from self.failure
if not self.confirmed:
return False
try:
now = time.monotonic_ns()
self.window.current(now).require(now)
except ClockMappingError:
return False
return True
async def loop(self):
while not self.stopping:
try:
await self.update()
if self.ending:
return
except (grpc.RpcError, TimeoutError, OSError) as exc:
self.confirmed = False
self.errors.append({"at_ns": time.monotonic_ns(), "type": type(exc).__name__})
if len(self.errors) > 128:
self.failure = RuntimeError("bounded clock transport error count exceeded")
return
except Exception as exc:
self.failure = exc
return
await asyncio.sleep(0.05)
def start(self):
self.task = asyncio.create_task(self.loop())
async def wait_anchor(self):
deadline = time.monotonic() + 10
while self.anchor is None or not self.confirmed:
if self.failure:
raise RuntimeError("initial source clock handshake failed") from self.failure
if time.monotonic() > deadline:
raise TimeoutError("bounded initial source clock handshake")
await asyncio.sleep(0.01)
return self.anchor
async def finish(self):
self.ending = True
await asyncio.wait_for(asyncio.shield(self.task), timeout=2)
if self.failure:
raise RuntimeError("source clock final acknowledgement failed") from self.failure
async def close(self):
self.stopping = True
if self.task:
self.task.cancel()
await asyncio.gather(self.task, return_exceptions=True)
await self.client.close()
@@ -276,6 +276,17 @@ def run(args):
"costmap_freshness_mode": args.costmap_freshness, "costmap_freshness_mode": args.costmap_freshness,
"ddrnet_min_source_interval_ms": args.ddrnet_min_source_interval_ms, "ddrnet_min_source_interval_ms": args.ddrnet_min_source_interval_ms,
"input_transport": args.input_transport, "input_transport": args.input_transport,
"network_clock_mode": "acknowledged-monotonic/v1"
if getattr(args, "network_cross_host", False)
else "local-only",
"network_clock_envelope": {
"relative_rate_ppm": 500,
"timestamp_error_ns": 50000,
"maximum_age_ns": 2000000000,
"maximum_uncertainty_ns": 5000000,
}
if getattr(args, "network_cross_host", False)
else None,
"recover_input": getattr(args, "recover_input", False), "recover_input": getattr(args, "recover_input", False),
"input_gap_plan": getattr(args, "input_gap", []), "input_gap_plan": getattr(args, "input_gap", []),
} }
@@ -549,6 +560,7 @@ def run(args):
private_key=args.network_private_key, private_key=args.network_private_key,
address=args.network_address, address=args.network_address,
reset_temporal=reset_temporal, reset_temporal=reset_temporal,
cross_host=args.network_cross_host,
) )
controller.source_clock = binary_bridge.source controller.source_clock = binary_bridge.source
report["scope"] = "full-graph-network-candidate; receiver evidence is separate" report["scope"] = "full-graph-network-candidate; receiver evidence is separate"
@@ -710,7 +722,10 @@ def run(args):
"scene_sha256": hashlib.sha256(encoded).hexdigest(), "scene_sha256": hashlib.sha256(encoded).hexdigest(),
} }
results.append(result) results.append(result)
except StreamSuspended: except StreamSuspended as exc:
report.setdefault("suspended_results", []).append(
{"sequence": bundle["sequence"], "reason": str(exc)}
)
mailbox.release(bundle, discard_reason="input-gap") mailbox.release(bundle, discard_reason="input-gap")
cpu_bundle = bundle = computed = mask = proposals = scene = received = view = ( cpu_bundle = bundle = computed = mask = proposals = scene = received = view = (
None None
@@ -947,6 +962,7 @@ if __name__ == "__main__":
parser.add_argument("--network-certificate") parser.add_argument("--network-certificate")
parser.add_argument("--network-private-key") parser.add_argument("--network-private-key")
parser.add_argument("--network-address", default="[::]:50061") parser.add_argument("--network-address", default="[::]:50061")
parser.add_argument("--network-cross-host", action="store_true")
parser.add_argument( parser.add_argument(
"--ddrnet-layout", choices=("reference", "channels-last"), default="reference" "--ddrnet-layout", choices=("reference", "channels-last"), default="reference"
) )
+56 -2
View File
@@ -69,6 +69,60 @@ class ClockProbe:
) )
@dataclass(frozen=True)
class ClockReceipt:
"""The responder's view AFTER an authenticated echo of its issued probe.
Responder receive=t2 and send=t3 bracket only processing, NOT network RTT.
t1-t2 is a lower bound and t4-t3 an upper bound for initiator-minus-responder.
Never construct an ordinary reversed ClockProbe: that would invert causality.
"""
probe: ClockProbe
acknowledged_ns: int
def __post_init__(self) -> None:
_stamp(self.acknowledged_ns)
if (
not self.probe.remote_send_ns
<= self.acknowledged_ns
<= (self.probe.remote_receive_ns + 500_000_000)
):
raise ClockMappingError("clock acknowledgement outside bounded round trip")
@property
def local_clock_id(self) -> str:
return self.probe.remote_clock_id
@property
def remote_clock_id(self) -> str:
return self.probe.local_clock_id
@property
def nonce(self) -> str:
return self.probe.nonce
@property
def local_send_ns(self) -> int:
return self.probe.remote_receive_ns
@property
def local_receive_ns(self) -> int:
return self.acknowledged_ns
def bounds_at(
self, local_ns: int, *, rate_ppm: int, timestamp_error_ns: int
) -> tuple[int, int]:
_stamp(local_ns)
if local_ns < self.acknowledged_ns:
raise ClockMappingError("acknowledgement is from the future")
widen = _drift(local_ns - self.probe.remote_receive_ns, rate_ppm) + 2 * timestamp_error_ns
return (
self.probe.local_send_ns - self.probe.remote_receive_ns - widen,
self.probe.local_receive_ns - self.probe.remote_send_ns + widen,
)
@dataclass(frozen=True) @dataclass(frozen=True)
class ClockBounds: class ClockBounds:
local_clock_id: str local_clock_id: str
@@ -161,11 +215,11 @@ class ClockWindow:
timestamp_error_ns, timestamp_error_ns,
maximum_age_ns, maximum_age_ns,
) )
self.samples: deque[ClockProbe] = deque(maxlen=16) self.samples: deque[ClockProbe | ClockReceipt] = deque(maxlen=16)
self.failed = False self.failed = False
self.last_receive_ns = -1 self.last_receive_ns = -1
def add(self, sample: ClockProbe) -> ClockBounds: def add(self, sample: ClockProbe | ClockReceipt) -> ClockBounds:
if self.failed: if self.failed:
raise ClockMappingError("clock session quarantined; explicit new session required") raise ClockMappingError("clock session quarantined; explicit new session required")
if ( if (
@@ -62,7 +62,13 @@ class InputContinuity:
raise StreamSuspended("input is waiting for resynchronization") raise StreamSuspended("input is waiting for resynchronization")
def pause(self, reason: str) -> None: def pause(self, reason: str) -> None:
if reason not in ("input-disconnected", "input-timeout", "source-gap", "worker-telemetry"): if reason not in (
"input-disconnected",
"input-timeout",
"source-gap",
"worker-telemetry",
"source-clock",
):
raise ValueError("unknown recoverable pause reason") raise ValueError("unknown recoverable pause reason")
if self.phase != "waiting": if self.phase != "waiting":
self.pauses += 1 self.pauses += 1
+113 -1
View File
@@ -23,10 +23,12 @@ from . import streaming_wire as wire
from .realtime_contract import StreamStart from .realtime_contract import StreamStart
from .streaming_clock import ClockProbe from .streaming_clock import ClockProbe
from .streaming_grpc import OPTIONS, StreamAccess from .streaming_grpc import OPTIONS, StreamAccess
from .streaming_source_clock import SourceAnchor
SERVICE = "missioncore.perception.v1.StreamControl" SERVICE = "missioncore.perception.v1.StreamControl"
METHOD = f"/{SERVICE}/Poll" METHOD = f"/{SERVICE}/Poll"
MAX_CONTROL = 8192 MAX_CONTROL = 8192
REPORT_METHOD = f"/{SERVICE}/ReportClock"
def _document(raw: bytes) -> dict[str, Any]: def _document(raw: bytes) -> dict[str, Any]:
@@ -77,6 +79,7 @@ class StreamControlEndpoint:
*, *,
clock_id: str, clock_id: str,
clock_ns: Callable[[], int] = time.monotonic_ns, clock_ns: Callable[[], int] = time.monotonic_ns,
observe: Callable[[ClockProbe, int, SourceAnchor, bool], None] | None = None,
) -> None: ) -> None:
wire.identifier(clock_id) wire.identifier(clock_id)
self.activation, self.pending, self.clock_id, self.clock_ns = ( self.activation, self.pending, self.clock_id, self.clock_ns = (
@@ -89,10 +92,16 @@ class StreamControlEndpoint:
self._lock = threading.Lock() self._lock = threading.Lock()
self._last_poll_ns = -1 self._last_poll_ns = -1
self.accepted = self.rejected = 0 self.accepted = self.rejected = 0
self.observe = observe
self._challenge: dict[str, Any] | None = None
def handler(self) -> Any: def handler(self) -> Any:
return grpc.method_handlers_generic_handler( return grpc.method_handlers_generic_handler(
SERVICE, {"Poll": grpc.unary_unary_rpc_method_handler(self.poll)} SERVICE,
{
"Poll": grpc.unary_unary_rpc_method_handler(self.poll),
"ReportClock": grpc.unary_unary_rpc_method_handler(self.report_clock),
},
) )
def _authenticate(self, context: Any) -> None: def _authenticate(self, context: Any) -> None:
@@ -163,6 +172,19 @@ class StreamControlEndpoint:
encoded = wire.canonical(response) encoded = wire.canonical(response)
if len(encoded) > MAX_CONTROL: if len(encoded) > MAX_CONTROL:
raise ValueError("trusted control snapshot exceeds bound") raise ValueError("trusted control snapshot exceeds bound")
if self.observe is not None:
with self._lock:
self._challenge = {
key: response[key]
for key in (
"nonce",
"local_clock_id",
"local_send_ns",
"remote_clock_id",
"remote_receive_ns",
"remote_send_ns",
)
}
except (ValueError, RuntimeError): except (ValueError, RuntimeError):
await context.abort( await context.abort(
grpc.StatusCode.UNAVAILABLE, "local controller has no current offer" grpc.StatusCode.UNAVAILABLE, "local controller has no current offer"
@@ -171,6 +193,52 @@ class StreamControlEndpoint:
self.accepted += 1 self.accepted += 1
return encoded return encoded
async def report_clock(self, raw: bytes, context: Any) -> bytes:
received = self.clock_ns()
try:
self._authenticate(context)
value = _document(raw)
if set(value) != {"probe", "anchor", "ended"} or self.observe is None:
raise ValueError("clock receipt not admitted")
evidence = value["probe"]
with self._lock:
challenge = self._challenge
if (
challenge is None
or not isinstance(evidence, dict)
or set(evidence) != {*challenge, "local_receive_ns"}
or any(evidence[k] != v for k, v in challenge.items())
):
raise ValueError("clock receipt is not the current issued challenge")
self._challenge = None # Single-use, including malformed/late receipts.
probe = ClockProbe(
evidence["local_clock_id"],
evidence["remote_clock_id"],
evidence["nonce"],
*(
wire.uint64(evidence[k])
for k in (
"local_send_ns",
"remote_receive_ns",
"remote_send_ns",
"local_receive_ns",
)
),
)
anchor = SourceAnchor.from_dict(value["anchor"])
self.observe(probe, received, anchor, value["ended"])
except (ValueError, TypeError):
self.rejected += 1
await context.abort(grpc.StatusCode.INVALID_ARGUMENT, "clock receipt rejected")
return b""
return wire.canonical(
{
"schema_version": "missioncore.stream-clock-receipt/v1",
"activation_binding": wire.binding(self.activation),
"nonce": probe.nonce,
}
)
class StreamControlClient: class StreamControlClient:
def __init__( def __init__(
@@ -196,6 +264,7 @@ class StreamControlClient:
), ),
) )
self.call = self.channel.unary_unary(METHOD) self.call = self.channel.unary_unary(METHOD)
self.report_call = self.channel.unary_unary(REPORT_METHOD)
self.polling = False self.polling = False
async def poll(self) -> tuple[ClockProbe, StreamAccess | None]: async def poll(self) -> tuple[ClockProbe, StreamAccess | None]:
@@ -267,3 +336,46 @@ class StreamControlClient:
async def close(self) -> None: async def close(self) -> None:
await self.channel.close() await self.channel.close()
async def acknowledge(
self, probe: ClockProbe, anchor: SourceAnchor, *, ended: bool = False
) -> None:
if self.polling or probe.local_clock_id != self.clock_id or type(ended) is not bool:
raise ValueError("one scoped control request at a time")
self.polling = True
try:
evidence = {
name: getattr(probe, name)
for name in (
"nonce",
"local_clock_id",
"remote_clock_id",
)
}
evidence.update(
{
name: str(getattr(probe, name))
for name in (
"local_send_ns",
"remote_receive_ns",
"remote_send_ns",
"local_receive_ns",
)
}
)
response = _document(
await self.report_call(
wire.canonical({"probe": evidence, "anchor": anchor.to_dict(), "ended": ended}),
metadata=self.ticket.metadata(),
timeout=0.5,
wait_for_ready=False,
)
)
if response != {
"schema_version": "missioncore.stream-clock-receipt/v1",
"activation_binding": wire.binding(self.ticket.activation),
"nonce": probe.nonce,
}:
raise ValueError("clock receipt response binding mismatch")
finally:
self.polling = False
+17 -16
View File
@@ -291,22 +291,23 @@ class StreamingIngress:
except Exception as exc: except Exception as exc:
self.error = str(exc) self.error = str(exc)
self.terminal = "failed" self.terminal = "failed"
if self.opened: if self.runtime.continuity is not None and isinstance(
if self.runtime.continuity is not None and isinstance( exc, (InputInterrupted, StreamSuspended)
exc, (InputInterrupted, StreamSuspended) ):
): # Admission already consumed the single-use grant. A transport
self.terminal = "paused" # loss before the application OPEN is still WAIT, not an active
# A late old socket cannot pause a replacement connection. # owner stranded forever. A late old socket remains fenced.
with suppress(RuntimeError): # Already replaced, stopping, or fenced. self.terminal = "paused"
self.runtime.pause_input( with suppress(RuntimeError): # Already replaced, stopping, or fenced.
self.input_epoch, self.runtime.pause_input(
"input-disconnected" self.input_epoch,
if isinstance(exc, InputInterrupted) "input-disconnected"
else "source-gap", if isinstance(exc, InputInterrupted)
) else "source-gap",
else: )
self.runtime.mailbox.finish(self.error) elif self.opened:
self.runtime.request_stop("failed") self.runtime.mailbox.finish(self.error)
self.runtime.request_stop("failed")
finally: finally:
self.connection.close() self.connection.close()
if reservation is not None: if reservation is not None:
+12 -5
View File
@@ -223,11 +223,17 @@ class StreamingLifecycle:
return return
self._check(self.start) self._check(self.start)
self.continuity.check(epoch, synchronizing=synchronizing) self.continuity.check(epoch, synchronizing=synchronizing)
if event_ns is not None: assert self._source_clock_ns is not None
assert self._source_clock_ns is not None try:
now = self._source_clock_ns() now = self._source_clock_ns()
if event_ns < self.continuity.cutoff_ns or not 0 <= now - event_ns <= 250_000_000: except StreamSuspended:
raise StreamSuspended("obsolete or future input, no backlog replay") self.continuity.pause("source-clock")
self.mailbox.pause()
raise
if event_ns is not None and (
event_ns < self.continuity.cutoff_ns or not 0 <= now - event_ns <= 250_000_000
):
raise StreamSuspended("obsolete or future input, no backlog replay")
def pause_input(self, epoch: StreamStart, reason: str) -> None: def pause_input(self, epoch: StreamStart, reason: str) -> None:
with self._lock: with self._lock:
@@ -253,11 +259,12 @@ class StreamingLifecycle:
raise StreamSuspended("old epoch callbacks or connection still active") raise StreamSuspended("old epoch callbacks or connection still active")
if self.continuity.phase != "waiting": if self.continuity.phase != "waiting":
raise StreamSuspended("pause required before a new input epoch") raise StreamSuspended("pause required before a new input epoch")
source_now = self._source_clock_ns() # Check BEFORE mutating the paused mailbox.
self.mailbox.begin_epoch() self.mailbox.begin_epoch()
if self._ingress_thread is not None: if self._ingress_thread is not None:
self._threads = [t for t in self._threads if t is not self._ingress_thread] self._threads = [t for t in self._threads if t is not self._ingress_thread]
self._ingress_thread = None self._ingress_thread = None
return self.continuity.begin(self._source_clock_ns()) return self.continuity.begin(source_now)
def resume_input( def resume_input(
self, epoch: StreamStart, evidence: ResumeEvidence, reset_temporal: Callable[[], None] self, epoch: StreamStart, evidence: ResumeEvidence, reset_temporal: Callable[[], None]
@@ -0,0 +1,94 @@
"""Responder-owned source timeline and conditional clock readiness, no GPU authority."""
from __future__ import annotations
import threading
from dataclasses import dataclass
from . import streaming_wire as wire
from .streaming_clock import ClockMappingError, ClockProbe, ClockReceipt, ClockWindow, _stamp
from .streaming_continuity import StreamSuspended
@dataclass(frozen=True)
class SourceAnchor:
source_zero_ns: int
local_zero_ns: int
def __post_init__(self) -> None:
_stamp(self.source_zero_ns)
_stamp(self.local_zero_ns)
def to_dict(self) -> dict[str, str]:
return {
"source_zero_ns": str(self.source_zero_ns),
"local_zero_ns": str(self.local_zero_ns),
}
@classmethod
def from_dict(cls, value: object) -> SourceAnchor:
if not isinstance(value, dict) or set(value) != {"source_zero_ns", "local_zero_ns"}:
raise ValueError("invalid immutable source anchor")
return cls(wire.uint64(value["source_zero_ns"]), wire.uint64(value["local_zero_ns"]))
class SourceClockMonitor:
def __init__(self, worker_clock_id: str, source_zero_ns: int) -> None:
wire.identifier(worker_clock_id)
_stamp(source_zero_ns)
self.worker_clock_id, self.source_zero = worker_clock_id, source_zero_ns
self.wall_zero = 0 # Not a comparable host clock; use due()/observed().
self.anchor: SourceAnchor | None = None
self.window: ClockWindow | None = None
self.ended = False
self._lock = threading.RLock()
def observe(
self, probe: ClockProbe, received_ns: int, anchor: SourceAnchor, ended: bool
) -> None:
"""Called only after the transport verifies the one-use issued challenge."""
receipt = ClockReceipt(probe, received_ns)
if probe.remote_clock_id != self.worker_clock_id or type(ended) is not bool:
raise ValueError("foreign clock/status")
with self._lock:
if self.ended or anchor.source_zero_ns != self.source_zero:
raise ValueError("source activation ended or source anchor changed")
if self.anchor is None:
if not 0 <= anchor.local_zero_ns - probe.local_receive_ns <= 10_000_000_000:
raise ValueError("initial source start must be bounded and not in the past")
self.anchor = anchor
self.window = ClockWindow(
self.worker_clock_id,
probe.local_clock_id,
rate_ppm=500,
timestamp_error_ns=50_000,
)
elif anchor != self.anchor:
raise ValueError("source timeline is immutable for the activation")
assert self.window is not None
self.window.add(receipt)
self.ended = ended
def observed(self, now_ns: int) -> tuple[int, int]:
"""Source time midpoint + integer half-width, freshly checked at use."""
with self._lock:
try:
if self.window is None or self.anchor is None:
raise ClockMappingError("source clock not observed")
bounds = self.window.current(now_ns)
bounds.require(now_ns)
lower, upper = bounds.offset_at(now_ns)
stamp = self.source_zero + now_ns + (lower + upper) // 2 - self.anchor.local_zero_ns
_stamp(stamp)
return stamp, (upper - lower + 1) // 2
except ClockMappingError as exc:
raise StreamSuspended("source clock unavailable") from exc
def source_now(self, now_ns: int) -> int:
stamp, uncertainty = self.observed(now_ns)
# Admission/cutoff/resume use the upper age bound, never a younger estimate.
return stamp + uncertainty
def due(self, stamp: int, now_ns: int) -> tuple[int, int]:
observed, uncertainty = self.observed(now_ns)
return now_ns + stamp - observed, uncertainty
+189 -5
View File
@@ -10,11 +10,18 @@ import pytest
pytest.importorskip("grpc") pytest.importorskip("grpc")
import time
from test_perception_streaming_grpc import eventually, identity, tls # noqa: F401,E402 from test_perception_streaming_grpc import eventually, identity, tls # noqa: F401,E402
from k1link.perception.streaming_control_grpc import ( # noqa: E402
ControlTicket,
StreamControlClient,
)
from k1link.perception.streaming_grpc import GrpcStreamClient # noqa: E402 from k1link.perception.streaming_grpc import GrpcStreamClient # noqa: E402
from k1link.perception.streaming_lifecycle import StreamingLifecycle # noqa: E402 from k1link.perception.streaming_lifecycle import StreamingLifecycle # noqa: E402
from k1link.perception.streaming_queue import StreamMailbox # noqa: E402 from k1link.perception.streaming_queue import StreamMailbox # noqa: E402
from k1link.perception.streaming_source_clock import SourceAnchor # noqa: E402
@pytest.fixture @pytest.fixture
@@ -33,7 +40,8 @@ def adapter(monkeypatch):
return module, control return module, control
def test_adapter_rotates_grant_only_after_old_stream_drains(tmp_path, tls, adapter): # noqa: F811 @pytest.mark.parametrize("before_open", [False, True])
def test_adapter_rotates_grant_only_after_old_stream_drains(tmp_path, tls, adapter, before_open): # noqa: F811
module, control = adapter module, control = adapter
cert, key = tmp_path / "cert", tmp_path / "key" cert, key = tmp_path / "cert", tmp_path / "key"
cert.write_bytes(tls[0]) cert.write_bytes(tls[0])
@@ -78,11 +86,21 @@ def test_adapter_rotates_grant_only_after_old_stream_drains(tmp_path, tls, adapt
_, first = control.read_grant(tmp_path / "grant.json") _, first = control.read_grant(tmp_path / "grant.json")
client = GrpcStreamClient(f"localhost:{bridge.port}", tls[0], first) client = GrpcStreamClient(f"localhost:{bridge.port}", tls[0], first)
try: try:
await client.open() if before_open:
await client.call.initial_metadata() # Grant consumed, no application OPEN sent.
else:
await client.open()
await client.close() # True network disconnect, not a synthetic runtime pause. await client.close() # True network disconnect, not a synthetic runtime pause.
await eventually( try:
lambda: control.read_grant(tmp_path / "grant.json")[1].epoch != first.epoch await eventually(
) lambda: control.read_grant(tmp_path / "grant.json")[1].epoch != first.epoch
)
except TimeoutError:
pytest.fail(
f"epoch did not rotate: phase={runtime.continuity.phase}; "
f"stop={runtime.stop_event.is_set()}; adapter={bridge.failure}; "
f"last={bridge.endpoint.last}; active={bridge.endpoint.active is not None}"
)
_, second = control.read_grant(tmp_path / "grant.json") _, second = control.read_grant(tmp_path / "grant.json")
assert runtime.continuity.phase == "synchronizing" assert runtime.continuity.phase == "synchronizing"
assert second.epoch.lease_generation == first.epoch.lease_generation assert second.epoch.lease_generation == first.epoch.lease_generation
@@ -114,3 +132,169 @@ def test_delayed_source_start_does_not_open_an_idle_connection(adapter):
assert source.connection_delay(3_000_000_000, 1_000_000_000) == 1.9 assert source.connection_delay(3_000_000_000, 1_000_000_000) == 1.9
assert source.connection_delay(3_000_000_000, 2_950_000_000) == 0 assert source.connection_delay(3_000_000_000, 2_950_000_000) == 0
assert source.connection_delay(3_000_000_000, 3_010_000_000) == 0 assert source.connection_delay(3_000_000_000, 3_010_000_000) == 0
@pytest.mark.parametrize("has_stream", [False, True])
def test_source_eof_ack_precedes_data_channel_retirement(adapter, has_stream):
source = importlib.import_module("pilot_grpc_source")
calls = []
async def acknowledge():
assert "end" not in calls
calls.append("ack")
async def end():
assert calls == ["ack"]
calls.append("end")
async def read():
calls.append("drain")
async def check():
await source.finish_source(
SimpleNamespace(end=end) if has_stream else None,
read() if has_stream else None,
SimpleNamespace(finish=acknowledge),
)
asyncio.run(check())
assert calls == (["ack", "end", "drain"] if has_stream else ["ack"])
def test_source_anchor_latches_only_after_initial_clock_gate_and_never_on_recovery(
adapter, monkeypatch
):
module = importlib.import_module("pilot_source_control")
healthy, acknowledgements = [False], []
class Bounds:
def require(self, now):
if not healthy[0]:
raise module.ClockMappingError("synthetic excessive uncertainty")
def to_dict(self):
return {}
def uncertainty_ns(self, now):
return 1_000_000 if healthy[0] else 6_000_000
class Client:
def __init__(self, *args, **kwargs):
pass
async def poll(self):
return object(), None
async def acknowledge(self, probe, anchor, *, ended):
acknowledgements.append(anchor)
monkeypatch.setattr(module, "StreamControlClient", Client)
monkeypatch.setattr(
module,
"ClockWindow",
lambda *a, **kw: SimpleNamespace(add=lambda probe: Bounds(), current=lambda now: Bounds()),
)
control = module.SourceControl(
"unused",
b"",
{
"activation": identity().to_dict(),
"token": "b" * 64,
"remote_clock_id": "worker",
"source_zero_ns": str(10**12),
},
)
async def check():
await control.update()
assert control.anchor is None and not control.ready() and not acknowledgements
healthy[0] = True
await control.update()
anchor = await control.wait_anchor()
assert anchor == control.anchor and control.ready()
healthy[0] = False
await control.update()
assert not control.ready() and control.anchor == anchor
healthy[0] = True
await control.update()
assert control.ready() and acknowledgements == [anchor] * 3
assert [row["acknowledged"] for row in control.samples] == [False, True, True, True]
asyncio.run(check())
def test_cross_host_bridge_clock_expiry_retains_owner_and_rotates_epoch(tmp_path, tls, adapter): # noqa: F811
module, _ = adapter
cert, key = tmp_path / "cert", tmp_path / "key"
cert.write_bytes(tls[0])
key.write_bytes(tls[1])
run = StreamingLifecycle(
identity(),
tmp_path / "lease",
StreamMailbox(),
threading.Event(),
clock_ns=lambda: 1_000_000_000,
recover_input=True,
source_clock_ns=lambda: bridge.source.source_now(time.monotonic_ns()),
)
run.ready()
bridge = module.NetworkGraphBridge(
run,
SimpleNamespace(close=lambda: None),
{},
control=tmp_path,
source_status=tmp_path / "unused",
source_zero=10**12,
certificate=cert,
private_key=key,
address="localhost:0",
reset_temporal=lambda: None,
cross_host=True,
)
real_serve = bridge.endpoint.serve
async def serve(*args, **kwargs):
server, port = await real_serve(*args, **kwargs)
bridge.port = port
return server, port
bridge.endpoint.serve = serve
async def check():
bridge.start()
ticket = ControlTicket(identity(), bridge.ticket.token)
client = StreamControlClient(f"localhost:{bridge.port}", tls[0], ticket, clock_id="source")
try:
probe, missing = await client.poll()
assert missing is None and run.continuity.phase == "waiting"
anchor = SourceAnchor(10**12, probe.local_receive_ns + 1_000_000_000)
await client.acknowledge(probe, anchor)
await eventually(lambda: bridge.access is not None)
first = bridge.access
stream = GrpcStreamClient(f"localhost:{bridge.port}", tls[0], first)
await stream.open()
# The listener keeps observing clock freshness even with no data.
# Shorten the TEST clock evidence lifetime, not any production gate.
bridge.source.window.maximum_age_ns = 50_000_000
await eventually(lambda: run.continuity.phase == "waiting")
await stream.close()
await eventually(lambda: bridge.endpoint.active is None)
assert not run.stop_event.is_set() and run.lease.start == identity()
run.renew(identity())
bridge.source.window.maximum_age_ns = 2_000_000_000
# Old evidence must not itself resume; new controller epoch still required.
await asyncio.sleep(0.03)
probe, _ = await client.poll()
await client.acknowledge(probe, anchor)
await eventually(lambda: bridge.access.epoch != first.epoch)
assert run.continuity.phase == "synchronizing"
assert len(bridge.clock_states) >= 3
finally:
await client.close()
try:
asyncio.run(check())
finally:
run.request_stop("completed")
assert bridge.close() and run.close()
assert not (tmp_path / "bootstrap.json").exists()
+42
View File
@@ -362,6 +362,48 @@ def test_consumer_expires_only_old_cell_and_rehashes_view_without_mutating_wire(
pilot.assess_receipt(view, derived, bundle=bundle, now_ns=bundle["due_ns"]) pilot.assess_receipt(view, derived, bundle=bundle, now_ns=bundle["due_ns"])
def test_cross_host_uncertainty_expires_per_cell_and_survives_publication(pilot):
payload, bundle, ddr = cell_input()
# Without uncertainty old ground would be 249 ms old; upper bound is253 ms.
bundle["clock_observer"] = lambda _: (NOW + 49_000_000, 4_000_000)
pilot.prepare_publication(payload, bundle, ddr, epoch_id="pilot", now_ns=123, mode="per-cell")
assert payload["cell_assessment"]["expired_ground_cells"] == 1
assert payload["policy_actions"] == [2, 0]
assert payload["freshness_at_publication"]["clock_uncertainty_ms"] == 4
fresh = pilot.validate_receipt(payload, bundle, epoch_id="pilot")
# An independently measured later receipt can have a slightly smaller
# midpoint while still overlapping the published interval.
local_bundle = {k: v for k, v in bundle.items() if k != "clock_observer"}
view, checked = pilot.assess_receipt(
payload, fresh, bundle=local_bundle, now_ns=bundle["due_ns"] + 48_000_000
)
assert checked.clock_uncertainty_ms == 0 and view["policy_actions"] == [2, 0]
with pytest.raises(ValueError, match="backwards"):
pilot.assess_receipt(
payload, fresh, bundle=local_bundle, now_ns=bundle["due_ns"] + 44_000_000
)
def test_one_clock_snapshot_per_publication_and_receipt_boundary(pilot):
payload, bundle, ddr = cell_input()
calls = []
def observe(now):
assert now not in calls # A later refresh may no longer describe this instant.
calls.append(now)
return NOW + now, 4_000_000
bundle["clock_observer"] = observe
pilot.prepare_publication(
payload, bundle, ddr, epoch_id="pilot", now_ns=260_000_000, mode="per-cell"
)
assert payload["policy_actions"] == [2, 2] # Exercises suppression/reassessment too.
fresh = pilot.validate_receipt(payload, bundle, epoch_id="pilot")
view, checked = pilot.assess_receipt(payload, fresh, bundle=bundle, now_ns=270_000_000)
assert calls == [260_000_000, 270_000_000]
assert not checked.fresh_complete and view["policy_actions"] == [2, 2]
@pytest.mark.parametrize("missing,held_ms", [(True, 0), (False, 220)]) @pytest.mark.parametrize("missing,held_ms", [(True, 0), (False, 220)])
def test_cell_freshness_cannot_override_missing_lidar_or_stale_segmentation( def test_cell_freshness_cannot_override_missing_lidar_or_stale_segmentation(
pilot, missing, held_ms pilot, missing, held_ms
+124
View File
@@ -0,0 +1,124 @@
"""Two-sided clock evidence and recoverable owner-preserving admission."""
import threading
from dataclasses import replace
import pytest
from test_perception_streaming_grpc import identity
from k1link.perception.streaming_clock import (
ClockMappingError,
ClockProbe,
ClockReceipt,
ClockWindow,
)
from k1link.perception.streaming_continuity import StreamSuspended
from k1link.perception.streaming_lifecycle import StreamingLifecycle
from k1link.perception.streaming_queue import StreamMailbox
from k1link.perception.streaming_source_clock import SourceAnchor, SourceClockMonitor
def sample(number=1, outbound=1_000_000, inbound=2_000_000, offset=10**12):
t1 = 10**16 + number * 100_000_000
return ClockProbe(
"source",
"worker",
str(number),
t1,
t1 + outbound + offset,
t1 + outbound + offset + 10_000,
t1 + outbound + inbound + 10_000,
)
@pytest.mark.parametrize("outbound,inbound", [(1, 9_000_000), (9_000_000, 1), (1, 1)])
@pytest.mark.parametrize("offset", [-(10**12), 0, 10**12])
def test_responder_interval_contains_true_reverse_offset(outbound, inbound, offset):
probe = sample(outbound=outbound, inbound=inbound, offset=offset)
receipt = ClockReceipt(probe, probe.local_receive_ns + offset + 1_000_000)
window = ClockWindow("worker", "source", rate_ppm=500, timestamp_error_ns=50_000)
bounds = window.add(receipt)
lower, upper = bounds.offset_at(receipt.acknowledged_ns)
assert lower <= -offset <= upper
assert bounds.uncertainty_ns(receipt.acknowledged_ns + 1_000_000) > bounds.uncertainty_ns(
receipt.acknowledged_ns
)
def test_source_timeline_immutable_expiring_and_observation_bounds_conservative():
first = sample()
anchor = SourceAnchor(10**12, first.local_receive_ns + 2_000_000_000)
monitor = SourceClockMonitor("worker", anchor.source_zero_ns)
with pytest.raises(StreamSuspended):
monitor.observed(first.remote_send_ns)
now = first.local_receive_ns + 10**12 + 1_000_000
monitor.observe(first, now, anchor, False)
midpoint, uncertainty = monitor.observed(now)
actual = anchor.source_zero_ns + (now - 10**12) - anchor.local_zero_ns
assert midpoint - uncertainty <= actual <= monitor.source_now(now)
with pytest.raises(StreamSuspended):
monitor.observed(now + 2_000_000_000)
second = sample(30)
later = second.local_receive_ns + 10**12 + 1_000_000
with pytest.raises(ValueError, match="immutable"):
monitor.observe(
second, later, replace(anchor, local_zero_ns=anchor.local_zero_ns + 1), False
)
monitor.observe(second, later, anchor, True)
assert monitor.ended
with pytest.raises(ValueError, match="ended"):
monitor.observe(sample(31), later + 100_000_000, anchor, False)
def test_bad_ack_order_expired_jump_and_foreign_clock():
first = sample()
for stamp in (first.remote_send_ns - 1, first.remote_receive_ns + 500_000_001):
with pytest.raises(ClockMappingError):
ClockReceipt(first, stamp)
monitor = SourceClockMonitor("worker", 10**12)
anchor = SourceAnchor(10**12, first.local_receive_ns + 2_000_000_000)
monitor.observe(first, first.local_receive_ns + 10**12 + 1_000_000, anchor, False)
jumped = sample(2, offset=10**12 + 100_000_000)
with pytest.raises(ClockMappingError, match="envelope"):
monitor.observe(jumped, jumped.local_receive_ns + 10**12 + 101_000_000, anchor, False)
with pytest.raises(StreamSuspended):
monitor.observed(jumped.local_receive_ns + 10**12 + 101_000_000)
def test_clock_wait_does_not_stop_owner_or_reopen_mailbox_prematurely(tmp_path):
healthy = [True]
def source_now():
if not healthy[0]:
raise StreamSuspended("synthetic expired clock mapping")
return 1_000_000_000
run = StreamingLifecycle(
identity(),
tmp_path,
StreamMailbox(),
threading.Event(),
clock_ns=lambda: 1_000_000_000,
recover_input=True,
source_clock_ns=source_now,
)
run.ready()
try:
healthy[0] = False
with pytest.raises(StreamSuspended):
run.check_input(identity())
assert run.continuity.reason == "source-clock" and not run.stop_event.is_set()
run.renew(identity())
with pytest.raises(StreamSuspended):
run.begin_input(identity())
assert run.mailbox.epoch_drained and run.continuity.phase == "waiting"
healthy[0] = True
epoch = run.begin_input(identity())
assert epoch != identity() and epoch.lease_generation == 1
assert run.continuity.phase == "synchronizing"
healthy[0] = False
with pytest.raises(StreamSuspended, match="obsolete"):
run.check_input(identity())
assert run.continuity.phase == "synchronizing" # Stale peer cannot pause new epoch.
finally:
assert run.close()
@@ -28,6 +28,7 @@ from k1link.perception.streaming_control_grpc import (
from k1link.perception.streaming_grpc import GrpcStreamClient, GrpcStreamEndpoint, StreamAccess from k1link.perception.streaming_grpc import GrpcStreamClient, GrpcStreamEndpoint, StreamAccess
from k1link.perception.streaming_lifecycle import StreamingLifecycle from k1link.perception.streaming_lifecycle import StreamingLifecycle
from k1link.perception.streaming_queue import StreamMailbox from k1link.perception.streaming_queue import StreamMailbox
from k1link.perception.streaming_source_clock import SourceAnchor, SourceClockMonitor
@asynccontextmanager @asynccontextmanager
@@ -204,3 +205,52 @@ def test_malicious_response_nonce_or_offer_rejected(tls):
await client.close() await client.close()
asyncio.run(check()) asyncio.run(check())
def test_acknowledged_clock_is_single_use_and_coexists_with_active_stream(tmp_path, tls):
async def check():
async with controlled(tmp_path, tls) as (_, endpoint, control, client, pending, port):
monitor = SourceClockMonitor("worker", 10**12)
control.observe = monitor.observe
probe, access = await client.poll()
anchor = SourceAnchor(10**12, probe.local_receive_ns + 2_000_000_000)
await client.acknowledge(probe, anchor)
assert monitor.observed(time.monotonic_ns())[1] > 0
with pytest.raises(grpc.aio.AioRpcError):
await client.acknowledge(probe, anchor)
stream = GrpcStreamClient(f"localhost:{port}", tls[0], access)
try:
await stream.open()
pending[0] = None
for _ in range(12):
await asyncio.sleep(0.03)
probe, missing = await client.poll()
assert missing is None
await client.acknowledge(probe, anchor)
assert monitor.observed(time.monotonic_ns())[1] < 5_000_000
assert endpoint.active is not None
finally:
await stream.close()
asyncio.run(check())
@pytest.mark.parametrize("fault", ["nonce", "remote_send", "anchor", "end_type"])
def test_wrong_clock_receipt_cannot_update_mapping(tmp_path, tls, fault):
async def check():
async with controlled(tmp_path, tls) as (_, _, control, client, _, _):
monitor = SourceClockMonitor("worker", 10**12)
control.observe = monitor.observe
probe, _ = await client.poll()
anchor = SourceAnchor(10**12, probe.local_receive_ns + 2_000_000_000)
if fault == "nonce":
probe = replace(probe, nonce="foreign")
elif fault == "remote_send":
probe = replace(probe, remote_send_ns=probe.remote_send_ns + 1)
elif fault == "anchor":
anchor = replace(anchor, source_zero_ns=anchor.source_zero_ns + 1)
with pytest.raises((grpc.aio.AioRpcError, ValueError)):
await client.acknowledge(probe, anchor, ended=1 if fault == "end_type" else False)
assert monitor.anchor is None and monitor.window is None
asyncio.run(check())