feat(perception): gate cross-host graph with acknowledged source clocks
This commit is contained in:
@@ -126,7 +126,11 @@ class BinaryGraphInput:
|
|||||||
length = len(e.value[0])
|
length = len(e.value[0])
|
||||||
rolling_times[offset : offset + length] = e.time_ns
|
rolling_times[offset : offset + length] = e.time_ns
|
||||||
offset += length
|
offset += length
|
||||||
due = self.wall_zero + stamp - self.source_zero
|
due = (
|
||||||
|
self.source.due(stamp, time.monotonic_ns())[0]
|
||||||
|
if hasattr(self.source, "observed")
|
||||||
|
else self.wall_zero + stamp - self.source_zero
|
||||||
|
)
|
||||||
bundle = {
|
bundle = {
|
||||||
"input_start": self.epoch,
|
"input_start": self.epoch,
|
||||||
"sequence": event.source_sequence,
|
"sequence": event.source_sequence,
|
||||||
@@ -158,6 +162,8 @@ class BinaryGraphInput:
|
|||||||
"enqueued_ns": time.monotonic_ns(),
|
"enqueued_ns": time.monotonic_ns(),
|
||||||
"payload_bytes": size,
|
"payload_bytes": size,
|
||||||
}
|
}
|
||||||
|
if hasattr(self.source, "observed"):
|
||||||
|
bundle["clock_observer"] = self.source.observed
|
||||||
self.bgr_hashes.append(hashlib.sha256(image).hexdigest())
|
self.bgr_hashes.append(hashlib.sha256(image).hexdigest())
|
||||||
bundle["enqueued_ns"] = time.monotonic_ns()
|
bundle["enqueued_ns"] = time.monotonic_ns()
|
||||||
transferred = self.runtime.admit_reserved(self.epoch, bundle, reservation)
|
transferred = self.runtime.admit_reserved(self.epoch, bundle, reservation)
|
||||||
|
|||||||
@@ -89,15 +89,25 @@ def build_freshness(scene, bundle, ddr_layer, epoch_id):
|
|||||||
return SceneFreshness(epoch_id, CLOCK_DOMAIN, sequence, stamp, tuple(layers.values()))
|
return SceneFreshness(epoch_id, CLOCK_DOMAIN, sequence, stamp, tuple(layers.values()))
|
||||||
|
|
||||||
|
|
||||||
|
def source_observation(bundle, now_ns):
|
||||||
|
if "clock_observer" in bundle:
|
||||||
|
stamp, uncertainty_ns = bundle["clock_observer"](now_ns)
|
||||||
|
return stamp, uncertainty_ns / 1e6
|
||||||
|
# Legacy same-kernel path, or source/consumer on the originating Mac clock.
|
||||||
|
return bundle["time_ns"] + now_ns - bundle["due_ns"], 0
|
||||||
|
|
||||||
|
|
||||||
def assess(freshness, *, bundle, now_ns):
|
def assess(freshness, *, bundle, now_ns):
|
||||||
# Both sides of this pilot use the SAME monotonic clock. Map it back to
|
observed, uncertainty = source_observation(bundle, now_ns)
|
||||||
# original source arrival time; zero mapping uncertainty is local-only and
|
return assess_observation(freshness, observed, uncertainty)
|
||||||
# is not a claim of hardware camera/LiDAR synchronization or network quality.
|
|
||||||
|
|
||||||
|
def assess_observation(freshness, observed, uncertainty):
|
||||||
return freshness.assess(
|
return freshness.assess(
|
||||||
epoch_id=freshness.epoch_id,
|
epoch_id=freshness.epoch_id,
|
||||||
clock_domain_id=CLOCK_DOMAIN,
|
clock_domain_id=CLOCK_DOMAIN,
|
||||||
observed_source_time_ns=bundle["time_ns"] + now_ns - bundle["due_ns"],
|
observed_source_time_ns=observed,
|
||||||
clock_uncertainty_ms=0,
|
clock_uncertainty_ms=uncertainty,
|
||||||
maximum_clock_uncertainty_ms=5,
|
maximum_clock_uncertainty_ms=5,
|
||||||
maximum_layer_age_ms=250,
|
maximum_layer_age_ms=250,
|
||||||
)
|
)
|
||||||
@@ -141,19 +151,23 @@ def prepare_publication(scene, bundle, ddr_layer, *, epoch_id, now_ns, mode="who
|
|||||||
if mode not in ("whole-scene", "per-cell"):
|
if mode not in ("whole-scene", "per-cell"):
|
||||||
raise ValueError("unknown costmap freshness mode")
|
raise ValueError("unknown costmap freshness mode")
|
||||||
scene["costmap_freshness_mode"] = mode
|
scene["costmap_freshness_mode"] = mode
|
||||||
|
observed, uncertainty = source_observation(bundle, now_ns)
|
||||||
if mode == "per-cell":
|
if mode == "per-cell":
|
||||||
apply_cell_expiry(
|
apply_cell_expiry(
|
||||||
scene,
|
scene,
|
||||||
source_time_ns=bundle["time_ns"],
|
source_time_ns=bundle["time_ns"],
|
||||||
observed_source_time_ns=bundle["time_ns"] + now_ns - bundle["due_ns"],
|
observed_source_time_ns=observed,
|
||||||
|
uncertainty_ms=uncertainty,
|
||||||
)
|
)
|
||||||
freshness = build_freshness(scene, bundle, ddr_layer, epoch_id)
|
freshness = build_freshness(scene, bundle, ddr_layer, epoch_id)
|
||||||
checked = assess(freshness, bundle=bundle, now_ns=now_ns)
|
# One immutable clock observation per boundary, including cell and policy
|
||||||
|
# reassessment. A concurrent refresh must not reinterpret this same instant.
|
||||||
|
checked = assess_observation(freshness, observed, uncertainty)
|
||||||
if not checked.fresh_complete:
|
if not checked.fresh_complete:
|
||||||
suppress_policy(scene)
|
suppress_policy(scene)
|
||||||
# The envelope hashes the actual guarded policy, not the discarded one.
|
# The envelope hashes the actual guarded policy, not the discarded one.
|
||||||
freshness = build_freshness(scene, bundle, ddr_layer, epoch_id)
|
freshness = build_freshness(scene, bundle, ddr_layer, epoch_id)
|
||||||
checked = assess(freshness, bundle=bundle, now_ns=now_ns)
|
checked = assess_observation(freshness, observed, uncertainty)
|
||||||
scene["freshness"] = freshness.to_dict()
|
scene["freshness"] = freshness.to_dict()
|
||||||
scene["freshness_at_publication"] = checked.to_dict()
|
scene["freshness_at_publication"] = checked.to_dict()
|
||||||
scene["stale_at_publication"] = any(item.state == "stale" for item in checked.layers)
|
scene["stale_at_publication"] = any(item.state == "stale" for item in checked.layers)
|
||||||
@@ -202,9 +216,14 @@ def assess_receipt(scene, freshness, *, bundle, now_ns):
|
|||||||
their original support time. Layer identity/source timestamps do not change;
|
their original support time. Layer identity/source timestamps do not change;
|
||||||
updated costmap/policy hashes belong to the derived view, not the wire bytes.
|
updated costmap/policy hashes belong to the derived view, not the wire bytes.
|
||||||
"""
|
"""
|
||||||
observed = bundle["time_ns"] + now_ns - bundle["due_ns"]
|
observed, uncertainty = source_observation(bundle, now_ns)
|
||||||
previous = scene.get("freshness_at_receipt") or scene["freshness_at_publication"]
|
previous = scene.get("freshness_at_receipt") or scene["freshness_at_publication"]
|
||||||
if observed < _read_wire_integer(previous["checked_at_source_time_ns"]):
|
# Independent clocks can have overlapping intervals even though receipt is
|
||||||
|
# later. Reject a provably reversed interval, not a shifted midpoint alone.
|
||||||
|
if observed + int(uncertainty * 1e6) < (
|
||||||
|
_read_wire_integer(previous["checked_at_source_time_ns"])
|
||||||
|
- int(previous["clock_uncertainty_ms"] * 1e6)
|
||||||
|
):
|
||||||
raise ValueError("consumer clock moved backwards")
|
raise ValueError("consumer clock moved backwards")
|
||||||
view = dict(scene)
|
view = dict(scene)
|
||||||
if scene.get("costmap_freshness_mode") == "per-cell":
|
if scene.get("costmap_freshness_mode") == "per-cell":
|
||||||
@@ -212,6 +231,7 @@ def assess_receipt(scene, freshness, *, bundle, now_ns):
|
|||||||
view,
|
view,
|
||||||
source_time_ns=freshness.source_time_ns,
|
source_time_ns=freshness.source_time_ns,
|
||||||
observed_source_time_ns=observed,
|
observed_source_time_ns=observed,
|
||||||
|
uncertainty_ms=uncertainty,
|
||||||
)
|
)
|
||||||
layers = {item.layer: item for item in freshness.layers[:4]}
|
layers = {item.layer: item for item in freshness.layers[:4]}
|
||||||
for name in ("costmap", "policy"):
|
for name in ("costmap", "policy"):
|
||||||
@@ -230,7 +250,7 @@ def assess_receipt(scene, freshness, *, bundle, now_ns):
|
|||||||
oldest_required_input_time_ns=oldest,
|
oldest_required_input_time_ns=oldest,
|
||||||
)
|
)
|
||||||
freshness = replace(freshness, layers=tuple(layers.values()))
|
freshness = replace(freshness, layers=tuple(layers.values()))
|
||||||
checked = assess(freshness, bundle=bundle, now_ns=now_ns)
|
checked = assess_observation(freshness, observed, uncertainty)
|
||||||
view = receipt_view(view, checked)
|
view = receipt_view(view, checked)
|
||||||
# Global suppression also changes the policy payload. A derived view must
|
# Global suppression also changes the policy payload. A derived view must
|
||||||
# remain internally verifiable, including on a second consumer boundary.
|
# remain internally verifiable, including on a second consumer boundary.
|
||||||
@@ -238,7 +258,7 @@ def assess_receipt(scene, freshness, *, bundle, now_ns):
|
|||||||
if policy.payload_sha256 is not None:
|
if policy.payload_sha256 is not None:
|
||||||
policy = replace(policy, payload_sha256=payload_digest(view, "policy"))
|
policy = replace(policy, payload_sha256=payload_digest(view, "policy"))
|
||||||
freshness = replace(freshness, layers=(*freshness.layers[:-1], policy))
|
freshness = replace(freshness, layers=(*freshness.layers[:-1], policy))
|
||||||
checked = assess(freshness, bundle=bundle, now_ns=now_ns)
|
checked = assess_observation(freshness, observed, uncertainty)
|
||||||
view["freshness"] = freshness.to_dict()
|
view["freshness"] = freshness.to_dict()
|
||||||
view["freshness_at_receipt"] = checked.to_dict()
|
view["freshness_at_receipt"] = checked.to_dict()
|
||||||
return view, checked
|
return view, checked
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ accepted from the network. Grant rotation stays in this trusted local adapter.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import secrets
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import traceback
|
import traceback
|
||||||
@@ -15,7 +16,9 @@ from pilot_binary_bridge import BinaryGraphInput
|
|||||||
from pilot_network_control import kernel_clock, read_control, write_control
|
from pilot_network_control import kernel_clock, read_control, write_control
|
||||||
|
|
||||||
from k1link.perception.streaming_continuity import StreamSuspended
|
from k1link.perception.streaming_continuity import StreamSuspended
|
||||||
|
from k1link.perception.streaming_control_grpc import ControlTicket, StreamControlEndpoint
|
||||||
from k1link.perception.streaming_grpc import GrpcStreamEndpoint
|
from k1link.perception.streaming_grpc import GrpcStreamEndpoint
|
||||||
|
from k1link.perception.streaming_source_clock import SourceClockMonitor
|
||||||
|
|
||||||
|
|
||||||
class NetworkGraphBridge(BinaryGraphInput):
|
class NetworkGraphBridge(BinaryGraphInput):
|
||||||
@@ -33,16 +36,40 @@ class NetworkGraphBridge(BinaryGraphInput):
|
|||||||
address,
|
address,
|
||||||
reset_temporal,
|
reset_temporal,
|
||||||
start_delay=2,
|
start_delay=2,
|
||||||
|
cross_host=False,
|
||||||
):
|
):
|
||||||
self.clock = kernel_clock()
|
self.cross_host = cross_host
|
||||||
source = SimpleNamespace(
|
self.clock_id = "worker-clock-" + secrets.token_hex(12)
|
||||||
source_zero=source_zero, wall_zero=time.monotonic_ns() + int(start_delay * 1e9)
|
self.clock = (
|
||||||
|
{"scope": "cross-host-conditional", "clock_id": self.clock_id}
|
||||||
|
if cross_host
|
||||||
|
else kernel_clock()
|
||||||
|
)
|
||||||
|
source = (
|
||||||
|
SourceClockMonitor(self.clock_id, source_zero)
|
||||||
|
if cross_host
|
||||||
|
else SimpleNamespace(
|
||||||
|
source_zero=source_zero, wall_zero=time.monotonic_ns() + int(start_delay * 1e9)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
super().__init__(runtime, decoder, source, report, reset_temporal=reset_temporal)
|
super().__init__(runtime, decoder, source, report, reset_temporal=reset_temporal)
|
||||||
self.control = Path(control)
|
self.control = Path(control)
|
||||||
self.source_status = Path(source_status)
|
self.source_status = Path(source_status)
|
||||||
self.certificate, self.private_key, self.address = certificate, private_key, address
|
self.certificate, self.private_key, self.address = certificate, private_key, address
|
||||||
self.endpoint = GrpcStreamEndpoint(runtime, self.consume, self.notice)
|
self.endpoint = GrpcStreamEndpoint(runtime, self.consume, self.notice)
|
||||||
|
self.access = None
|
||||||
|
self.ticket = ControlTicket(runtime.start, secrets.token_hex(32)) if cross_host else None
|
||||||
|
self.control_endpoint = (
|
||||||
|
StreamControlEndpoint(
|
||||||
|
self.ticket,
|
||||||
|
self._pending,
|
||||||
|
clock_id=self.clock_id,
|
||||||
|
observe=source.observe,
|
||||||
|
)
|
||||||
|
if cross_host
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
self.clock_states = []
|
||||||
self.stopping, self.ready = threading.Event(), threading.Event()
|
self.stopping, self.ready = threading.Event(), threading.Event()
|
||||||
self.failure = None
|
self.failure = None
|
||||||
self.published = []
|
self.published = []
|
||||||
@@ -64,6 +91,9 @@ class NetworkGraphBridge(BinaryGraphInput):
|
|||||||
|
|
||||||
def _grant(self):
|
def _grant(self):
|
||||||
access = self.endpoint.issue(self.epoch, "recorded-acquisition", 1)
|
access = self.endpoint.issue(self.epoch, "recorded-acquisition", 1)
|
||||||
|
if self.cross_host:
|
||||||
|
self.access = access
|
||||||
|
return
|
||||||
write_control(
|
write_control(
|
||||||
self.control / "grant.json",
|
self.control / "grant.json",
|
||||||
{
|
{
|
||||||
@@ -76,6 +106,13 @@ class NetworkGraphBridge(BinaryGraphInput):
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _pending(self):
|
||||||
|
try:
|
||||||
|
self.source.observed(time.monotonic_ns())
|
||||||
|
except StreamSuspended:
|
||||||
|
return None
|
||||||
|
return self.access if self.endpoint.grant is not None else None
|
||||||
|
|
||||||
def _run(self):
|
def _run(self):
|
||||||
try:
|
try:
|
||||||
asyncio.run(self._serve())
|
asyncio.run(self._serve())
|
||||||
@@ -90,21 +127,55 @@ class NetworkGraphBridge(BinaryGraphInput):
|
|||||||
self.address,
|
self.address,
|
||||||
certificate=Path(self.certificate).read_bytes(),
|
certificate=Path(self.certificate).read_bytes(),
|
||||||
private_key=Path(self.private_key).read_bytes(),
|
private_key=Path(self.private_key).read_bytes(),
|
||||||
|
control_handlers=(self.control_endpoint.handler(),) if self.cross_host else (),
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
self._grant()
|
if self.cross_host:
|
||||||
|
self.runtime.pause_input(self.epoch, "source-clock")
|
||||||
|
write_control(
|
||||||
|
self.control / "bootstrap.json",
|
||||||
|
{
|
||||||
|
"activation": self.runtime.start.to_dict(),
|
||||||
|
"token": self.ticket.token,
|
||||||
|
"remote_clock_id": self.clock_id,
|
||||||
|
"source_zero_ns": str(self.source_zero),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self._grant()
|
||||||
self.ready.set()
|
self.ready.set()
|
||||||
while not self.stopping.is_set() and not self.runtime.stop_event.is_set():
|
while not self.stopping.is_set() and not self.runtime.stop_event.is_set():
|
||||||
|
if self.cross_host:
|
||||||
|
try:
|
||||||
|
self.source.observed(time.monotonic_ns())
|
||||||
|
clock_ready = True
|
||||||
|
if not self.report["wall_zero_ns"]:
|
||||||
|
self.report["wall_zero_ns"] = self.source.due(
|
||||||
|
self.source_zero, time.monotonic_ns()
|
||||||
|
)[0]
|
||||||
|
except StreamSuspended:
|
||||||
|
clock_ready = False
|
||||||
|
self.runtime.pause_input(self.epoch, "source-clock")
|
||||||
|
if not self.clock_states or self.clock_states[-1]["ready"] != clock_ready:
|
||||||
|
self.clock_states.append(
|
||||||
|
{"at_ns": time.monotonic_ns(), "ready": clock_ready}
|
||||||
|
)
|
||||||
|
if len(self.clock_states) > 128:
|
||||||
|
raise ValueError("bounded clock transition count exceeded")
|
||||||
if self.endpoint.active is None:
|
if self.endpoint.active is None:
|
||||||
ended = self.source_status / "source-end.json"
|
ended = self.source_status / "source-end.json"
|
||||||
if ended.exists():
|
if self.cross_host and self.source.ended:
|
||||||
|
self.runtime.mailbox.finish()
|
||||||
|
elif not self.cross_host and ended.exists():
|
||||||
terminal = read_control(ended)
|
terminal = read_control(ended)
|
||||||
if terminal.get("run_id") != self.runtime.start.run_id:
|
if terminal.get("run_id") != self.runtime.start.run_id:
|
||||||
raise ValueError("external source completion binding mismatch")
|
raise ValueError("external source completion binding mismatch")
|
||||||
if terminal.get("error"):
|
if terminal.get("error"):
|
||||||
raise ValueError("external source failed; inspect its bounded report")
|
raise ValueError("external source failed; inspect its bounded report")
|
||||||
self.runtime.mailbox.finish()
|
self.runtime.mailbox.finish()
|
||||||
elif self.runtime.continuity.phase == "waiting":
|
elif self.runtime.continuity.phase == "waiting" and (
|
||||||
|
not self.cross_host or clock_ready
|
||||||
|
):
|
||||||
try:
|
try:
|
||||||
epoch = self.runtime.begin_input(self.runtime.start)
|
epoch = self.runtime.begin_input(self.runtime.start)
|
||||||
except StreamSuspended:
|
except StreamSuspended:
|
||||||
@@ -118,6 +189,8 @@ class NetworkGraphBridge(BinaryGraphInput):
|
|||||||
await asyncio.sleep(0.01)
|
await asyncio.sleep(0.01)
|
||||||
finally:
|
finally:
|
||||||
await server.stop(0)
|
await server.stop(0)
|
||||||
|
if self.cross_host:
|
||||||
|
(self.control / "bootstrap.json").unlink(missing_ok=True)
|
||||||
deadline = time.monotonic() + 2
|
deadline = time.monotonic() + 2
|
||||||
while self.endpoint.active is not None and time.monotonic() < deadline:
|
while self.endpoint.active is not None and time.monotonic() < deadline:
|
||||||
await asyncio.sleep(0.01)
|
await asyncio.sleep(0.01)
|
||||||
@@ -157,6 +230,7 @@ class NetworkGraphBridge(BinaryGraphInput):
|
|||||||
accepted_camera_sequences=self.accepted,
|
accepted_camera_sequences=self.accepted,
|
||||||
published=self.published,
|
published=self.published,
|
||||||
adapter_failure=self.failure,
|
adapter_failure=self.failure,
|
||||||
|
clock_states=self.clock_states,
|
||||||
)
|
)
|
||||||
if not self.thread.is_alive() and self.endpoint.active is None:
|
if not self.thread.is_alive() and self.endpoint.active is None:
|
||||||
self.window.close()
|
self.window.close()
|
||||||
|
|||||||
@@ -18,8 +18,9 @@ from pathlib import Path
|
|||||||
import grpc
|
import grpc
|
||||||
from pilot_binary_source import event_bytes, input_gaps, read_member
|
from pilot_binary_source import event_bytes, input_gaps, read_member
|
||||||
from pilot_freshness import assess_receipt, validate_receipt
|
from pilot_freshness import assess_receipt, validate_receipt
|
||||||
from pilot_network_control import read_grant, write_control
|
from pilot_network_control import read_control, read_grant, write_control
|
||||||
from pilot_source import SensorArchive, camera_events, merged_events
|
from pilot_source import SensorArchive, camera_events, merged_events
|
||||||
|
from pilot_source_control import SourceControl
|
||||||
|
|
||||||
from k1link.compute.live_perception import LiveIngressEvent
|
from k1link.compute.live_perception import LiveIngressEvent
|
||||||
from k1link.perception.streaming_grpc import GrpcStreamClient
|
from k1link.perception.streaming_grpc import GrpcStreamClient
|
||||||
@@ -32,6 +33,16 @@ def connection_delay(wall_zero_ns, now_ns):
|
|||||||
return max(0, (wall_zero_ns - now_ns - 100_000_000) / 1e9)
|
return max(0, (wall_zero_ns - now_ns - 100_000_000) / 1e9)
|
||||||
|
|
||||||
|
|
||||||
|
async def finish_source(client, reader, clock_control):
|
||||||
|
# EOF on Exchange can finish the graph and retire its control endpoint.
|
||||||
|
# Confirm source completion first while that data stream is still alive.
|
||||||
|
if clock_control:
|
||||||
|
await clock_control.finish()
|
||||||
|
if client is not None:
|
||||||
|
await client.end()
|
||||||
|
await asyncio.wait_for(reader, timeout=5)
|
||||||
|
|
||||||
|
|
||||||
async def run(args):
|
async def run(args):
|
||||||
report = {
|
report = {
|
||||||
"started_utc": datetime.now(UTC).isoformat(),
|
"started_utc": datetime.now(UTC).isoformat(),
|
||||||
@@ -43,7 +54,9 @@ async def run(args):
|
|||||||
"models": 0,
|
"models": 0,
|
||||||
"source_clock_speed": 1.0,
|
"source_clock_speed": 1.0,
|
||||||
"full_source_prepass": False,
|
"full_source_prepass": False,
|
||||||
"clock_scope": "two-containers-same-unshifted-Linux-kernel",
|
"clock_scope": "cross-host-conditional"
|
||||||
|
if args.cross_host
|
||||||
|
else "two-containers-same-unshifted-Linux-kernel",
|
||||||
"error": None,
|
"error": None,
|
||||||
"actuation_allowed": False,
|
"actuation_allowed": False,
|
||||||
"commands_enabled": False,
|
"commands_enabled": False,
|
||||||
@@ -59,6 +72,7 @@ async def run(args):
|
|||||||
gaps, next_gap, outage_until = input_gaps(args.input_gap), 0, 0
|
gaps, next_gap, outage_until = input_gaps(args.input_gap), 0, 0
|
||||||
previous_epoch = None
|
previous_epoch = None
|
||||||
clock = None
|
clock = None
|
||||||
|
clock_control = None
|
||||||
|
|
||||||
async def disconnect():
|
async def disconnect():
|
||||||
nonlocal client, reader
|
nonlocal client, reader
|
||||||
@@ -71,11 +85,25 @@ async def run(args):
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
deadline = time.monotonic() + 120
|
deadline = time.monotonic() + 120
|
||||||
while not grant_path.exists():
|
startup_path = control / "bootstrap.json" if args.cross_host else grant_path
|
||||||
|
while not startup_path.exists():
|
||||||
if time.monotonic() > deadline:
|
if time.monotonic() > deadline:
|
||||||
raise TimeoutError("trusted graph grant was not issued after warmup")
|
raise TimeoutError("trusted graph grant was not issued after warmup")
|
||||||
await asyncio.sleep(0.02)
|
await asyncio.sleep(0.02)
|
||||||
clock, _ = read_grant(grant_path)
|
if args.cross_host:
|
||||||
|
bootstrap = read_control(startup_path)
|
||||||
|
clock_control = SourceControl(
|
||||||
|
args.target, Path(args.certificate).read_bytes(), bootstrap
|
||||||
|
)
|
||||||
|
clock_control.start()
|
||||||
|
anchor = await clock_control.wait_anchor()
|
||||||
|
clock = {
|
||||||
|
"source_zero_ns": anchor.source_zero_ns,
|
||||||
|
"wall_zero_ns": anchor.local_zero_ns,
|
||||||
|
"clock": {"scope": "source-and-consumer-same-Mac-clock"},
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
clock, _ = read_grant(grant_path)
|
||||||
first = next(camera_events(args.camera_index, 1)).time_ns
|
first = next(camera_events(args.camera_index, 1)).time_ns
|
||||||
if clock["source_zero_ns"] != first - 500_000_000:
|
if clock["source_zero_ns"] != first - 500_000_000:
|
||||||
raise ValueError("recording prefix does not match admitted source clock")
|
raise ValueError("recording prefix does not match admitted source clock")
|
||||||
@@ -104,6 +132,20 @@ async def run(args):
|
|||||||
"due_ns": clock["wall_zero_ns"] + source_stamp - clock["source_zero_ns"],
|
"due_ns": clock["wall_zero_ns"] + source_stamp - clock["source_zero_ns"],
|
||||||
}
|
}
|
||||||
freshness = validate_receipt(scene, bundle, epoch_id=epoch.epoch_id)
|
freshness = validate_receipt(scene, bundle, epoch_id=epoch.epoch_id)
|
||||||
|
if clock_control and not clock_control.ready():
|
||||||
|
rejected = report.setdefault("clock_rejected_results", [])
|
||||||
|
if len(rejected) >= args.frames:
|
||||||
|
raise ValueError("bounded rejected receipt ledger exceeded")
|
||||||
|
rejected.append(
|
||||||
|
{
|
||||||
|
"sequence": sequence,
|
||||||
|
"epoch_id": epoch.epoch_id,
|
||||||
|
"payload_sha256": hashlib.sha256(payload).hexdigest(),
|
||||||
|
"reason": "source-clock-unavailable",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
# No fresh scene/policy authority while clock admission is lost.
|
||||||
|
continue
|
||||||
checked_at = time.monotonic_ns()
|
checked_at = time.monotonic_ns()
|
||||||
view, checked = assess_receipt(
|
view, checked = assess_receipt(
|
||||||
scene, freshness, bundle=bundle, now_ns=checked_at
|
scene, freshness, bundle=bundle, now_ns=checked_at
|
||||||
@@ -140,7 +182,18 @@ async def run(args):
|
|||||||
|
|
||||||
async def connect():
|
async def connect():
|
||||||
nonlocal client, reader, access, previous_epoch
|
nonlocal client, reader, access, previous_epoch
|
||||||
value, candidate = read_grant(grant_path)
|
if clock_control:
|
||||||
|
if not clock_control.ready() or clock_control.access is None:
|
||||||
|
return False
|
||||||
|
candidate = clock_control.access
|
||||||
|
value = {
|
||||||
|
**clock,
|
||||||
|
"cutoff_ns": clock["source_zero_ns"]
|
||||||
|
+ time.monotonic_ns()
|
||||||
|
- clock["wall_zero_ns"],
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
value, candidate = read_grant(grant_path)
|
||||||
if candidate.epoch.epoch_id == previous_epoch:
|
if candidate.epoch.epoch_id == previous_epoch:
|
||||||
return False
|
return False
|
||||||
if any(
|
if any(
|
||||||
@@ -185,6 +238,8 @@ async def run(args):
|
|||||||
if event.channel == "camera":
|
if event.channel == "camera":
|
||||||
released.append(event.sequence)
|
released.append(event.sequence)
|
||||||
try:
|
try:
|
||||||
|
if clock_control and not clock_control.ready():
|
||||||
|
await disconnect()
|
||||||
if reader is not None and reader.done():
|
if reader is not None and reader.done():
|
||||||
reader.result() # Invalid payload is fatal; transport interruption is not.
|
reader.result() # Invalid payload is fatal; transport interruption is not.
|
||||||
raise ConnectionError("result stream ended before source EOF")
|
raise ConnectionError("result stream ended before source EOF")
|
||||||
@@ -240,20 +295,23 @@ async def run(args):
|
|||||||
break
|
break
|
||||||
source_end = True
|
source_end = True
|
||||||
report["end_sent_monotonic_ns"] = time.monotonic_ns()
|
report["end_sent_monotonic_ns"] = time.monotonic_ns()
|
||||||
if client is not None:
|
await finish_source(client, reader, clock_control)
|
||||||
await client.end()
|
if not clock_control:
|
||||||
await asyncio.wait_for(reader, timeout=5)
|
write_control(
|
||||||
write_control(
|
status / "source-end.json", {"run_id": access.epoch.run_id, "error": False}
|
||||||
status / "source-end.json", {"run_id": access.epoch.run_id, "error": False}
|
)
|
||||||
)
|
|
||||||
except Exception:
|
except Exception:
|
||||||
report["error"] = traceback.format_exc()
|
report["error"] = traceback.format_exc()
|
||||||
if access is not None:
|
if access is not None and not args.cross_host:
|
||||||
write_control(
|
write_control(
|
||||||
status / "source-end.json", {"run_id": access.epoch.run_id, "error": True}
|
status / "source-end.json", {"run_id": access.epoch.run_id, "error": True}
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
await disconnect()
|
await disconnect()
|
||||||
|
if clock_control:
|
||||||
|
await clock_control.close()
|
||||||
|
report["clock_samples"] = clock_control.samples
|
||||||
|
report["clock_errors"] = clock_control.errors
|
||||||
from run_joint_pilot import distribution
|
from run_joint_pilot import distribution
|
||||||
|
|
||||||
report.update(
|
report.update(
|
||||||
@@ -291,6 +349,7 @@ async def run(args):
|
|||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument("--target", required=True)
|
parser.add_argument("--target", required=True)
|
||||||
|
parser.add_argument("--cross-host", action="store_true")
|
||||||
parser.add_argument("--control", required=True)
|
parser.add_argument("--control", required=True)
|
||||||
parser.add_argument("--source-status", required=True)
|
parser.add_argument("--source-status", required=True)
|
||||||
parser.add_argument("--certificate", required=True)
|
parser.add_argument("--certificate", required=True)
|
||||||
|
|||||||
@@ -166,6 +166,8 @@ class PilotController:
|
|||||||
def source_now(self):
|
def source_now(self):
|
||||||
if self.source_clock is None:
|
if self.source_clock is None:
|
||||||
raise WorkerLeaseError("source clock mapping not configured")
|
raise WorkerLeaseError("source clock mapping not configured")
|
||||||
|
if hasattr(self.source_clock, "source_now"):
|
||||||
|
return self.source_clock.source_now(time.monotonic_ns())
|
||||||
return self.source_clock.source_zero + time.monotonic_ns() - self.source_clock.wall_zero
|
return self.source_clock.source_zero + time.monotonic_ns() - self.source_clock.wall_zero
|
||||||
|
|
||||||
def _renew(self):
|
def _renew(self):
|
||||||
|
|||||||
@@ -0,0 +1,113 @@
|
|||||||
|
"""Bounded application-side clock loop; no lease or model control."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import secrets
|
||||||
|
import time
|
||||||
|
|
||||||
|
import grpc
|
||||||
|
|
||||||
|
from k1link.perception.realtime_contract import StreamStart
|
||||||
|
from k1link.perception.streaming_clock import ClockMappingError, ClockWindow
|
||||||
|
from k1link.perception.streaming_control_grpc import ControlTicket, StreamControlClient
|
||||||
|
from k1link.perception.streaming_source_clock import SourceAnchor
|
||||||
|
|
||||||
|
|
||||||
|
class SourceControl:
|
||||||
|
def __init__(self, target, roots, bootstrap):
|
||||||
|
ticket = ControlTicket(StreamStart.from_dict(bootstrap["activation"]), bootstrap["token"])
|
||||||
|
clock_id = "source-clock-" + secrets.token_hex(12)
|
||||||
|
self.client = StreamControlClient(target, roots, ticket, clock_id=clock_id)
|
||||||
|
self.window = ClockWindow(
|
||||||
|
clock_id, bootstrap["remote_clock_id"], rate_ppm=500, timestamp_error_ns=50_000
|
||||||
|
)
|
||||||
|
self.source_zero_ns = int(bootstrap["source_zero_ns"])
|
||||||
|
self.anchor = None # Latch once after the initial clock handshake, never on resume.
|
||||||
|
self.access = None
|
||||||
|
self.confirmed = False
|
||||||
|
self.failure = None
|
||||||
|
self.stopping = self.ending = False
|
||||||
|
self.samples, self.errors = [], []
|
||||||
|
self.task = None
|
||||||
|
|
||||||
|
async def update(self):
|
||||||
|
probe, access = await self.client.poll()
|
||||||
|
bounds = self.window.add(probe)
|
||||||
|
now = time.monotonic_ns()
|
||||||
|
if self.anchor is None:
|
||||||
|
try:
|
||||||
|
self.window.current(now).require(now)
|
||||||
|
except ClockMappingError:
|
||||||
|
pass # Pre-start clock warmup, not a running source pause or retimestamp.
|
||||||
|
else:
|
||||||
|
self.anchor = SourceAnchor(self.source_zero_ns, now + 2_000_000_000)
|
||||||
|
if self.anchor is not None:
|
||||||
|
await self.client.acknowledge(probe, self.anchor, ended=self.ending)
|
||||||
|
self.confirmed = True
|
||||||
|
self.access = access
|
||||||
|
now = time.monotonic_ns()
|
||||||
|
self.samples.append(
|
||||||
|
{
|
||||||
|
"at_ns": now,
|
||||||
|
"bounds": bounds.to_dict(),
|
||||||
|
"uncertainty_ns": bounds.uncertainty_ns(now),
|
||||||
|
"ready": self.ready(),
|
||||||
|
"acknowledged": self.anchor is not None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if len(self.samples) > 2048:
|
||||||
|
raise ValueError("bounded clock diagnostic count exceeded")
|
||||||
|
|
||||||
|
def ready(self):
|
||||||
|
if self.failure:
|
||||||
|
raise RuntimeError("source clock task failed") from self.failure
|
||||||
|
if not self.confirmed:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
now = time.monotonic_ns()
|
||||||
|
self.window.current(now).require(now)
|
||||||
|
except ClockMappingError:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def loop(self):
|
||||||
|
while not self.stopping:
|
||||||
|
try:
|
||||||
|
await self.update()
|
||||||
|
if self.ending:
|
||||||
|
return
|
||||||
|
except (grpc.RpcError, TimeoutError, OSError) as exc:
|
||||||
|
self.confirmed = False
|
||||||
|
self.errors.append({"at_ns": time.monotonic_ns(), "type": type(exc).__name__})
|
||||||
|
if len(self.errors) > 128:
|
||||||
|
self.failure = RuntimeError("bounded clock transport error count exceeded")
|
||||||
|
return
|
||||||
|
except Exception as exc:
|
||||||
|
self.failure = exc
|
||||||
|
return
|
||||||
|
await asyncio.sleep(0.05)
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
self.task = asyncio.create_task(self.loop())
|
||||||
|
|
||||||
|
async def wait_anchor(self):
|
||||||
|
deadline = time.monotonic() + 10
|
||||||
|
while self.anchor is None or not self.confirmed:
|
||||||
|
if self.failure:
|
||||||
|
raise RuntimeError("initial source clock handshake failed") from self.failure
|
||||||
|
if time.monotonic() > deadline:
|
||||||
|
raise TimeoutError("bounded initial source clock handshake")
|
||||||
|
await asyncio.sleep(0.01)
|
||||||
|
return self.anchor
|
||||||
|
|
||||||
|
async def finish(self):
|
||||||
|
self.ending = True
|
||||||
|
await asyncio.wait_for(asyncio.shield(self.task), timeout=2)
|
||||||
|
if self.failure:
|
||||||
|
raise RuntimeError("source clock final acknowledgement failed") from self.failure
|
||||||
|
|
||||||
|
async def close(self):
|
||||||
|
self.stopping = True
|
||||||
|
if self.task:
|
||||||
|
self.task.cancel()
|
||||||
|
await asyncio.gather(self.task, return_exceptions=True)
|
||||||
|
await self.client.close()
|
||||||
@@ -276,6 +276,17 @@ def run(args):
|
|||||||
"costmap_freshness_mode": args.costmap_freshness,
|
"costmap_freshness_mode": args.costmap_freshness,
|
||||||
"ddrnet_min_source_interval_ms": args.ddrnet_min_source_interval_ms,
|
"ddrnet_min_source_interval_ms": args.ddrnet_min_source_interval_ms,
|
||||||
"input_transport": args.input_transport,
|
"input_transport": args.input_transport,
|
||||||
|
"network_clock_mode": "acknowledged-monotonic/v1"
|
||||||
|
if getattr(args, "network_cross_host", False)
|
||||||
|
else "local-only",
|
||||||
|
"network_clock_envelope": {
|
||||||
|
"relative_rate_ppm": 500,
|
||||||
|
"timestamp_error_ns": 50000,
|
||||||
|
"maximum_age_ns": 2000000000,
|
||||||
|
"maximum_uncertainty_ns": 5000000,
|
||||||
|
}
|
||||||
|
if getattr(args, "network_cross_host", False)
|
||||||
|
else None,
|
||||||
"recover_input": getattr(args, "recover_input", False),
|
"recover_input": getattr(args, "recover_input", False),
|
||||||
"input_gap_plan": getattr(args, "input_gap", []),
|
"input_gap_plan": getattr(args, "input_gap", []),
|
||||||
}
|
}
|
||||||
@@ -549,6 +560,7 @@ def run(args):
|
|||||||
private_key=args.network_private_key,
|
private_key=args.network_private_key,
|
||||||
address=args.network_address,
|
address=args.network_address,
|
||||||
reset_temporal=reset_temporal,
|
reset_temporal=reset_temporal,
|
||||||
|
cross_host=args.network_cross_host,
|
||||||
)
|
)
|
||||||
controller.source_clock = binary_bridge.source
|
controller.source_clock = binary_bridge.source
|
||||||
report["scope"] = "full-graph-network-candidate; receiver evidence is separate"
|
report["scope"] = "full-graph-network-candidate; receiver evidence is separate"
|
||||||
@@ -710,7 +722,10 @@ def run(args):
|
|||||||
"scene_sha256": hashlib.sha256(encoded).hexdigest(),
|
"scene_sha256": hashlib.sha256(encoded).hexdigest(),
|
||||||
}
|
}
|
||||||
results.append(result)
|
results.append(result)
|
||||||
except StreamSuspended:
|
except StreamSuspended as exc:
|
||||||
|
report.setdefault("suspended_results", []).append(
|
||||||
|
{"sequence": bundle["sequence"], "reason": str(exc)}
|
||||||
|
)
|
||||||
mailbox.release(bundle, discard_reason="input-gap")
|
mailbox.release(bundle, discard_reason="input-gap")
|
||||||
cpu_bundle = bundle = computed = mask = proposals = scene = received = view = (
|
cpu_bundle = bundle = computed = mask = proposals = scene = received = view = (
|
||||||
None
|
None
|
||||||
@@ -947,6 +962,7 @@ if __name__ == "__main__":
|
|||||||
parser.add_argument("--network-certificate")
|
parser.add_argument("--network-certificate")
|
||||||
parser.add_argument("--network-private-key")
|
parser.add_argument("--network-private-key")
|
||||||
parser.add_argument("--network-address", default="[::]:50061")
|
parser.add_argument("--network-address", default="[::]:50061")
|
||||||
|
parser.add_argument("--network-cross-host", action="store_true")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--ddrnet-layout", choices=("reference", "channels-last"), default="reference"
|
"--ddrnet-layout", choices=("reference", "channels-last"), default="reference"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -69,6 +69,60 @@ class ClockProbe:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ClockReceipt:
|
||||||
|
"""The responder's view AFTER an authenticated echo of its issued probe.
|
||||||
|
|
||||||
|
Responder receive=t2 and send=t3 bracket only processing, NOT network RTT.
|
||||||
|
t1-t2 is a lower bound and t4-t3 an upper bound for initiator-minus-responder.
|
||||||
|
Never construct an ordinary reversed ClockProbe: that would invert causality.
|
||||||
|
"""
|
||||||
|
|
||||||
|
probe: ClockProbe
|
||||||
|
acknowledged_ns: int
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_stamp(self.acknowledged_ns)
|
||||||
|
if (
|
||||||
|
not self.probe.remote_send_ns
|
||||||
|
<= self.acknowledged_ns
|
||||||
|
<= (self.probe.remote_receive_ns + 500_000_000)
|
||||||
|
):
|
||||||
|
raise ClockMappingError("clock acknowledgement outside bounded round trip")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def local_clock_id(self) -> str:
|
||||||
|
return self.probe.remote_clock_id
|
||||||
|
|
||||||
|
@property
|
||||||
|
def remote_clock_id(self) -> str:
|
||||||
|
return self.probe.local_clock_id
|
||||||
|
|
||||||
|
@property
|
||||||
|
def nonce(self) -> str:
|
||||||
|
return self.probe.nonce
|
||||||
|
|
||||||
|
@property
|
||||||
|
def local_send_ns(self) -> int:
|
||||||
|
return self.probe.remote_receive_ns
|
||||||
|
|
||||||
|
@property
|
||||||
|
def local_receive_ns(self) -> int:
|
||||||
|
return self.acknowledged_ns
|
||||||
|
|
||||||
|
def bounds_at(
|
||||||
|
self, local_ns: int, *, rate_ppm: int, timestamp_error_ns: int
|
||||||
|
) -> tuple[int, int]:
|
||||||
|
_stamp(local_ns)
|
||||||
|
if local_ns < self.acknowledged_ns:
|
||||||
|
raise ClockMappingError("acknowledgement is from the future")
|
||||||
|
widen = _drift(local_ns - self.probe.remote_receive_ns, rate_ppm) + 2 * timestamp_error_ns
|
||||||
|
return (
|
||||||
|
self.probe.local_send_ns - self.probe.remote_receive_ns - widen,
|
||||||
|
self.probe.local_receive_ns - self.probe.remote_send_ns + widen,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class ClockBounds:
|
class ClockBounds:
|
||||||
local_clock_id: str
|
local_clock_id: str
|
||||||
@@ -161,11 +215,11 @@ class ClockWindow:
|
|||||||
timestamp_error_ns,
|
timestamp_error_ns,
|
||||||
maximum_age_ns,
|
maximum_age_ns,
|
||||||
)
|
)
|
||||||
self.samples: deque[ClockProbe] = deque(maxlen=16)
|
self.samples: deque[ClockProbe | ClockReceipt] = deque(maxlen=16)
|
||||||
self.failed = False
|
self.failed = False
|
||||||
self.last_receive_ns = -1
|
self.last_receive_ns = -1
|
||||||
|
|
||||||
def add(self, sample: ClockProbe) -> ClockBounds:
|
def add(self, sample: ClockProbe | ClockReceipt) -> ClockBounds:
|
||||||
if self.failed:
|
if self.failed:
|
||||||
raise ClockMappingError("clock session quarantined; explicit new session required")
|
raise ClockMappingError("clock session quarantined; explicit new session required")
|
||||||
if (
|
if (
|
||||||
|
|||||||
@@ -62,7 +62,13 @@ class InputContinuity:
|
|||||||
raise StreamSuspended("input is waiting for resynchronization")
|
raise StreamSuspended("input is waiting for resynchronization")
|
||||||
|
|
||||||
def pause(self, reason: str) -> None:
|
def pause(self, reason: str) -> None:
|
||||||
if reason not in ("input-disconnected", "input-timeout", "source-gap", "worker-telemetry"):
|
if reason not in (
|
||||||
|
"input-disconnected",
|
||||||
|
"input-timeout",
|
||||||
|
"source-gap",
|
||||||
|
"worker-telemetry",
|
||||||
|
"source-clock",
|
||||||
|
):
|
||||||
raise ValueError("unknown recoverable pause reason")
|
raise ValueError("unknown recoverable pause reason")
|
||||||
if self.phase != "waiting":
|
if self.phase != "waiting":
|
||||||
self.pauses += 1
|
self.pauses += 1
|
||||||
|
|||||||
@@ -23,10 +23,12 @@ from . import streaming_wire as wire
|
|||||||
from .realtime_contract import StreamStart
|
from .realtime_contract import StreamStart
|
||||||
from .streaming_clock import ClockProbe
|
from .streaming_clock import ClockProbe
|
||||||
from .streaming_grpc import OPTIONS, StreamAccess
|
from .streaming_grpc import OPTIONS, StreamAccess
|
||||||
|
from .streaming_source_clock import SourceAnchor
|
||||||
|
|
||||||
SERVICE = "missioncore.perception.v1.StreamControl"
|
SERVICE = "missioncore.perception.v1.StreamControl"
|
||||||
METHOD = f"/{SERVICE}/Poll"
|
METHOD = f"/{SERVICE}/Poll"
|
||||||
MAX_CONTROL = 8192
|
MAX_CONTROL = 8192
|
||||||
|
REPORT_METHOD = f"/{SERVICE}/ReportClock"
|
||||||
|
|
||||||
|
|
||||||
def _document(raw: bytes) -> dict[str, Any]:
|
def _document(raw: bytes) -> dict[str, Any]:
|
||||||
@@ -77,6 +79,7 @@ class StreamControlEndpoint:
|
|||||||
*,
|
*,
|
||||||
clock_id: str,
|
clock_id: str,
|
||||||
clock_ns: Callable[[], int] = time.monotonic_ns,
|
clock_ns: Callable[[], int] = time.monotonic_ns,
|
||||||
|
observe: Callable[[ClockProbe, int, SourceAnchor, bool], None] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
wire.identifier(clock_id)
|
wire.identifier(clock_id)
|
||||||
self.activation, self.pending, self.clock_id, self.clock_ns = (
|
self.activation, self.pending, self.clock_id, self.clock_ns = (
|
||||||
@@ -89,10 +92,16 @@ class StreamControlEndpoint:
|
|||||||
self._lock = threading.Lock()
|
self._lock = threading.Lock()
|
||||||
self._last_poll_ns = -1
|
self._last_poll_ns = -1
|
||||||
self.accepted = self.rejected = 0
|
self.accepted = self.rejected = 0
|
||||||
|
self.observe = observe
|
||||||
|
self._challenge: dict[str, Any] | None = None
|
||||||
|
|
||||||
def handler(self) -> Any:
|
def handler(self) -> Any:
|
||||||
return grpc.method_handlers_generic_handler(
|
return grpc.method_handlers_generic_handler(
|
||||||
SERVICE, {"Poll": grpc.unary_unary_rpc_method_handler(self.poll)}
|
SERVICE,
|
||||||
|
{
|
||||||
|
"Poll": grpc.unary_unary_rpc_method_handler(self.poll),
|
||||||
|
"ReportClock": grpc.unary_unary_rpc_method_handler(self.report_clock),
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
def _authenticate(self, context: Any) -> None:
|
def _authenticate(self, context: Any) -> None:
|
||||||
@@ -163,6 +172,19 @@ class StreamControlEndpoint:
|
|||||||
encoded = wire.canonical(response)
|
encoded = wire.canonical(response)
|
||||||
if len(encoded) > MAX_CONTROL:
|
if len(encoded) > MAX_CONTROL:
|
||||||
raise ValueError("trusted control snapshot exceeds bound")
|
raise ValueError("trusted control snapshot exceeds bound")
|
||||||
|
if self.observe is not None:
|
||||||
|
with self._lock:
|
||||||
|
self._challenge = {
|
||||||
|
key: response[key]
|
||||||
|
for key in (
|
||||||
|
"nonce",
|
||||||
|
"local_clock_id",
|
||||||
|
"local_send_ns",
|
||||||
|
"remote_clock_id",
|
||||||
|
"remote_receive_ns",
|
||||||
|
"remote_send_ns",
|
||||||
|
)
|
||||||
|
}
|
||||||
except (ValueError, RuntimeError):
|
except (ValueError, RuntimeError):
|
||||||
await context.abort(
|
await context.abort(
|
||||||
grpc.StatusCode.UNAVAILABLE, "local controller has no current offer"
|
grpc.StatusCode.UNAVAILABLE, "local controller has no current offer"
|
||||||
@@ -171,6 +193,52 @@ class StreamControlEndpoint:
|
|||||||
self.accepted += 1
|
self.accepted += 1
|
||||||
return encoded
|
return encoded
|
||||||
|
|
||||||
|
async def report_clock(self, raw: bytes, context: Any) -> bytes:
|
||||||
|
received = self.clock_ns()
|
||||||
|
try:
|
||||||
|
self._authenticate(context)
|
||||||
|
value = _document(raw)
|
||||||
|
if set(value) != {"probe", "anchor", "ended"} or self.observe is None:
|
||||||
|
raise ValueError("clock receipt not admitted")
|
||||||
|
evidence = value["probe"]
|
||||||
|
with self._lock:
|
||||||
|
challenge = self._challenge
|
||||||
|
if (
|
||||||
|
challenge is None
|
||||||
|
or not isinstance(evidence, dict)
|
||||||
|
or set(evidence) != {*challenge, "local_receive_ns"}
|
||||||
|
or any(evidence[k] != v for k, v in challenge.items())
|
||||||
|
):
|
||||||
|
raise ValueError("clock receipt is not the current issued challenge")
|
||||||
|
self._challenge = None # Single-use, including malformed/late receipts.
|
||||||
|
probe = ClockProbe(
|
||||||
|
evidence["local_clock_id"],
|
||||||
|
evidence["remote_clock_id"],
|
||||||
|
evidence["nonce"],
|
||||||
|
*(
|
||||||
|
wire.uint64(evidence[k])
|
||||||
|
for k in (
|
||||||
|
"local_send_ns",
|
||||||
|
"remote_receive_ns",
|
||||||
|
"remote_send_ns",
|
||||||
|
"local_receive_ns",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
anchor = SourceAnchor.from_dict(value["anchor"])
|
||||||
|
self.observe(probe, received, anchor, value["ended"])
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
self.rejected += 1
|
||||||
|
await context.abort(grpc.StatusCode.INVALID_ARGUMENT, "clock receipt rejected")
|
||||||
|
return b""
|
||||||
|
return wire.canonical(
|
||||||
|
{
|
||||||
|
"schema_version": "missioncore.stream-clock-receipt/v1",
|
||||||
|
"activation_binding": wire.binding(self.activation),
|
||||||
|
"nonce": probe.nonce,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class StreamControlClient:
|
class StreamControlClient:
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -196,6 +264,7 @@ class StreamControlClient:
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
self.call = self.channel.unary_unary(METHOD)
|
self.call = self.channel.unary_unary(METHOD)
|
||||||
|
self.report_call = self.channel.unary_unary(REPORT_METHOD)
|
||||||
self.polling = False
|
self.polling = False
|
||||||
|
|
||||||
async def poll(self) -> tuple[ClockProbe, StreamAccess | None]:
|
async def poll(self) -> tuple[ClockProbe, StreamAccess | None]:
|
||||||
@@ -267,3 +336,46 @@ class StreamControlClient:
|
|||||||
|
|
||||||
async def close(self) -> None:
|
async def close(self) -> None:
|
||||||
await self.channel.close()
|
await self.channel.close()
|
||||||
|
|
||||||
|
async def acknowledge(
|
||||||
|
self, probe: ClockProbe, anchor: SourceAnchor, *, ended: bool = False
|
||||||
|
) -> None:
|
||||||
|
if self.polling or probe.local_clock_id != self.clock_id or type(ended) is not bool:
|
||||||
|
raise ValueError("one scoped control request at a time")
|
||||||
|
self.polling = True
|
||||||
|
try:
|
||||||
|
evidence = {
|
||||||
|
name: getattr(probe, name)
|
||||||
|
for name in (
|
||||||
|
"nonce",
|
||||||
|
"local_clock_id",
|
||||||
|
"remote_clock_id",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
evidence.update(
|
||||||
|
{
|
||||||
|
name: str(getattr(probe, name))
|
||||||
|
for name in (
|
||||||
|
"local_send_ns",
|
||||||
|
"remote_receive_ns",
|
||||||
|
"remote_send_ns",
|
||||||
|
"local_receive_ns",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
response = _document(
|
||||||
|
await self.report_call(
|
||||||
|
wire.canonical({"probe": evidence, "anchor": anchor.to_dict(), "ended": ended}),
|
||||||
|
metadata=self.ticket.metadata(),
|
||||||
|
timeout=0.5,
|
||||||
|
wait_for_ready=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if response != {
|
||||||
|
"schema_version": "missioncore.stream-clock-receipt/v1",
|
||||||
|
"activation_binding": wire.binding(self.ticket.activation),
|
||||||
|
"nonce": probe.nonce,
|
||||||
|
}:
|
||||||
|
raise ValueError("clock receipt response binding mismatch")
|
||||||
|
finally:
|
||||||
|
self.polling = False
|
||||||
|
|||||||
@@ -291,22 +291,23 @@ class StreamingIngress:
|
|||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self.error = str(exc)
|
self.error = str(exc)
|
||||||
self.terminal = "failed"
|
self.terminal = "failed"
|
||||||
if self.opened:
|
if self.runtime.continuity is not None and isinstance(
|
||||||
if self.runtime.continuity is not None and isinstance(
|
exc, (InputInterrupted, StreamSuspended)
|
||||||
exc, (InputInterrupted, StreamSuspended)
|
):
|
||||||
):
|
# Admission already consumed the single-use grant. A transport
|
||||||
self.terminal = "paused"
|
# loss before the application OPEN is still WAIT, not an active
|
||||||
# A late old socket cannot pause a replacement connection.
|
# owner stranded forever. A late old socket remains fenced.
|
||||||
with suppress(RuntimeError): # Already replaced, stopping, or fenced.
|
self.terminal = "paused"
|
||||||
self.runtime.pause_input(
|
with suppress(RuntimeError): # Already replaced, stopping, or fenced.
|
||||||
self.input_epoch,
|
self.runtime.pause_input(
|
||||||
"input-disconnected"
|
self.input_epoch,
|
||||||
if isinstance(exc, InputInterrupted)
|
"input-disconnected"
|
||||||
else "source-gap",
|
if isinstance(exc, InputInterrupted)
|
||||||
)
|
else "source-gap",
|
||||||
else:
|
)
|
||||||
self.runtime.mailbox.finish(self.error)
|
elif self.opened:
|
||||||
self.runtime.request_stop("failed")
|
self.runtime.mailbox.finish(self.error)
|
||||||
|
self.runtime.request_stop("failed")
|
||||||
finally:
|
finally:
|
||||||
self.connection.close()
|
self.connection.close()
|
||||||
if reservation is not None:
|
if reservation is not None:
|
||||||
|
|||||||
@@ -223,11 +223,17 @@ class StreamingLifecycle:
|
|||||||
return
|
return
|
||||||
self._check(self.start)
|
self._check(self.start)
|
||||||
self.continuity.check(epoch, synchronizing=synchronizing)
|
self.continuity.check(epoch, synchronizing=synchronizing)
|
||||||
if event_ns is not None:
|
assert self._source_clock_ns is not None
|
||||||
assert self._source_clock_ns is not None
|
try:
|
||||||
now = self._source_clock_ns()
|
now = self._source_clock_ns()
|
||||||
if event_ns < self.continuity.cutoff_ns or not 0 <= now - event_ns <= 250_000_000:
|
except StreamSuspended:
|
||||||
raise StreamSuspended("obsolete or future input, no backlog replay")
|
self.continuity.pause("source-clock")
|
||||||
|
self.mailbox.pause()
|
||||||
|
raise
|
||||||
|
if event_ns is not None and (
|
||||||
|
event_ns < self.continuity.cutoff_ns or not 0 <= now - event_ns <= 250_000_000
|
||||||
|
):
|
||||||
|
raise StreamSuspended("obsolete or future input, no backlog replay")
|
||||||
|
|
||||||
def pause_input(self, epoch: StreamStart, reason: str) -> None:
|
def pause_input(self, epoch: StreamStart, reason: str) -> None:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
@@ -253,11 +259,12 @@ class StreamingLifecycle:
|
|||||||
raise StreamSuspended("old epoch callbacks or connection still active")
|
raise StreamSuspended("old epoch callbacks or connection still active")
|
||||||
if self.continuity.phase != "waiting":
|
if self.continuity.phase != "waiting":
|
||||||
raise StreamSuspended("pause required before a new input epoch")
|
raise StreamSuspended("pause required before a new input epoch")
|
||||||
|
source_now = self._source_clock_ns() # Check BEFORE mutating the paused mailbox.
|
||||||
self.mailbox.begin_epoch()
|
self.mailbox.begin_epoch()
|
||||||
if self._ingress_thread is not None:
|
if self._ingress_thread is not None:
|
||||||
self._threads = [t for t in self._threads if t is not self._ingress_thread]
|
self._threads = [t for t in self._threads if t is not self._ingress_thread]
|
||||||
self._ingress_thread = None
|
self._ingress_thread = None
|
||||||
return self.continuity.begin(self._source_clock_ns())
|
return self.continuity.begin(source_now)
|
||||||
|
|
||||||
def resume_input(
|
def resume_input(
|
||||||
self, epoch: StreamStart, evidence: ResumeEvidence, reset_temporal: Callable[[], None]
|
self, epoch: StreamStart, evidence: ResumeEvidence, reset_temporal: Callable[[], None]
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
"""Responder-owned source timeline and conditional clock readiness, no GPU authority."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import threading
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from . import streaming_wire as wire
|
||||||
|
from .streaming_clock import ClockMappingError, ClockProbe, ClockReceipt, ClockWindow, _stamp
|
||||||
|
from .streaming_continuity import StreamSuspended
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SourceAnchor:
|
||||||
|
source_zero_ns: int
|
||||||
|
local_zero_ns: int
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_stamp(self.source_zero_ns)
|
||||||
|
_stamp(self.local_zero_ns)
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, str]:
|
||||||
|
return {
|
||||||
|
"source_zero_ns": str(self.source_zero_ns),
|
||||||
|
"local_zero_ns": str(self.local_zero_ns),
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, value: object) -> SourceAnchor:
|
||||||
|
if not isinstance(value, dict) or set(value) != {"source_zero_ns", "local_zero_ns"}:
|
||||||
|
raise ValueError("invalid immutable source anchor")
|
||||||
|
return cls(wire.uint64(value["source_zero_ns"]), wire.uint64(value["local_zero_ns"]))
|
||||||
|
|
||||||
|
|
||||||
|
class SourceClockMonitor:
|
||||||
|
def __init__(self, worker_clock_id: str, source_zero_ns: int) -> None:
|
||||||
|
wire.identifier(worker_clock_id)
|
||||||
|
_stamp(source_zero_ns)
|
||||||
|
self.worker_clock_id, self.source_zero = worker_clock_id, source_zero_ns
|
||||||
|
self.wall_zero = 0 # Not a comparable host clock; use due()/observed().
|
||||||
|
self.anchor: SourceAnchor | None = None
|
||||||
|
self.window: ClockWindow | None = None
|
||||||
|
self.ended = False
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
|
||||||
|
def observe(
|
||||||
|
self, probe: ClockProbe, received_ns: int, anchor: SourceAnchor, ended: bool
|
||||||
|
) -> None:
|
||||||
|
"""Called only after the transport verifies the one-use issued challenge."""
|
||||||
|
receipt = ClockReceipt(probe, received_ns)
|
||||||
|
if probe.remote_clock_id != self.worker_clock_id or type(ended) is not bool:
|
||||||
|
raise ValueError("foreign clock/status")
|
||||||
|
with self._lock:
|
||||||
|
if self.ended or anchor.source_zero_ns != self.source_zero:
|
||||||
|
raise ValueError("source activation ended or source anchor changed")
|
||||||
|
if self.anchor is None:
|
||||||
|
if not 0 <= anchor.local_zero_ns - probe.local_receive_ns <= 10_000_000_000:
|
||||||
|
raise ValueError("initial source start must be bounded and not in the past")
|
||||||
|
self.anchor = anchor
|
||||||
|
self.window = ClockWindow(
|
||||||
|
self.worker_clock_id,
|
||||||
|
probe.local_clock_id,
|
||||||
|
rate_ppm=500,
|
||||||
|
timestamp_error_ns=50_000,
|
||||||
|
)
|
||||||
|
elif anchor != self.anchor:
|
||||||
|
raise ValueError("source timeline is immutable for the activation")
|
||||||
|
assert self.window is not None
|
||||||
|
self.window.add(receipt)
|
||||||
|
self.ended = ended
|
||||||
|
|
||||||
|
def observed(self, now_ns: int) -> tuple[int, int]:
|
||||||
|
"""Source time midpoint + integer half-width, freshly checked at use."""
|
||||||
|
with self._lock:
|
||||||
|
try:
|
||||||
|
if self.window is None or self.anchor is None:
|
||||||
|
raise ClockMappingError("source clock not observed")
|
||||||
|
bounds = self.window.current(now_ns)
|
||||||
|
bounds.require(now_ns)
|
||||||
|
lower, upper = bounds.offset_at(now_ns)
|
||||||
|
stamp = self.source_zero + now_ns + (lower + upper) // 2 - self.anchor.local_zero_ns
|
||||||
|
_stamp(stamp)
|
||||||
|
return stamp, (upper - lower + 1) // 2
|
||||||
|
except ClockMappingError as exc:
|
||||||
|
raise StreamSuspended("source clock unavailable") from exc
|
||||||
|
|
||||||
|
def source_now(self, now_ns: int) -> int:
|
||||||
|
stamp, uncertainty = self.observed(now_ns)
|
||||||
|
# Admission/cutoff/resume use the upper age bound, never a younger estimate.
|
||||||
|
return stamp + uncertainty
|
||||||
|
|
||||||
|
def due(self, stamp: int, now_ns: int) -> tuple[int, int]:
|
||||||
|
observed, uncertainty = self.observed(now_ns)
|
||||||
|
return now_ns + stamp - observed, uncertainty
|
||||||
@@ -10,11 +10,18 @@ import pytest
|
|||||||
|
|
||||||
pytest.importorskip("grpc")
|
pytest.importorskip("grpc")
|
||||||
|
|
||||||
|
import time
|
||||||
|
|
||||||
from test_perception_streaming_grpc import eventually, identity, tls # noqa: F401,E402
|
from test_perception_streaming_grpc import eventually, identity, tls # noqa: F401,E402
|
||||||
|
|
||||||
|
from k1link.perception.streaming_control_grpc import ( # noqa: E402
|
||||||
|
ControlTicket,
|
||||||
|
StreamControlClient,
|
||||||
|
)
|
||||||
from k1link.perception.streaming_grpc import GrpcStreamClient # noqa: E402
|
from k1link.perception.streaming_grpc import GrpcStreamClient # noqa: E402
|
||||||
from k1link.perception.streaming_lifecycle import StreamingLifecycle # noqa: E402
|
from k1link.perception.streaming_lifecycle import StreamingLifecycle # noqa: E402
|
||||||
from k1link.perception.streaming_queue import StreamMailbox # noqa: E402
|
from k1link.perception.streaming_queue import StreamMailbox # noqa: E402
|
||||||
|
from k1link.perception.streaming_source_clock import SourceAnchor # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
@@ -33,7 +40,8 @@ def adapter(monkeypatch):
|
|||||||
return module, control
|
return module, control
|
||||||
|
|
||||||
|
|
||||||
def test_adapter_rotates_grant_only_after_old_stream_drains(tmp_path, tls, adapter): # noqa: F811
|
@pytest.mark.parametrize("before_open", [False, True])
|
||||||
|
def test_adapter_rotates_grant_only_after_old_stream_drains(tmp_path, tls, adapter, before_open): # noqa: F811
|
||||||
module, control = adapter
|
module, control = adapter
|
||||||
cert, key = tmp_path / "cert", tmp_path / "key"
|
cert, key = tmp_path / "cert", tmp_path / "key"
|
||||||
cert.write_bytes(tls[0])
|
cert.write_bytes(tls[0])
|
||||||
@@ -78,11 +86,21 @@ def test_adapter_rotates_grant_only_after_old_stream_drains(tmp_path, tls, adapt
|
|||||||
_, first = control.read_grant(tmp_path / "grant.json")
|
_, first = control.read_grant(tmp_path / "grant.json")
|
||||||
client = GrpcStreamClient(f"localhost:{bridge.port}", tls[0], first)
|
client = GrpcStreamClient(f"localhost:{bridge.port}", tls[0], first)
|
||||||
try:
|
try:
|
||||||
await client.open()
|
if before_open:
|
||||||
|
await client.call.initial_metadata() # Grant consumed, no application OPEN sent.
|
||||||
|
else:
|
||||||
|
await client.open()
|
||||||
await client.close() # True network disconnect, not a synthetic runtime pause.
|
await client.close() # True network disconnect, not a synthetic runtime pause.
|
||||||
await eventually(
|
try:
|
||||||
lambda: control.read_grant(tmp_path / "grant.json")[1].epoch != first.epoch
|
await eventually(
|
||||||
)
|
lambda: control.read_grant(tmp_path / "grant.json")[1].epoch != first.epoch
|
||||||
|
)
|
||||||
|
except TimeoutError:
|
||||||
|
pytest.fail(
|
||||||
|
f"epoch did not rotate: phase={runtime.continuity.phase}; "
|
||||||
|
f"stop={runtime.stop_event.is_set()}; adapter={bridge.failure}; "
|
||||||
|
f"last={bridge.endpoint.last}; active={bridge.endpoint.active is not None}"
|
||||||
|
)
|
||||||
_, second = control.read_grant(tmp_path / "grant.json")
|
_, second = control.read_grant(tmp_path / "grant.json")
|
||||||
assert runtime.continuity.phase == "synchronizing"
|
assert runtime.continuity.phase == "synchronizing"
|
||||||
assert second.epoch.lease_generation == first.epoch.lease_generation
|
assert second.epoch.lease_generation == first.epoch.lease_generation
|
||||||
@@ -114,3 +132,169 @@ def test_delayed_source_start_does_not_open_an_idle_connection(adapter):
|
|||||||
assert source.connection_delay(3_000_000_000, 1_000_000_000) == 1.9
|
assert source.connection_delay(3_000_000_000, 1_000_000_000) == 1.9
|
||||||
assert source.connection_delay(3_000_000_000, 2_950_000_000) == 0
|
assert source.connection_delay(3_000_000_000, 2_950_000_000) == 0
|
||||||
assert source.connection_delay(3_000_000_000, 3_010_000_000) == 0
|
assert source.connection_delay(3_000_000_000, 3_010_000_000) == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("has_stream", [False, True])
|
||||||
|
def test_source_eof_ack_precedes_data_channel_retirement(adapter, has_stream):
|
||||||
|
source = importlib.import_module("pilot_grpc_source")
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
async def acknowledge():
|
||||||
|
assert "end" not in calls
|
||||||
|
calls.append("ack")
|
||||||
|
|
||||||
|
async def end():
|
||||||
|
assert calls == ["ack"]
|
||||||
|
calls.append("end")
|
||||||
|
|
||||||
|
async def read():
|
||||||
|
calls.append("drain")
|
||||||
|
|
||||||
|
async def check():
|
||||||
|
await source.finish_source(
|
||||||
|
SimpleNamespace(end=end) if has_stream else None,
|
||||||
|
read() if has_stream else None,
|
||||||
|
SimpleNamespace(finish=acknowledge),
|
||||||
|
)
|
||||||
|
|
||||||
|
asyncio.run(check())
|
||||||
|
assert calls == (["ack", "end", "drain"] if has_stream else ["ack"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_anchor_latches_only_after_initial_clock_gate_and_never_on_recovery(
|
||||||
|
adapter, monkeypatch
|
||||||
|
):
|
||||||
|
module = importlib.import_module("pilot_source_control")
|
||||||
|
healthy, acknowledgements = [False], []
|
||||||
|
|
||||||
|
class Bounds:
|
||||||
|
def require(self, now):
|
||||||
|
if not healthy[0]:
|
||||||
|
raise module.ClockMappingError("synthetic excessive uncertainty")
|
||||||
|
|
||||||
|
def to_dict(self):
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def uncertainty_ns(self, now):
|
||||||
|
return 1_000_000 if healthy[0] else 6_000_000
|
||||||
|
|
||||||
|
class Client:
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def poll(self):
|
||||||
|
return object(), None
|
||||||
|
|
||||||
|
async def acknowledge(self, probe, anchor, *, ended):
|
||||||
|
acknowledgements.append(anchor)
|
||||||
|
|
||||||
|
monkeypatch.setattr(module, "StreamControlClient", Client)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
module,
|
||||||
|
"ClockWindow",
|
||||||
|
lambda *a, **kw: SimpleNamespace(add=lambda probe: Bounds(), current=lambda now: Bounds()),
|
||||||
|
)
|
||||||
|
control = module.SourceControl(
|
||||||
|
"unused",
|
||||||
|
b"",
|
||||||
|
{
|
||||||
|
"activation": identity().to_dict(),
|
||||||
|
"token": "b" * 64,
|
||||||
|
"remote_clock_id": "worker",
|
||||||
|
"source_zero_ns": str(10**12),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
async def check():
|
||||||
|
await control.update()
|
||||||
|
assert control.anchor is None and not control.ready() and not acknowledgements
|
||||||
|
healthy[0] = True
|
||||||
|
await control.update()
|
||||||
|
anchor = await control.wait_anchor()
|
||||||
|
assert anchor == control.anchor and control.ready()
|
||||||
|
healthy[0] = False
|
||||||
|
await control.update()
|
||||||
|
assert not control.ready() and control.anchor == anchor
|
||||||
|
healthy[0] = True
|
||||||
|
await control.update()
|
||||||
|
assert control.ready() and acknowledgements == [anchor] * 3
|
||||||
|
assert [row["acknowledged"] for row in control.samples] == [False, True, True, True]
|
||||||
|
|
||||||
|
asyncio.run(check())
|
||||||
|
|
||||||
|
|
||||||
|
def test_cross_host_bridge_clock_expiry_retains_owner_and_rotates_epoch(tmp_path, tls, adapter): # noqa: F811
|
||||||
|
module, _ = adapter
|
||||||
|
cert, key = tmp_path / "cert", tmp_path / "key"
|
||||||
|
cert.write_bytes(tls[0])
|
||||||
|
key.write_bytes(tls[1])
|
||||||
|
run = StreamingLifecycle(
|
||||||
|
identity(),
|
||||||
|
tmp_path / "lease",
|
||||||
|
StreamMailbox(),
|
||||||
|
threading.Event(),
|
||||||
|
clock_ns=lambda: 1_000_000_000,
|
||||||
|
recover_input=True,
|
||||||
|
source_clock_ns=lambda: bridge.source.source_now(time.monotonic_ns()),
|
||||||
|
)
|
||||||
|
run.ready()
|
||||||
|
bridge = module.NetworkGraphBridge(
|
||||||
|
run,
|
||||||
|
SimpleNamespace(close=lambda: None),
|
||||||
|
{},
|
||||||
|
control=tmp_path,
|
||||||
|
source_status=tmp_path / "unused",
|
||||||
|
source_zero=10**12,
|
||||||
|
certificate=cert,
|
||||||
|
private_key=key,
|
||||||
|
address="localhost:0",
|
||||||
|
reset_temporal=lambda: None,
|
||||||
|
cross_host=True,
|
||||||
|
)
|
||||||
|
real_serve = bridge.endpoint.serve
|
||||||
|
|
||||||
|
async def serve(*args, **kwargs):
|
||||||
|
server, port = await real_serve(*args, **kwargs)
|
||||||
|
bridge.port = port
|
||||||
|
return server, port
|
||||||
|
|
||||||
|
bridge.endpoint.serve = serve
|
||||||
|
|
||||||
|
async def check():
|
||||||
|
bridge.start()
|
||||||
|
ticket = ControlTicket(identity(), bridge.ticket.token)
|
||||||
|
client = StreamControlClient(f"localhost:{bridge.port}", tls[0], ticket, clock_id="source")
|
||||||
|
try:
|
||||||
|
probe, missing = await client.poll()
|
||||||
|
assert missing is None and run.continuity.phase == "waiting"
|
||||||
|
anchor = SourceAnchor(10**12, probe.local_receive_ns + 1_000_000_000)
|
||||||
|
await client.acknowledge(probe, anchor)
|
||||||
|
await eventually(lambda: bridge.access is not None)
|
||||||
|
first = bridge.access
|
||||||
|
stream = GrpcStreamClient(f"localhost:{bridge.port}", tls[0], first)
|
||||||
|
await stream.open()
|
||||||
|
# The listener keeps observing clock freshness even with no data.
|
||||||
|
# Shorten the TEST clock evidence lifetime, not any production gate.
|
||||||
|
bridge.source.window.maximum_age_ns = 50_000_000
|
||||||
|
await eventually(lambda: run.continuity.phase == "waiting")
|
||||||
|
await stream.close()
|
||||||
|
await eventually(lambda: bridge.endpoint.active is None)
|
||||||
|
assert not run.stop_event.is_set() and run.lease.start == identity()
|
||||||
|
run.renew(identity())
|
||||||
|
bridge.source.window.maximum_age_ns = 2_000_000_000
|
||||||
|
# Old evidence must not itself resume; new controller epoch still required.
|
||||||
|
await asyncio.sleep(0.03)
|
||||||
|
probe, _ = await client.poll()
|
||||||
|
await client.acknowledge(probe, anchor)
|
||||||
|
await eventually(lambda: bridge.access.epoch != first.epoch)
|
||||||
|
assert run.continuity.phase == "synchronizing"
|
||||||
|
assert len(bridge.clock_states) >= 3
|
||||||
|
finally:
|
||||||
|
await client.close()
|
||||||
|
|
||||||
|
try:
|
||||||
|
asyncio.run(check())
|
||||||
|
finally:
|
||||||
|
run.request_stop("completed")
|
||||||
|
assert bridge.close() and run.close()
|
||||||
|
assert not (tmp_path / "bootstrap.json").exists()
|
||||||
|
|||||||
@@ -362,6 +362,48 @@ def test_consumer_expires_only_old_cell_and_rehashes_view_without_mutating_wire(
|
|||||||
pilot.assess_receipt(view, derived, bundle=bundle, now_ns=bundle["due_ns"])
|
pilot.assess_receipt(view, derived, bundle=bundle, now_ns=bundle["due_ns"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_cross_host_uncertainty_expires_per_cell_and_survives_publication(pilot):
|
||||||
|
payload, bundle, ddr = cell_input()
|
||||||
|
# Without uncertainty old ground would be 249 ms old; upper bound is253 ms.
|
||||||
|
bundle["clock_observer"] = lambda _: (NOW + 49_000_000, 4_000_000)
|
||||||
|
pilot.prepare_publication(payload, bundle, ddr, epoch_id="pilot", now_ns=123, mode="per-cell")
|
||||||
|
assert payload["cell_assessment"]["expired_ground_cells"] == 1
|
||||||
|
assert payload["policy_actions"] == [2, 0]
|
||||||
|
assert payload["freshness_at_publication"]["clock_uncertainty_ms"] == 4
|
||||||
|
fresh = pilot.validate_receipt(payload, bundle, epoch_id="pilot")
|
||||||
|
# An independently measured later receipt can have a slightly smaller
|
||||||
|
# midpoint while still overlapping the published interval.
|
||||||
|
local_bundle = {k: v for k, v in bundle.items() if k != "clock_observer"}
|
||||||
|
view, checked = pilot.assess_receipt(
|
||||||
|
payload, fresh, bundle=local_bundle, now_ns=bundle["due_ns"] + 48_000_000
|
||||||
|
)
|
||||||
|
assert checked.clock_uncertainty_ms == 0 and view["policy_actions"] == [2, 0]
|
||||||
|
with pytest.raises(ValueError, match="backwards"):
|
||||||
|
pilot.assess_receipt(
|
||||||
|
payload, fresh, bundle=local_bundle, now_ns=bundle["due_ns"] + 44_000_000
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_one_clock_snapshot_per_publication_and_receipt_boundary(pilot):
|
||||||
|
payload, bundle, ddr = cell_input()
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def observe(now):
|
||||||
|
assert now not in calls # A later refresh may no longer describe this instant.
|
||||||
|
calls.append(now)
|
||||||
|
return NOW + now, 4_000_000
|
||||||
|
|
||||||
|
bundle["clock_observer"] = observe
|
||||||
|
pilot.prepare_publication(
|
||||||
|
payload, bundle, ddr, epoch_id="pilot", now_ns=260_000_000, mode="per-cell"
|
||||||
|
)
|
||||||
|
assert payload["policy_actions"] == [2, 2] # Exercises suppression/reassessment too.
|
||||||
|
fresh = pilot.validate_receipt(payload, bundle, epoch_id="pilot")
|
||||||
|
view, checked = pilot.assess_receipt(payload, fresh, bundle=bundle, now_ns=270_000_000)
|
||||||
|
assert calls == [260_000_000, 270_000_000]
|
||||||
|
assert not checked.fresh_complete and view["policy_actions"] == [2, 2]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("missing,held_ms", [(True, 0), (False, 220)])
|
@pytest.mark.parametrize("missing,held_ms", [(True, 0), (False, 220)])
|
||||||
def test_cell_freshness_cannot_override_missing_lidar_or_stale_segmentation(
|
def test_cell_freshness_cannot_override_missing_lidar_or_stale_segmentation(
|
||||||
pilot, missing, held_ms
|
pilot, missing, held_ms
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
"""Two-sided clock evidence and recoverable owner-preserving admission."""
|
||||||
|
|
||||||
|
import threading
|
||||||
|
from dataclasses import replace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from test_perception_streaming_grpc import identity
|
||||||
|
|
||||||
|
from k1link.perception.streaming_clock import (
|
||||||
|
ClockMappingError,
|
||||||
|
ClockProbe,
|
||||||
|
ClockReceipt,
|
||||||
|
ClockWindow,
|
||||||
|
)
|
||||||
|
from k1link.perception.streaming_continuity import StreamSuspended
|
||||||
|
from k1link.perception.streaming_lifecycle import StreamingLifecycle
|
||||||
|
from k1link.perception.streaming_queue import StreamMailbox
|
||||||
|
from k1link.perception.streaming_source_clock import SourceAnchor, SourceClockMonitor
|
||||||
|
|
||||||
|
|
||||||
|
def sample(number=1, outbound=1_000_000, inbound=2_000_000, offset=10**12):
|
||||||
|
t1 = 10**16 + number * 100_000_000
|
||||||
|
return ClockProbe(
|
||||||
|
"source",
|
||||||
|
"worker",
|
||||||
|
str(number),
|
||||||
|
t1,
|
||||||
|
t1 + outbound + offset,
|
||||||
|
t1 + outbound + offset + 10_000,
|
||||||
|
t1 + outbound + inbound + 10_000,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("outbound,inbound", [(1, 9_000_000), (9_000_000, 1), (1, 1)])
|
||||||
|
@pytest.mark.parametrize("offset", [-(10**12), 0, 10**12])
|
||||||
|
def test_responder_interval_contains_true_reverse_offset(outbound, inbound, offset):
|
||||||
|
probe = sample(outbound=outbound, inbound=inbound, offset=offset)
|
||||||
|
receipt = ClockReceipt(probe, probe.local_receive_ns + offset + 1_000_000)
|
||||||
|
window = ClockWindow("worker", "source", rate_ppm=500, timestamp_error_ns=50_000)
|
||||||
|
bounds = window.add(receipt)
|
||||||
|
lower, upper = bounds.offset_at(receipt.acknowledged_ns)
|
||||||
|
assert lower <= -offset <= upper
|
||||||
|
assert bounds.uncertainty_ns(receipt.acknowledged_ns + 1_000_000) > bounds.uncertainty_ns(
|
||||||
|
receipt.acknowledged_ns
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_timeline_immutable_expiring_and_observation_bounds_conservative():
|
||||||
|
first = sample()
|
||||||
|
anchor = SourceAnchor(10**12, first.local_receive_ns + 2_000_000_000)
|
||||||
|
monitor = SourceClockMonitor("worker", anchor.source_zero_ns)
|
||||||
|
with pytest.raises(StreamSuspended):
|
||||||
|
monitor.observed(first.remote_send_ns)
|
||||||
|
now = first.local_receive_ns + 10**12 + 1_000_000
|
||||||
|
monitor.observe(first, now, anchor, False)
|
||||||
|
midpoint, uncertainty = monitor.observed(now)
|
||||||
|
actual = anchor.source_zero_ns + (now - 10**12) - anchor.local_zero_ns
|
||||||
|
assert midpoint - uncertainty <= actual <= monitor.source_now(now)
|
||||||
|
with pytest.raises(StreamSuspended):
|
||||||
|
monitor.observed(now + 2_000_000_000)
|
||||||
|
second = sample(30)
|
||||||
|
later = second.local_receive_ns + 10**12 + 1_000_000
|
||||||
|
with pytest.raises(ValueError, match="immutable"):
|
||||||
|
monitor.observe(
|
||||||
|
second, later, replace(anchor, local_zero_ns=anchor.local_zero_ns + 1), False
|
||||||
|
)
|
||||||
|
monitor.observe(second, later, anchor, True)
|
||||||
|
assert monitor.ended
|
||||||
|
with pytest.raises(ValueError, match="ended"):
|
||||||
|
monitor.observe(sample(31), later + 100_000_000, anchor, False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_bad_ack_order_expired_jump_and_foreign_clock():
|
||||||
|
first = sample()
|
||||||
|
for stamp in (first.remote_send_ns - 1, first.remote_receive_ns + 500_000_001):
|
||||||
|
with pytest.raises(ClockMappingError):
|
||||||
|
ClockReceipt(first, stamp)
|
||||||
|
monitor = SourceClockMonitor("worker", 10**12)
|
||||||
|
anchor = SourceAnchor(10**12, first.local_receive_ns + 2_000_000_000)
|
||||||
|
monitor.observe(first, first.local_receive_ns + 10**12 + 1_000_000, anchor, False)
|
||||||
|
jumped = sample(2, offset=10**12 + 100_000_000)
|
||||||
|
with pytest.raises(ClockMappingError, match="envelope"):
|
||||||
|
monitor.observe(jumped, jumped.local_receive_ns + 10**12 + 101_000_000, anchor, False)
|
||||||
|
with pytest.raises(StreamSuspended):
|
||||||
|
monitor.observed(jumped.local_receive_ns + 10**12 + 101_000_000)
|
||||||
|
|
||||||
|
|
||||||
|
def test_clock_wait_does_not_stop_owner_or_reopen_mailbox_prematurely(tmp_path):
|
||||||
|
healthy = [True]
|
||||||
|
|
||||||
|
def source_now():
|
||||||
|
if not healthy[0]:
|
||||||
|
raise StreamSuspended("synthetic expired clock mapping")
|
||||||
|
return 1_000_000_000
|
||||||
|
|
||||||
|
run = StreamingLifecycle(
|
||||||
|
identity(),
|
||||||
|
tmp_path,
|
||||||
|
StreamMailbox(),
|
||||||
|
threading.Event(),
|
||||||
|
clock_ns=lambda: 1_000_000_000,
|
||||||
|
recover_input=True,
|
||||||
|
source_clock_ns=source_now,
|
||||||
|
)
|
||||||
|
run.ready()
|
||||||
|
try:
|
||||||
|
healthy[0] = False
|
||||||
|
with pytest.raises(StreamSuspended):
|
||||||
|
run.check_input(identity())
|
||||||
|
assert run.continuity.reason == "source-clock" and not run.stop_event.is_set()
|
||||||
|
run.renew(identity())
|
||||||
|
with pytest.raises(StreamSuspended):
|
||||||
|
run.begin_input(identity())
|
||||||
|
assert run.mailbox.epoch_drained and run.continuity.phase == "waiting"
|
||||||
|
healthy[0] = True
|
||||||
|
epoch = run.begin_input(identity())
|
||||||
|
assert epoch != identity() and epoch.lease_generation == 1
|
||||||
|
assert run.continuity.phase == "synchronizing"
|
||||||
|
healthy[0] = False
|
||||||
|
with pytest.raises(StreamSuspended, match="obsolete"):
|
||||||
|
run.check_input(identity())
|
||||||
|
assert run.continuity.phase == "synchronizing" # Stale peer cannot pause new epoch.
|
||||||
|
finally:
|
||||||
|
assert run.close()
|
||||||
@@ -28,6 +28,7 @@ from k1link.perception.streaming_control_grpc import (
|
|||||||
from k1link.perception.streaming_grpc import GrpcStreamClient, GrpcStreamEndpoint, StreamAccess
|
from k1link.perception.streaming_grpc import GrpcStreamClient, GrpcStreamEndpoint, StreamAccess
|
||||||
from k1link.perception.streaming_lifecycle import StreamingLifecycle
|
from k1link.perception.streaming_lifecycle import StreamingLifecycle
|
||||||
from k1link.perception.streaming_queue import StreamMailbox
|
from k1link.perception.streaming_queue import StreamMailbox
|
||||||
|
from k1link.perception.streaming_source_clock import SourceAnchor, SourceClockMonitor
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
@@ -204,3 +205,52 @@ def test_malicious_response_nonce_or_offer_rejected(tls):
|
|||||||
await client.close()
|
await client.close()
|
||||||
|
|
||||||
asyncio.run(check())
|
asyncio.run(check())
|
||||||
|
|
||||||
|
|
||||||
|
def test_acknowledged_clock_is_single_use_and_coexists_with_active_stream(tmp_path, tls):
|
||||||
|
async def check():
|
||||||
|
async with controlled(tmp_path, tls) as (_, endpoint, control, client, pending, port):
|
||||||
|
monitor = SourceClockMonitor("worker", 10**12)
|
||||||
|
control.observe = monitor.observe
|
||||||
|
probe, access = await client.poll()
|
||||||
|
anchor = SourceAnchor(10**12, probe.local_receive_ns + 2_000_000_000)
|
||||||
|
await client.acknowledge(probe, anchor)
|
||||||
|
assert monitor.observed(time.monotonic_ns())[1] > 0
|
||||||
|
with pytest.raises(grpc.aio.AioRpcError):
|
||||||
|
await client.acknowledge(probe, anchor)
|
||||||
|
stream = GrpcStreamClient(f"localhost:{port}", tls[0], access)
|
||||||
|
try:
|
||||||
|
await stream.open()
|
||||||
|
pending[0] = None
|
||||||
|
for _ in range(12):
|
||||||
|
await asyncio.sleep(0.03)
|
||||||
|
probe, missing = await client.poll()
|
||||||
|
assert missing is None
|
||||||
|
await client.acknowledge(probe, anchor)
|
||||||
|
assert monitor.observed(time.monotonic_ns())[1] < 5_000_000
|
||||||
|
assert endpoint.active is not None
|
||||||
|
finally:
|
||||||
|
await stream.close()
|
||||||
|
|
||||||
|
asyncio.run(check())
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("fault", ["nonce", "remote_send", "anchor", "end_type"])
|
||||||
|
def test_wrong_clock_receipt_cannot_update_mapping(tmp_path, tls, fault):
|
||||||
|
async def check():
|
||||||
|
async with controlled(tmp_path, tls) as (_, _, control, client, _, _):
|
||||||
|
monitor = SourceClockMonitor("worker", 10**12)
|
||||||
|
control.observe = monitor.observe
|
||||||
|
probe, _ = await client.poll()
|
||||||
|
anchor = SourceAnchor(10**12, probe.local_receive_ns + 2_000_000_000)
|
||||||
|
if fault == "nonce":
|
||||||
|
probe = replace(probe, nonce="foreign")
|
||||||
|
elif fault == "remote_send":
|
||||||
|
probe = replace(probe, remote_send_ns=probe.remote_send_ns + 1)
|
||||||
|
elif fault == "anchor":
|
||||||
|
anchor = replace(anchor, source_zero_ns=anchor.source_zero_ns + 1)
|
||||||
|
with pytest.raises((grpc.aio.AioRpcError, ValueError)):
|
||||||
|
await client.acknowledge(probe, anchor, ended=1 if fault == "end_type" else False)
|
||||||
|
assert monitor.anchor is None and monitor.window is None
|
||||||
|
|
||||||
|
asyncio.run(check())
|
||||||
|
|||||||
Reference in New Issue
Block a user