SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): gate cross-host graph with acknowledged source clocks

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-09-02 20:37:21 +03:00
parent d80df61a7e
commit 35b6cd9e9e
17 changed files with 1026 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/perception/worker/streaming_profile_stage1/pilot_binary_bridge.py
+7 -1
View File
@@ -126,7 +126,11 @@ class BinaryGraphInput:
length = len(e.value[0])
rolling_times[offset : offset + length] = e.time_ns
offset += length
due = self.wall_zero + stamp - self.source_zero
due = (
self.source.due(stamp, time.monotonic_ns())[0]
if hasattr(self.source, "observed")
else self.wall_zero + stamp - self.source_zero
)
bundle = {
"input_start": self.epoch,
"sequence": event.source_sequence,
@@ -158,6 +162,8 @@ class BinaryGraphInput:
"enqueued_ns": time.monotonic_ns(),
"payload_bytes": size,
}
if hasattr(self.source, "observed"):
bundle["clock_observer"] = self.source.observed
self.bgr_hashes.append(hashlib.sha256(image).hexdigest())
bundle["enqueued_ns"] = time.monotonic_ns()
transferred = self.runtime.admit_reserved(self.epoch, bundle, reservation)
experiments/perception/worker/streaming_profile_stage1/pilot_freshness.py
+32 -12
View File
@@ -89,15 +89,25 @@ def build_freshness(scene, bundle, ddr_layer, epoch_id):
return SceneFreshness(epoch_id, CLOCK_DOMAIN, sequence, stamp, tuple(layers.values()))
def source_observation(bundle, now_ns):
if "clock_observer" in bundle:
stamp, uncertainty_ns = bundle["clock_observer"](now_ns)
return stamp, uncertainty_ns / 1e6
# Legacy same-kernel path, or source/consumer on the originating Mac clock.
return bundle["time_ns"] + now_ns - bundle["due_ns"], 0
def assess(freshness, *, bundle, now_ns):
# Both sides of this pilot use the SAME monotonic clock. Map it back to
# original source arrival time; zero mapping uncertainty is local-only and
# is not a claim of hardware camera/LiDAR synchronization or network quality.
observed, uncertainty = source_observation(bundle, now_ns)
return assess_observation(freshness, observed, uncertainty)
def assess_observation(freshness, observed, uncertainty):
return freshness.assess(
epoch_id=freshness.epoch_id,
clock_domain_id=CLOCK_DOMAIN,
observed_source_time_ns=bundle["time_ns"] + now_ns - bundle["due_ns"],
clock_uncertainty_ms=0,
observed_source_time_ns=observed,
clock_uncertainty_ms=uncertainty,
maximum_clock_uncertainty_ms=5,
maximum_layer_age_ms=250,
)
@@ -141,19 +151,23 @@ def prepare_publication(scene, bundle, ddr_layer, *, epoch_id, now_ns, mode="who
if mode not in ("whole-scene", "per-cell"):
raise ValueError("unknown costmap freshness mode")
scene["costmap_freshness_mode"] = mode
observed, uncertainty = source_observation(bundle, now_ns)
if mode == "per-cell":
apply_cell_expiry(
scene,
source_time_ns=bundle["time_ns"],
observed_source_time_ns=bundle["time_ns"] + now_ns - bundle["due_ns"],
observed_source_time_ns=observed,
uncertainty_ms=uncertainty,
)
freshness = build_freshness(scene, bundle, ddr_layer, epoch_id)
checked = assess(freshness, bundle=bundle, now_ns=now_ns)
# One immutable clock observation per boundary, including cell and policy
# reassessment. A concurrent refresh must not reinterpret this same instant.
checked = assess_observation(freshness, observed, uncertainty)
if not checked.fresh_complete:
suppress_policy(scene)
# The envelope hashes the actual guarded policy, not the discarded one.
freshness = build_freshness(scene, bundle, ddr_layer, epoch_id)
checked = assess(freshness, bundle=bundle, now_ns=now_ns)
checked = assess_observation(freshness, observed, uncertainty)
scene["freshness"] = freshness.to_dict()
scene["freshness_at_publication"] = checked.to_dict()
scene["stale_at_publication"] = any(item.state == "stale" for item in checked.layers)
@@ -202,9 +216,14 @@ def assess_receipt(scene, freshness, *, bundle, now_ns):
their original support time. Layer identity/source timestamps do not change;
updated costmap/policy hashes belong to the derived view, not the wire bytes.
"""
observed = bundle["time_ns"] + now_ns - bundle["due_ns"]
observed, uncertainty = source_observation(bundle, now_ns)
previous = scene.get("freshness_at_receipt") or scene["freshness_at_publication"]
if observed < _read_wire_integer(previous["checked_at_source_time_ns"]):
# Independent clocks can have overlapping intervals even though receipt is
# later. Reject a provably reversed interval, not a shifted midpoint alone.
if observed + int(uncertainty * 1e6) < (
_read_wire_integer(previous["checked_at_source_time_ns"])
- int(previous["clock_uncertainty_ms"] * 1e6)
):
raise ValueError("consumer clock moved backwards")
view = dict(scene)
if scene.get("costmap_freshness_mode") == "per-cell":
@@ -212,6 +231,7 @@ def assess_receipt(scene, freshness, *, bundle, now_ns):
view,
source_time_ns=freshness.source_time_ns,
observed_source_time_ns=observed,
uncertainty_ms=uncertainty,
)
layers = {item.layer: item for item in freshness.layers[:4]}
for name in ("costmap", "policy"):
@@ -230,7 +250,7 @@ def assess_receipt(scene, freshness, *, bundle, now_ns):
oldest_required_input_time_ns=oldest,
)
freshness = replace(freshness, layers=tuple(layers.values()))
checked = assess(freshness, bundle=bundle, now_ns=now_ns)
checked = assess_observation(freshness, observed, uncertainty)
view = receipt_view(view, checked)
# Global suppression also changes the policy payload. A derived view must
# remain internally verifiable, including on a second consumer boundary.
@@ -238,7 +258,7 @@ def assess_receipt(scene, freshness, *, bundle, now_ns):
if policy.payload_sha256 is not None:
policy = replace(policy, payload_sha256=payload_digest(view, "policy"))
freshness = replace(freshness, layers=(*freshness.layers[:-1], policy))
checked = assess(freshness, bundle=bundle, now_ns=now_ns)
checked = assess_observation(freshness, observed, uncertainty)
view["freshness"] = freshness.to_dict()
view["freshness_at_receipt"] = checked.to_dict()
return view, checked
experiments/perception/worker/streaming_profile_stage1/pilot_grpc_graph.py
+80 -6
View File
@@ -5,6 +5,7 @@ accepted from the network. Grant rotation stays in this trusted local adapter.
"""
import asyncio
import secrets
import threading
import time
import traceback
@@ -15,7 +16,9 @@ from pilot_binary_bridge import BinaryGraphInput
from pilot_network_control import kernel_clock, read_control, write_control
from k1link.perception.streaming_continuity import StreamSuspended
from k1link.perception.streaming_control_grpc import ControlTicket, StreamControlEndpoint
from k1link.perception.streaming_grpc import GrpcStreamEndpoint
from k1link.perception.streaming_source_clock import SourceClockMonitor
class NetworkGraphBridge(BinaryGraphInput):
@@ -33,16 +36,40 @@ class NetworkGraphBridge(BinaryGraphInput):
address,
reset_temporal,
start_delay=2,
cross_host=False,
):
self.clock = kernel_clock()
source = SimpleNamespace(
source_zero=source_zero, wall_zero=time.monotonic_ns() + int(start_delay * 1e9)
self.cross_host = cross_host
self.clock_id = "worker-clock-" + secrets.token_hex(12)
self.clock = (
{"scope": "cross-host-conditional", "clock_id": self.clock_id}
if cross_host
else kernel_clock()
)
source = (
SourceClockMonitor(self.clock_id, source_zero)
if cross_host
else SimpleNamespace(
source_zero=source_zero, wall_zero=time.monotonic_ns() + int(start_delay * 1e9)
)
)
super().__init__(runtime, decoder, source, report, reset_temporal=reset_temporal)
self.control = Path(control)
self.source_status = Path(source_status)
self.certificate, self.private_key, self.address = certificate, private_key, address
self.endpoint = GrpcStreamEndpoint(runtime, self.consume, self.notice)
self.access = None
self.ticket = ControlTicket(runtime.start, secrets.token_hex(32)) if cross_host else None
self.control_endpoint = (
StreamControlEndpoint(
self.ticket,
self._pending,
clock_id=self.clock_id,
observe=source.observe,
)
if cross_host
else None
)
self.clock_states = []
self.stopping, self.ready = threading.Event(), threading.Event()
self.failure = None
self.published = []
@@ -64,6 +91,9 @@ class NetworkGraphBridge(BinaryGraphInput):
def _grant(self):
access = self.endpoint.issue(self.epoch, "recorded-acquisition", 1)
if self.cross_host:
self.access = access
return
write_control(
self.control / "grant.json",
{
@@ -76,6 +106,13 @@ class NetworkGraphBridge(BinaryGraphInput):
},
)
def _pending(self):
try:
self.source.observed(time.monotonic_ns())
except StreamSuspended:
return None
return self.access if self.endpoint.grant is not None else None
def _run(self):
try:
asyncio.run(self._serve())
@@ -90,21 +127,55 @@ class NetworkGraphBridge(BinaryGraphInput):
self.address,
certificate=Path(self.certificate).read_bytes(),
private_key=Path(self.private_key).read_bytes(),
control_handlers=(self.control_endpoint.handler(),) if self.cross_host else (),
)
try:
self._grant()
if self.cross_host:
self.runtime.pause_input(self.epoch, "source-clock")
write_control(
self.control / "bootstrap.json",
{
"activation": self.runtime.start.to_dict(),
"token": self.ticket.token,
"remote_clock_id": self.clock_id,
"source_zero_ns": str(self.source_zero),
},
)
else:
self._grant()
self.ready.set()
while not self.stopping.is_set() and not self.runtime.stop_event.is_set():
if self.cross_host:
try:
self.source.observed(time.monotonic_ns())
clock_ready = True
if not self.report["wall_zero_ns"]:
self.report["wall_zero_ns"] = self.source.due(
self.source_zero, time.monotonic_ns()
)[0]
except StreamSuspended:
clock_ready = False
self.runtime.pause_input(self.epoch, "source-clock")
if not self.clock_states or self.clock_states[-1]["ready"] != clock_ready:
self.clock_states.append(
{"at_ns": time.monotonic_ns(), "ready": clock_ready}
)
if len(self.clock_states) > 128:
raise ValueError("bounded clock transition count exceeded")
if self.endpoint.active is None:
ended = self.source_status / "source-end.json"
if ended.exists():
if self.cross_host and self.source.ended:
self.runtime.mailbox.finish()
elif not self.cross_host and ended.exists():
terminal = read_control(ended)
if terminal.get("run_id") != self.runtime.start.run_id:
raise ValueError("external source completion binding mismatch")
if terminal.get("error"):
raise ValueError("external source failed; inspect its bounded report")
self.runtime.mailbox.finish()
elif self.runtime.continuity.phase == "waiting":
elif self.runtime.continuity.phase == "waiting" and (
not self.cross_host or clock_ready
):
try:
epoch = self.runtime.begin_input(self.runtime.start)
except StreamSuspended:
@@ -118,6 +189,8 @@ class NetworkGraphBridge(BinaryGraphInput):
await asyncio.sleep(0.01)
finally:
await server.stop(0)
if self.cross_host:
(self.control / "bootstrap.json").unlink(missing_ok=True)
deadline = time.monotonic() + 2
while self.endpoint.active is not None and time.monotonic() < deadline:
await asyncio.sleep(0.01)
@@ -157,6 +230,7 @@ class NetworkGraphBridge(BinaryGraphInput):
accepted_camera_sequences=self.accepted,
published=self.published,
adapter_failure=self.failure,
clock_states=self.clock_states,
)
if not self.thread.is_alive() and self.endpoint.active is None:
self.window.close()
experiments/perception/worker/streaming_profile_stage1/pilot_grpc_source.py
+71 -12
View File
@@ -18,8 +18,9 @@ from pathlib import Path
import grpc
from pilot_binary_source import event_bytes, input_gaps, read_member
from pilot_freshness import assess_receipt, validate_receipt
from pilot_network_control import read_grant, write_control
from pilot_network_control import read_control, read_grant, write_control
from pilot_source import SensorArchive, camera_events, merged_events
from pilot_source_control import SourceControl
from k1link.compute.live_perception import LiveIngressEvent
from k1link.perception.streaming_grpc import GrpcStreamClient
@@ -32,6 +33,16 @@ def connection_delay(wall_zero_ns, now_ns):
return max(0, (wall_zero_ns - now_ns - 100_000_000) / 1e9)
async def finish_source(client, reader, clock_control):
# EOF on Exchange can finish the graph and retire its control endpoint.
# Confirm source completion first while that data stream is still alive.
if clock_control:
await clock_control.finish()
if client is not None:
await client.end()
await asyncio.wait_for(reader, timeout=5)
async def run(args):
report = {
"started_utc": datetime.now(UTC).isoformat(),
@@ -43,7 +54,9 @@ async def run(args):
"models": 0,
"source_clock_speed": 1.0,
"full_source_prepass": False,
"clock_scope": "two-containers-same-unshifted-Linux-kernel",
"clock_scope": "cross-host-conditional"
if args.cross_host
else "two-containers-same-unshifted-Linux-kernel",
"error": None,
"actuation_allowed": False,
"commands_enabled": False,
@@ -59,6 +72,7 @@ async def run(args):
gaps, next_gap, outage_until = input_gaps(args.input_gap), 0, 0
previous_epoch = None
clock = None
clock_control = None
async def disconnect():
nonlocal client, reader
@@ -71,11 +85,25 @@ async def run(args):
try:
deadline = time.monotonic() + 120
while not grant_path.exists():
startup_path = control / "bootstrap.json" if args.cross_host else grant_path
while not startup_path.exists():
if time.monotonic() > deadline:
raise TimeoutError("trusted graph grant was not issued after warmup")
await asyncio.sleep(0.02)
clock, _ = read_grant(grant_path)
if args.cross_host:
bootstrap = read_control(startup_path)
clock_control = SourceControl(
args.target, Path(args.certificate).read_bytes(), bootstrap
)
clock_control.start()
anchor = await clock_control.wait_anchor()
clock = {
"source_zero_ns": anchor.source_zero_ns,
"wall_zero_ns": anchor.local_zero_ns,
"clock": {"scope": "source-and-consumer-same-Mac-clock"},
}
else:
clock, _ = read_grant(grant_path)
first = next(camera_events(args.camera_index, 1)).time_ns
if clock["source_zero_ns"] != first - 500_000_000:
raise ValueError("recording prefix does not match admitted source clock")
@@ -104,6 +132,20 @@ async def run(args):
"due_ns": clock["wall_zero_ns"] + source_stamp - clock["source_zero_ns"],
}
freshness = validate_receipt(scene, bundle, epoch_id=epoch.epoch_id)
if clock_control and not clock_control.ready():
rejected = report.setdefault("clock_rejected_results", [])
if len(rejected) >= args.frames:
raise ValueError("bounded rejected receipt ledger exceeded")
rejected.append(
{
"sequence": sequence,
"epoch_id": epoch.epoch_id,
"payload_sha256": hashlib.sha256(payload).hexdigest(),
"reason": "source-clock-unavailable",
}
)
# No fresh scene/policy authority while clock admission is lost.
continue
checked_at = time.monotonic_ns()
view, checked = assess_receipt(
scene, freshness, bundle=bundle, now_ns=checked_at
@@ -140,7 +182,18 @@ async def run(args):
async def connect():
nonlocal client, reader, access, previous_epoch
value, candidate = read_grant(grant_path)
if clock_control:
if not clock_control.ready() or clock_control.access is None:
return False
candidate = clock_control.access
value = {
**clock,
"cutoff_ns": clock["source_zero_ns"]
+ time.monotonic_ns()
- clock["wall_zero_ns"],
}
else:
value, candidate = read_grant(grant_path)
if candidate.epoch.epoch_id == previous_epoch:
return False
if any(
@@ -185,6 +238,8 @@ async def run(args):
if event.channel == "camera":
released.append(event.sequence)
try:
if clock_control and not clock_control.ready():
await disconnect()
if reader is not None and reader.done():
reader.result() # Invalid payload is fatal; transport interruption is not.
raise ConnectionError("result stream ended before source EOF")
@@ -240,20 +295,23 @@ async def run(args):
break
source_end = True
report["end_sent_monotonic_ns"] = time.monotonic_ns()
if client is not None:
await client.end()
await asyncio.wait_for(reader, timeout=5)
write_control(
status / "source-end.json", {"run_id": access.epoch.run_id, "error": False}
)
await finish_source(client, reader, clock_control)
if not clock_control:
write_control(
status / "source-end.json", {"run_id": access.epoch.run_id, "error": False}
)
except Exception:
report["error"] = traceback.format_exc()
if access is not None:
if access is not None and not args.cross_host:
write_control(
status / "source-end.json", {"run_id": access.epoch.run_id, "error": True}
)
finally:
await disconnect()
if clock_control:
await clock_control.close()
report["clock_samples"] = clock_control.samples
report["clock_errors"] = clock_control.errors
from run_joint_pilot import distribution
report.update(
@@ -291,6 +349,7 @@ async def run(args):
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--target", required=True)
parser.add_argument("--cross-host", action="store_true")
parser.add_argument("--control", required=True)
parser.add_argument("--source-status", required=True)
parser.add_argument("--certificate", required=True)
experiments/perception/worker/streaming_profile_stage1/pilot_lifecycle.py
+2
View File
@@ -166,6 +166,8 @@ class PilotController:
def source_now(self):
if self.source_clock is None:
raise WorkerLeaseError("source clock mapping not configured")
if hasattr(self.source_clock, "source_now"):
return self.source_clock.source_now(time.monotonic_ns())
return self.source_clock.source_zero + time.monotonic_ns() - self.source_clock.wall_zero
def _renew(self):
experiments/perception/worker/streaming_profile_stage1/pilot_source_control.py
+113
View File
@@ -0,0 +1,113 @@
"""Bounded application-side clock loop; no lease or model control."""
import asyncio
import secrets
import time
import grpc
from k1link.perception.realtime_contract import StreamStart
from k1link.perception.streaming_clock import ClockMappingError, ClockWindow
from k1link.perception.streaming_control_grpc import ControlTicket, StreamControlClient
from k1link.perception.streaming_source_clock import SourceAnchor
class SourceControl:
def __init__(self, target, roots, bootstrap):
ticket = ControlTicket(StreamStart.from_dict(bootstrap["activation"]), bootstrap["token"])
clock_id = "source-clock-" + secrets.token_hex(12)
self.client = StreamControlClient(target, roots, ticket, clock_id=clock_id)
self.window = ClockWindow(
clock_id, bootstrap["remote_clock_id"], rate_ppm=500, timestamp_error_ns=50_000
)
self.source_zero_ns = int(bootstrap["source_zero_ns"])
self.anchor = None # Latch once after the initial clock handshake, never on resume.
self.access = None
self.confirmed = False
self.failure = None
self.stopping = self.ending = False
self.samples, self.errors = [], []
self.task = None
async def update(self):
probe, access = await self.client.poll()
bounds = self.window.add(probe)
now = time.monotonic_ns()
if self.anchor is None:
try:
self.window.current(now).require(now)
except ClockMappingError:
pass # Pre-start clock warmup, not a running source pause or retimestamp.
else:
self.anchor = SourceAnchor(self.source_zero_ns, now + 2_000_000_000)
if self.anchor is not None:
await self.client.acknowledge(probe, self.anchor, ended=self.ending)
self.confirmed = True
self.access = access
now = time.monotonic_ns()
self.samples.append(
{
"at_ns": now,
"bounds": bounds.to_dict(),
"uncertainty_ns": bounds.uncertainty_ns(now),
"ready": self.ready(),
"acknowledged": self.anchor is not None,
}
)
if len(self.samples) > 2048:
raise ValueError("bounded clock diagnostic count exceeded")
def ready(self):
if self.failure:
raise RuntimeError("source clock task failed") from self.failure
if not self.confirmed:
return False
try:
now = time.monotonic_ns()
self.window.current(now).require(now)
except ClockMappingError:
return False
return True
async def loop(self):
while not self.stopping:
try:
await self.update()
if self.ending:
return
except (grpc.RpcError, TimeoutError, OSError) as exc:
self.confirmed = False
self.errors.append({"at_ns": time.monotonic_ns(), "type": type(exc).__name__})
if len(self.errors) > 128:
self.failure = RuntimeError("bounded clock transport error count exceeded")
return
except Exception as exc:
self.failure = exc
return
await asyncio.sleep(0.05)
def start(self):
self.task = asyncio.create_task(self.loop())
async def wait_anchor(self):
deadline = time.monotonic() + 10
while self.anchor is None or not self.confirmed:
if self.failure:
raise RuntimeError("initial source clock handshake failed") from self.failure
if time.monotonic() > deadline:
raise TimeoutError("bounded initial source clock handshake")
await asyncio.sleep(0.01)
return self.anchor
async def finish(self):
self.ending = True
await asyncio.wait_for(asyncio.shield(self.task), timeout=2)
if self.failure:
raise RuntimeError("source clock final acknowledgement failed") from self.failure
async def close(self):
self.stopping = True
if self.task:
self.task.cancel()
await asyncio.gather(self.task, return_exceptions=True)
await self.client.close()
experiments/perception/worker/streaming_profile_stage1/run_joint_pilot.py
+17 -1
View File
@@ -276,6 +276,17 @@ def run(args):
"costmap_freshness_mode": args.costmap_freshness,
"ddrnet_min_source_interval_ms": args.ddrnet_min_source_interval_ms,
"input_transport": args.input_transport,
"network_clock_mode": "acknowledged-monotonic/v1"
if getattr(args, "network_cross_host", False)
else "local-only",
"network_clock_envelope": {
"relative_rate_ppm": 500,
"timestamp_error_ns": 50000,
"maximum_age_ns": 2000000000,
"maximum_uncertainty_ns": 5000000,
}
if getattr(args, "network_cross_host", False)
else None,
"recover_input": getattr(args, "recover_input", False),
"input_gap_plan": getattr(args, "input_gap", []),
}
@@ -549,6 +560,7 @@ def run(args):
private_key=args.network_private_key,
address=args.network_address,
reset_temporal=reset_temporal,
cross_host=args.network_cross_host,
)
controller.source_clock = binary_bridge.source
report["scope"] = "full-graph-network-candidate; receiver evidence is separate"
@@ -710,7 +722,10 @@ def run(args):
"scene_sha256": hashlib.sha256(encoded).hexdigest(),
}
results.append(result)
except StreamSuspended:
except StreamSuspended as exc:
report.setdefault("suspended_results", []).append(
{"sequence": bundle["sequence"], "reason": str(exc)}
)
mailbox.release(bundle, discard_reason="input-gap")
cpu_bundle = bundle = computed = mask = proposals = scene = received = view = (
None
@@ -947,6 +962,7 @@ if __name__ == "__main__":
parser.add_argument("--network-certificate")
parser.add_argument("--network-private-key")
parser.add_argument("--network-address", default="[::]:50061")
parser.add_argument("--network-cross-host", action="store_true")
parser.add_argument(
"--ddrnet-layout", choices=("reference", "channels-last"), default="reference"
)