feat(perception): gate cross-host graph with acknowledged source clocks

tests/test_perception_network_graph_adapter.py

@@ -10,11 +10,18 @@ import pytest
 
 pytest.importorskip("grpc")
 
+import time
+
 from test_perception_streaming_grpc import eventually, identity, tls  # noqa: F401,E402
 
+from k1link.perception.streaming_control_grpc import (  # noqa: E402
+    ControlTicket,
+    StreamControlClient,
+)
 from k1link.perception.streaming_grpc import GrpcStreamClient  # noqa: E402
 from k1link.perception.streaming_lifecycle import StreamingLifecycle  # noqa: E402
 from k1link.perception.streaming_queue import StreamMailbox  # noqa: E402
+from k1link.perception.streaming_source_clock import SourceAnchor  # noqa: E402
 
 
 @pytest.fixture
@@ -33,7 +40,8 @@ def adapter(monkeypatch):
     return module, control
 
 
-def test_adapter_rotates_grant_only_after_old_stream_drains(tmp_path, tls, adapter):  # noqa: F811
+@pytest.mark.parametrize("before_open", [False, True])
+def test_adapter_rotates_grant_only_after_old_stream_drains(tmp_path, tls, adapter, before_open):  # noqa: F811
     module, control = adapter
     cert, key = tmp_path / "cert", tmp_path / "key"
     cert.write_bytes(tls[0])
@@ -78,11 +86,21 @@ def test_adapter_rotates_grant_only_after_old_stream_drains(tmp_path, tls, adapt
         _, first = control.read_grant(tmp_path / "grant.json")
         client = GrpcStreamClient(f"localhost:{bridge.port}", tls[0], first)
         try:
-            await client.open()
+            if before_open:
+                await client.call.initial_metadata()  # Grant consumed, no application OPEN sent.
+            else:
+                await client.open()
             await client.close()  # True network disconnect, not a synthetic runtime pause.
-            await eventually(
-                lambda: control.read_grant(tmp_path / "grant.json")[1].epoch != first.epoch
-            )
+            try:
+                await eventually(
+                    lambda: control.read_grant(tmp_path / "grant.json")[1].epoch != first.epoch
+                )
+            except TimeoutError:
+                pytest.fail(
+                    f"epoch did not rotate: phase={runtime.continuity.phase}; "
+                    f"stop={runtime.stop_event.is_set()}; adapter={bridge.failure}; "
+                    f"last={bridge.endpoint.last}; active={bridge.endpoint.active is not None}"
+                )
             _, second = control.read_grant(tmp_path / "grant.json")
             assert runtime.continuity.phase == "synchronizing"
             assert second.epoch.lease_generation == first.epoch.lease_generation
@@ -114,3 +132,169 @@ def test_delayed_source_start_does_not_open_an_idle_connection(adapter):
     assert source.connection_delay(3_000_000_000, 1_000_000_000) == 1.9
     assert source.connection_delay(3_000_000_000, 2_950_000_000) == 0
     assert source.connection_delay(3_000_000_000, 3_010_000_000) == 0
+
+
+@pytest.mark.parametrize("has_stream", [False, True])
+def test_source_eof_ack_precedes_data_channel_retirement(adapter, has_stream):
+    source = importlib.import_module("pilot_grpc_source")
+    calls = []
+
+    async def acknowledge():
+        assert "end" not in calls
+        calls.append("ack")
+
+    async def end():
+        assert calls == ["ack"]
+        calls.append("end")
+
+    async def read():
+        calls.append("drain")
+
+    async def check():
+        await source.finish_source(
+            SimpleNamespace(end=end) if has_stream else None,
+            read() if has_stream else None,
+            SimpleNamespace(finish=acknowledge),
+        )
+
+    asyncio.run(check())
+    assert calls == (["ack", "end", "drain"] if has_stream else ["ack"])
+
+
+def test_source_anchor_latches_only_after_initial_clock_gate_and_never_on_recovery(
+    adapter, monkeypatch
+):
+    module = importlib.import_module("pilot_source_control")
+    healthy, acknowledgements = [False], []
+
+    class Bounds:
+        def require(self, now):
+            if not healthy[0]:
+                raise module.ClockMappingError("synthetic excessive uncertainty")
+
+        def to_dict(self):
+            return {}
+
+        def uncertainty_ns(self, now):
+            return 1_000_000 if healthy[0] else 6_000_000
+
+    class Client:
+        def __init__(self, *args, **kwargs):
+            pass
+
+        async def poll(self):
+            return object(), None
+
+        async def acknowledge(self, probe, anchor, *, ended):
+            acknowledgements.append(anchor)
+
+    monkeypatch.setattr(module, "StreamControlClient", Client)
+    monkeypatch.setattr(
+        module,
+        "ClockWindow",
+        lambda *a, **kw: SimpleNamespace(add=lambda probe: Bounds(), current=lambda now: Bounds()),
+    )
+    control = module.SourceControl(
+        "unused",
+        b"",
+        {
+            "activation": identity().to_dict(),
+            "token": "b" * 64,
+            "remote_clock_id": "worker",
+            "source_zero_ns": str(10**12),
+        },
+    )
+
+    async def check():
+        await control.update()
+        assert control.anchor is None and not control.ready() and not acknowledgements
+        healthy[0] = True
+        await control.update()
+        anchor = await control.wait_anchor()
+        assert anchor == control.anchor and control.ready()
+        healthy[0] = False
+        await control.update()
+        assert not control.ready() and control.anchor == anchor
+        healthy[0] = True
+        await control.update()
+        assert control.ready() and acknowledgements == [anchor] * 3
+        assert [row["acknowledged"] for row in control.samples] == [False, True, True, True]
+
+    asyncio.run(check())
+
+
+def test_cross_host_bridge_clock_expiry_retains_owner_and_rotates_epoch(tmp_path, tls, adapter):  # noqa: F811
+    module, _ = adapter
+    cert, key = tmp_path / "cert", tmp_path / "key"
+    cert.write_bytes(tls[0])
+    key.write_bytes(tls[1])
+    run = StreamingLifecycle(
+        identity(),
+        tmp_path / "lease",
+        StreamMailbox(),
+        threading.Event(),
+        clock_ns=lambda: 1_000_000_000,
+        recover_input=True,
+        source_clock_ns=lambda: bridge.source.source_now(time.monotonic_ns()),
+    )
+    run.ready()
+    bridge = module.NetworkGraphBridge(
+        run,
+        SimpleNamespace(close=lambda: None),
+        {},
+        control=tmp_path,
+        source_status=tmp_path / "unused",
+        source_zero=10**12,
+        certificate=cert,
+        private_key=key,
+        address="localhost:0",
+        reset_temporal=lambda: None,
+        cross_host=True,
+    )
+    real_serve = bridge.endpoint.serve
+
+    async def serve(*args, **kwargs):
+        server, port = await real_serve(*args, **kwargs)
+        bridge.port = port
+        return server, port
+
+    bridge.endpoint.serve = serve
+
+    async def check():
+        bridge.start()
+        ticket = ControlTicket(identity(), bridge.ticket.token)
+        client = StreamControlClient(f"localhost:{bridge.port}", tls[0], ticket, clock_id="source")
+        try:
+            probe, missing = await client.poll()
+            assert missing is None and run.continuity.phase == "waiting"
+            anchor = SourceAnchor(10**12, probe.local_receive_ns + 1_000_000_000)
+            await client.acknowledge(probe, anchor)
+            await eventually(lambda: bridge.access is not None)
+            first = bridge.access
+            stream = GrpcStreamClient(f"localhost:{bridge.port}", tls[0], first)
+            await stream.open()
+            # The listener keeps observing clock freshness even with no data.
+            # Shorten the TEST clock evidence lifetime, not any production gate.
+            bridge.source.window.maximum_age_ns = 50_000_000
+            await eventually(lambda: run.continuity.phase == "waiting")
+            await stream.close()
+            await eventually(lambda: bridge.endpoint.active is None)
+            assert not run.stop_event.is_set() and run.lease.start == identity()
+            run.renew(identity())
+            bridge.source.window.maximum_age_ns = 2_000_000_000
+            # Old evidence must not itself resume; new controller epoch still required.
+            await asyncio.sleep(0.03)
+            probe, _ = await client.poll()
+            await client.acknowledge(probe, anchor)
+            await eventually(lambda: bridge.access.epoch != first.epoch)
+            assert run.continuity.phase == "synchronizing"
+            assert len(bridge.clock_states) >= 3
+        finally:
+            await client.close()
+
+    try:
+        asyncio.run(check())
+    finally:
+        run.request_stop("completed")
+        assert bridge.close() and run.close()
+        assert not (tmp_path / "bootstrap.json").exists()

tests/test_perception_realtime_scene.py

@@ -362,6 +362,48 @@ def test_consumer_expires_only_old_cell_and_rehashes_view_without_mutating_wire(
         pilot.assess_receipt(view, derived, bundle=bundle, now_ns=bundle["due_ns"])
 
 
+def test_cross_host_uncertainty_expires_per_cell_and_survives_publication(pilot):
+    payload, bundle, ddr = cell_input()
+    # Without uncertainty old ground would be 249 ms old; upper bound is253 ms.
+    bundle["clock_observer"] = lambda _: (NOW + 49_000_000, 4_000_000)
+    pilot.prepare_publication(payload, bundle, ddr, epoch_id="pilot", now_ns=123, mode="per-cell")
+    assert payload["cell_assessment"]["expired_ground_cells"] == 1
+    assert payload["policy_actions"] == [2, 0]
+    assert payload["freshness_at_publication"]["clock_uncertainty_ms"] == 4
+    fresh = pilot.validate_receipt(payload, bundle, epoch_id="pilot")
+    # An independently measured later receipt can have a slightly smaller
+    # midpoint while still overlapping the published interval.
+    local_bundle = {k: v for k, v in bundle.items() if k != "clock_observer"}
+    view, checked = pilot.assess_receipt(
+        payload, fresh, bundle=local_bundle, now_ns=bundle["due_ns"] + 48_000_000
+    )
+    assert checked.clock_uncertainty_ms == 0 and view["policy_actions"] == [2, 0]
+    with pytest.raises(ValueError, match="backwards"):
+        pilot.assess_receipt(
+            payload, fresh, bundle=local_bundle, now_ns=bundle["due_ns"] + 44_000_000
+        )
+
+
+def test_one_clock_snapshot_per_publication_and_receipt_boundary(pilot):
+    payload, bundle, ddr = cell_input()
+    calls = []
+
+    def observe(now):
+        assert now not in calls  # A later refresh may no longer describe this instant.
+        calls.append(now)
+        return NOW + now, 4_000_000
+
+    bundle["clock_observer"] = observe
+    pilot.prepare_publication(
+        payload, bundle, ddr, epoch_id="pilot", now_ns=260_000_000, mode="per-cell"
+    )
+    assert payload["policy_actions"] == [2, 2]  # Exercises suppression/reassessment too.
+    fresh = pilot.validate_receipt(payload, bundle, epoch_id="pilot")
+    view, checked = pilot.assess_receipt(payload, fresh, bundle=bundle, now_ns=270_000_000)
+    assert calls == [260_000_000, 270_000_000]
+    assert not checked.fresh_complete and view["policy_actions"] == [2, 2]
+
+
 @pytest.mark.parametrize("missing,held_ms", [(True, 0), (False, 220)])
 def test_cell_freshness_cannot_override_missing_lidar_or_stale_segmentation(
     pilot, missing, held_ms

tests/test_perception_source_clock.py

@@ -0,0 +1,124 @@
+"""Two-sided clock evidence and recoverable owner-preserving admission."""
+
+import threading
+from dataclasses import replace
+
+import pytest
+from test_perception_streaming_grpc import identity
+
+from k1link.perception.streaming_clock import (
+    ClockMappingError,
+    ClockProbe,
+    ClockReceipt,
+    ClockWindow,
+)
+from k1link.perception.streaming_continuity import StreamSuspended
+from k1link.perception.streaming_lifecycle import StreamingLifecycle
+from k1link.perception.streaming_queue import StreamMailbox
+from k1link.perception.streaming_source_clock import SourceAnchor, SourceClockMonitor
+
+
+def sample(number=1, outbound=1_000_000, inbound=2_000_000, offset=10**12):
+    t1 = 10**16 + number * 100_000_000
+    return ClockProbe(
+        "source",
+        "worker",
+        str(number),
+        t1,
+        t1 + outbound + offset,
+        t1 + outbound + offset + 10_000,
+        t1 + outbound + inbound + 10_000,
+    )
+
+
+@pytest.mark.parametrize("outbound,inbound", [(1, 9_000_000), (9_000_000, 1), (1, 1)])
+@pytest.mark.parametrize("offset", [-(10**12), 0, 10**12])
+def test_responder_interval_contains_true_reverse_offset(outbound, inbound, offset):
+    probe = sample(outbound=outbound, inbound=inbound, offset=offset)
+    receipt = ClockReceipt(probe, probe.local_receive_ns + offset + 1_000_000)
+    window = ClockWindow("worker", "source", rate_ppm=500, timestamp_error_ns=50_000)
+    bounds = window.add(receipt)
+    lower, upper = bounds.offset_at(receipt.acknowledged_ns)
+    assert lower <= -offset <= upper
+    assert bounds.uncertainty_ns(receipt.acknowledged_ns + 1_000_000) > bounds.uncertainty_ns(
+        receipt.acknowledged_ns
+    )
+
+
+def test_source_timeline_immutable_expiring_and_observation_bounds_conservative():
+    first = sample()
+    anchor = SourceAnchor(10**12, first.local_receive_ns + 2_000_000_000)
+    monitor = SourceClockMonitor("worker", anchor.source_zero_ns)
+    with pytest.raises(StreamSuspended):
+        monitor.observed(first.remote_send_ns)
+    now = first.local_receive_ns + 10**12 + 1_000_000
+    monitor.observe(first, now, anchor, False)
+    midpoint, uncertainty = monitor.observed(now)
+    actual = anchor.source_zero_ns + (now - 10**12) - anchor.local_zero_ns
+    assert midpoint - uncertainty <= actual <= monitor.source_now(now)
+    with pytest.raises(StreamSuspended):
+        monitor.observed(now + 2_000_000_000)
+    second = sample(30)
+    later = second.local_receive_ns + 10**12 + 1_000_000
+    with pytest.raises(ValueError, match="immutable"):
+        monitor.observe(
+            second, later, replace(anchor, local_zero_ns=anchor.local_zero_ns + 1), False
+        )
+    monitor.observe(second, later, anchor, True)
+    assert monitor.ended
+    with pytest.raises(ValueError, match="ended"):
+        monitor.observe(sample(31), later + 100_000_000, anchor, False)
+
+
+def test_bad_ack_order_expired_jump_and_foreign_clock():
+    first = sample()
+    for stamp in (first.remote_send_ns - 1, first.remote_receive_ns + 500_000_001):
+        with pytest.raises(ClockMappingError):
+            ClockReceipt(first, stamp)
+    monitor = SourceClockMonitor("worker", 10**12)
+    anchor = SourceAnchor(10**12, first.local_receive_ns + 2_000_000_000)
+    monitor.observe(first, first.local_receive_ns + 10**12 + 1_000_000, anchor, False)
+    jumped = sample(2, offset=10**12 + 100_000_000)
+    with pytest.raises(ClockMappingError, match="envelope"):
+        monitor.observe(jumped, jumped.local_receive_ns + 10**12 + 101_000_000, anchor, False)
+    with pytest.raises(StreamSuspended):
+        monitor.observed(jumped.local_receive_ns + 10**12 + 101_000_000)
+
+
+def test_clock_wait_does_not_stop_owner_or_reopen_mailbox_prematurely(tmp_path):
+    healthy = [True]
+
+    def source_now():
+        if not healthy[0]:
+            raise StreamSuspended("synthetic expired clock mapping")
+        return 1_000_000_000
+
+    run = StreamingLifecycle(
+        identity(),
+        tmp_path,
+        StreamMailbox(),
+        threading.Event(),
+        clock_ns=lambda: 1_000_000_000,
+        recover_input=True,
+        source_clock_ns=source_now,
+    )
+    run.ready()
+    try:
+        healthy[0] = False
+        with pytest.raises(StreamSuspended):
+            run.check_input(identity())
+        assert run.continuity.reason == "source-clock" and not run.stop_event.is_set()
+        run.renew(identity())
+        with pytest.raises(StreamSuspended):
+            run.begin_input(identity())
+        assert run.mailbox.epoch_drained and run.continuity.phase == "waiting"
+        healthy[0] = True
+        epoch = run.begin_input(identity())
+        assert epoch != identity() and epoch.lease_generation == 1
+        assert run.continuity.phase == "synchronizing"
+        healthy[0] = False
+        with pytest.raises(StreamSuspended, match="obsolete"):
+            run.check_input(identity())
+        assert run.continuity.phase == "synchronizing"  # Stale peer cannot pause new epoch.
+    finally:
+        assert run.close()

tests/test_perception_streaming_control_grpc.py

@@ -28,6 +28,7 @@ from k1link.perception.streaming_control_grpc import (
 from k1link.perception.streaming_grpc import GrpcStreamClient, GrpcStreamEndpoint, StreamAccess
 from k1link.perception.streaming_lifecycle import StreamingLifecycle
 from k1link.perception.streaming_queue import StreamMailbox
+from k1link.perception.streaming_source_clock import SourceAnchor, SourceClockMonitor
 
 
 @asynccontextmanager
@@ -204,3 +205,52 @@ def test_malicious_response_nonce_or_offer_rejected(tls):
         await client.close()
 
     asyncio.run(check())
+
+
+def test_acknowledged_clock_is_single_use_and_coexists_with_active_stream(tmp_path, tls):
+    async def check():
+        async with controlled(tmp_path, tls) as (_, endpoint, control, client, pending, port):
+            monitor = SourceClockMonitor("worker", 10**12)
+            control.observe = monitor.observe
+            probe, access = await client.poll()
+            anchor = SourceAnchor(10**12, probe.local_receive_ns + 2_000_000_000)
+            await client.acknowledge(probe, anchor)
+            assert monitor.observed(time.monotonic_ns())[1] > 0
+            with pytest.raises(grpc.aio.AioRpcError):
+                await client.acknowledge(probe, anchor)
+            stream = GrpcStreamClient(f"localhost:{port}", tls[0], access)
+            try:
+                await stream.open()
+                pending[0] = None
+                for _ in range(12):
+                    await asyncio.sleep(0.03)
+                    probe, missing = await client.poll()
+                    assert missing is None
+                    await client.acknowledge(probe, anchor)
+                    assert monitor.observed(time.monotonic_ns())[1] < 5_000_000
+                assert endpoint.active is not None
+            finally:
+                await stream.close()
+
+    asyncio.run(check())
+
+
+@pytest.mark.parametrize("fault", ["nonce", "remote_send", "anchor", "end_type"])
+def test_wrong_clock_receipt_cannot_update_mapping(tmp_path, tls, fault):
+    async def check():
+        async with controlled(tmp_path, tls) as (_, _, control, client, _, _):
+            monitor = SourceClockMonitor("worker", 10**12)
+            control.observe = monitor.observe
+            probe, _ = await client.poll()
+            anchor = SourceAnchor(10**12, probe.local_receive_ns + 2_000_000_000)
+            if fault == "nonce":
+                probe = replace(probe, nonce="foreign")
+            elif fault == "remote_send":
+                probe = replace(probe, remote_send_ns=probe.remote_send_ns + 1)
+            elif fault == "anchor":
+                anchor = replace(anchor, source_zero_ns=anchor.source_zero_ns + 1)
+            with pytest.raises((grpc.aio.AioRpcError, ValueError)):
+                await client.acknowledge(probe, anchor, ended=1 if fault == "end_type" else False)
+            assert monitor.anchor is None and monitor.window is None
+
+    asyncio.run(check())