feat(perception): preserve resident runtime across recoverable input gaps

experiments/perception/worker/streaming_profile_stage1/pilot_continuity_probe.py

@@ -0,0 +1,281 @@
+"""CPU-only recovery proof: real H264 fixture, synthetic clock/pose/points.
+
+Not a full graph run: a resident CPU sentinel stands in for loaded GPU models.
+The real decoder process must survive two IPC reconnects and reinitialize its
+codec. The source clock/tick counter continues through each outage, no backlog.
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import select
+import socket
+import struct
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+
+from k1link.compute.live_perception import LiveIngressEvent
+from k1link.perception.realtime_contract import StreamStart
+from k1link.perception.streaming_continuity import ResumeEvidence, StreamSuspended
+from k1link.perception.streaming_decoder_client import StreamingDecoderClient
+from k1link.perception.streaming_ingress import StreamingIngress
+from k1link.perception.streaming_lifecycle import StreamingLifecycle
+from k1link.perception.streaming_queue import StreamMailbox
+from k1link.perception.streaming_sender import StreamingSender
+from k1link.perception.streaming_sensors import CausalSensorWindow, normalized_sensor
+from k1link.perception.worker_lease import WorkerLeaseError
+
+
+def run(output, camera):
+    output.mkdir()
+    with (camera / "index.jsonl").open() as stream:
+        raw = stream.readline(65537)
+        assert len(raw) <= 65536
+        row = json.loads(raw)
+    frame_path = (camera / row["path"]).resolve(strict=True)
+    assert frame_path.is_relative_to(camera.resolve())
+    assert 0 < frame_path.stat().st_size <= 1024 * 1024
+    assert 0 < (camera / "init.mp4").stat().st_size <= 65536
+    fragment, init = frame_path.read_bytes(), (camera / "init.mp4").read_bytes()
+    assert len(init) <= 65536 and hashlib.sha256(fragment).hexdigest() == row["sha256"]
+    report = {
+        "created_utc": datetime.now(UTC).isoformat(),
+        "started_monotonic_ns": time.monotonic_ns(),
+        "scope": (
+            "real decoder and Linux processes; synthetic source timestamps/pose/points; "
+            "NO GPU models or full graph"
+        ),
+        "source_full_prepass": False,
+        "source_index_rows_read": 1,
+        "fixture_fragment_sha256": hashlib.sha256(fragment).hexdigest(),
+        "gpu_devices": sorted(str(p) for p in Path("/dev").glob("nvidia*")),
+        "cycles": [],
+    }
+    assert not report["gpu_devices"] and os.environ.get("CUDA_VISIBLE_DEVICES") == ""
+    activation = StreamStart(
+        output.name,
+        "synthetic-recovery",
+        "synthetic-worker",
+        "activation-1",
+        1,
+        "a" * 64,
+        "b" * 64,
+        hashlib.sha256(b"resumable-input-real-decoder-probe-v1").hexdigest(),
+        "d" * 64,
+        "worker-mapped-fixture-clock",
+        "live",
+    )
+    with tempfile.TemporaryDirectory(prefix="continuity-lease-") as directory:
+        runtime = StreamingLifecycle(
+            activation,
+            Path(directory),
+            StreamMailbox(),
+            threading.Event(),
+            recover_input=True,
+            source_clock_ns=time.monotonic_ns,
+        )
+        finish = threading.Event()
+        ticks = [0]
+
+        def controller():
+            while not finish.wait(0.05):
+                ticks[0] += 1
+                try:
+                    runtime.renew(activation)
+                except WorkerLeaseError:
+                    return
+
+        thread = threading.Thread(target=controller, daemon=True)
+        thread.start()
+        decoder = window = receiver = sender = None
+        try:
+            sentinel = runtime.spawn(
+                lambda: subprocess.Popen(
+                    [
+                        sys.executable,
+                        "-c",
+                        "import time; print('ready',flush=True); time.sleep(30)",
+                    ],
+                    stdout=subprocess.PIPE,
+                    start_new_session=True,
+                )
+            )
+            assert (
+                select.select([sentinel.stdout], [], [], 2)[0]
+                and sentinel.stdout.readline() == b"ready\n"
+            )
+            child = runtime.spawn(
+                lambda: subprocess.Popen(
+                    ["python3", "-B", "/probe/pilot_fragment_decoder.py"],
+                    stdin=subprocess.PIPE,
+                    stdout=subprocess.PIPE,
+                    start_new_session=True,
+                )
+            )
+            decoder = StreamingDecoderClient(runtime, child.stdout.fileno(), child.stdin.fileno())
+            window = CausalSensorWindow(runtime.mailbox)
+            runtime.ready()
+            original_pids = [sentinel.pid, child.pid]
+            report["resident_pids"] = original_pids
+            epoch = activation
+            for cycle in range(3):
+                received = threading.Event()
+                item = {"cycle": cycle, "input_start": epoch.to_dict()}
+
+                def consume(event, cycle=cycle, epoch=epoch, item=item, received=received):
+                    if event.modality == "camera-init":
+                        if cycle:
+                            decoder.reset(epoch, event.payload)
+                            window.reset()
+                        else:
+                            decoder.configure(event.payload)
+                        return
+                    if event.modality in ("lidar", "pose"):
+                        window.append(
+                            normalized_sensor(
+                                event.modality,
+                                event.source_sequence,
+                                event.received_monotonic_ns,
+                                event.payload,
+                            )
+                        )
+                        return
+                    ticket = runtime.mailbox.reserve_ingress(1440000)
+                    target = None
+                    try:
+                        target = bytearray(1440000)
+                        decoded = decoder.decode(event.payload, target)
+                        binding = window.bind(event.received_monotonic_ns)
+                        assert binding.available and decoded["frame_index"] == 0
+                        if cycle:
+                            evidence = ResumeEvidence(
+                                event.received_monotonic_ns,
+                                window.pose.time_ns,
+                                binding.increments[0].time_ns,
+                                binding.increments[-1].time_ns,
+                                decoder.frames == 1,
+                            )
+                            # Synthetic temporal-state owner only; not JointGraph/TGS reset proof.
+                            runtime.resume_input(
+                                epoch, evidence, lambda: item.update(temporal_fixture_reset=True)
+                            )
+                        with runtime.work(epoch, "gpu"):
+                            item["bgr_sha256"] = hashlib.sha256(target).hexdigest()
+                        runtime.validate_result_binding(epoch.to_dict())
+                        window.finish_camera(event.received_monotonic_ns)
+                        item["completed_monotonic_ns"] = time.monotonic_ns()
+                        received.set()
+                    finally:
+                        target = None
+                        ticket.release()
+
+                left, right = socket.socketpair()
+                receiver = StreamingIngress(
+                    right,
+                    runtime,
+                    "fixture-acquisition",
+                    1,
+                    consume,
+                    lambda _: None,
+                    input_epoch=epoch,
+                )
+                receiver.start()
+                sender = StreamingSender(
+                    left,
+                    epoch,
+                    "fixture-acquisition",
+                    1,
+                    lambda epoch=epoch: runtime.check_input(epoch, synchronizing=True),
+                )
+                stamp = time.monotonic_ns()
+                payloads = [
+                    ("camera-init", "camera", init),
+                    ("pose", "pose", struct.pack("<7d", 0, 0, 0, 0, 0, 0, 1)),
+                    ("lidar", "points", struct.pack("<I3dB", 1, 1, 0, 0, 1)),
+                    ("camera-frame", "camera", fragment),
+                ]
+                for sequence, (modality, source, payload) in enumerate(payloads, 1):
+                    sender.send(
+                        LiveIngressEvent(
+                            sequence,
+                            "fixture-acquisition",
+                            1,
+                            modality,
+                            source,
+                            0,
+                            0,
+                            stamp,
+                            payload,
+                        )
+                    )
+                assert received.wait(2), receiver.error
+                assert sentinel.poll() is None and child.poll() is None
+                if cycle == 2:
+                    sender.end()
+                    assert receiver.join() and receiver.terminal == "end"
+                else:
+                    sender.close()  # No End: a temporary cable/route loss.
+                    assert receiver.join() and receiver.terminal == "paused"
+                    assert not runtime.stop_event.is_set() and not runtime.lease.released
+                    before = ticks[0]
+                    delay = 0.15 if cycle == 0 else 2.2
+                    assert not runtime.stop_event.wait(delay)
+                    item["outage_seconds"] = delay
+                    item["source_ticks_during_outage"] = ticks[0] - before
+                    assert item["source_ticks_during_outage"] > 0
+                    assert sentinel.poll() is None and child.poll() is None
+                    with_rejected = False
+                    try:
+                        runtime.validate_result_binding(epoch.to_dict())
+                    except StreamSuspended:
+                        with_rejected = True
+                    assert with_rejected
+                    epoch = runtime.begin_input(activation)
+                item["receiver"] = receiver.snapshot()
+                item["resident_pids_unchanged"] = original_pids == [sentinel.pid, child.pid]
+                report["cycles"].append(item)
+            assert len({item["bgr_sha256"] for item in report["cycles"]}) == 1
+            report["passed"] = True
+        finally:
+            finish.set()
+            thread.join(timeout=1)
+            if sender:
+                sender.close()
+            runtime.request_stop("completed" if report.get("passed") else "failed")
+            if receiver:
+                receiver.join(timeout=2)
+            runtime.stop_children()
+            if window:
+                window.close()
+            if decoder:
+                decoder.close()
+            report["released"] = runtime.close()
+            report["runtime"] = runtime.snapshot()
+            report["input_bytes"] = runtime.mailbox.bytes
+            report["peak_input_bytes"] = runtime.mailbox.peak_bytes
+            report["finished_monotonic_ns"] = time.monotonic_ns()
+            report["source_ticks_total"] = ticks[0]
+            report["full_graph_recovery_proved"] = False
+            (output / "report.json").write_text(json.dumps(report, indent=2) + "\n")
+            for process in runtime._children:
+                for stream in (process.stdin, process.stdout):
+                    if stream:
+                        stream.close()
+        assert report["released"] and not report["input_bytes"]
+        print(
+            json.dumps({"passed": True, "cycles": 3, "same_decoder_pid": True, "full_graph": False})
+        )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", required=True)
+    parser.add_argument("--camera", required=True)
+    args = parser.parse_args()
+    run(Path(args.output), Path(args.camera))

experiments/perception/worker/streaming_profile_stage1/pilot_fragment_decoder.py

@@ -20,6 +20,9 @@ def main():
             if header == {"op": "init"}:
                 decoder.configure(raw)
                 send(sys.stdout.buffer, {"initialized": True})
+            elif header == {"op": "reset"}:
+                decoder.reset(raw)
+                send(sys.stdout.buffer, {"reset": True})
             elif header == {"op": "decode"}:
                 image = decoder.decode(raw)
                 send(