feat(perception): preserve resident runtime across recoverable input gaps

2026-09-02 15:27:06 +03:00
parent 06a66a3da8
commit c31c46caa5
14 changed files with 1041 additions and 33 deletions
@@ -21,10 +21,15 @@ from typing import Any, Literal
 from .graph_contracts import GraphState
 from .realtime_contract import StreamStart
 from .realtime_scene import _wire_integer
+from .streaming_continuity import InputContinuity, ResumeEvidence, StreamSuspended
 from .streaming_queue import IngressReservation, StreamBundle, StreamMailbox
 from .worker_lease import WorkerLease, WorkerLeaseError
 from .worker_operating_envelope import WorkerSnapshot
-from .worker_readiness import WorkerReadinessError, WorkerReadinessMonitor
+from .worker_readiness import (
+    WorkerReadinessError,
+    WorkerReadinessMonitor,
+    WorkerTelemetryUnavailable,
+)


 def _group_exists(pgid: int) -> bool:
@@ -48,16 +53,25 @@ class StreamingLifecycle:
        ttl_seconds: float = 2.0,
        clock_ns: Callable[[], int] = time.monotonic_ns,
        readiness: WorkerReadinessMonitor | None = None,
+        recover_input: bool = False,
+        source_clock_ns: Callable[[], int] | None = None,
    ) -> None:
        self.start, self.mailbox, self.stop_event = start, mailbox, stop
        self._clock_ns, self._readiness = clock_ns, readiness
+        if recover_input and (
+            source_clock_ns is None or (readiness is not None and not readiness.recoverable)
+        ):
+            raise ValueError("resumable input needs mapped source clock and recoverable telemetry")
+        self.continuity = InputContinuity(start) if recover_input else None
+        self._source_clock_ns = source_clock_ns
+        self._ingress_thread: threading.Thread | None = None
        if readiness is not None:
            readiness.check(start, now_monotonic_ns=clock_ns(), require_warmup=False)
        self._lock = threading.RLock()
        self._cleanup_lock = threading.Lock()
        self._children: list[subprocess.Popen[bytes]] = []
        self._threads: list[threading.Thread] = []
-        self._active = {"gpu": 0, "cpu": 0}
+        self._active = {"gpu": 0, "cpu": 0, "resync": 0}
        self.state = GraphState.CREATED
        self.reason: str | None = None
        self.stop_requested_ns: int | None = None
@@ -70,7 +84,9 @@ class StreamingLifecycle:
        )
        self._watchdog.start()

-    def _check(self, start: StreamStart, *, starting: bool = False) -> None:
+    def _check(
+        self, start: StreamStart, *, starting: bool = False, allow_unavailable: bool = False
+    ) -> None:
        allowed = (GraphState.STARTING, GraphState.RUNNING) if starting else (GraphState.RUNNING,)
        if self.state not in allowed or self.stop_event.is_set():
            raise WorkerLeaseError("runtime is not accepting work")
@@ -81,9 +97,11 @@ class StreamingLifecycle:
            if start == self.start:
                self.request_stop("lease-lost")
            raise
-        self._check_readiness(require_warmup=self.state == GraphState.RUNNING)
+        self._check_readiness(
+            require_warmup=self.state == GraphState.RUNNING, allow_unavailable=allow_unavailable
+        )

-    def _check_readiness(self, *, require_warmup: bool) -> None:
+    def _check_readiness(self, *, require_warmup: bool, allow_unavailable: bool = False) -> None:
        if self._readiness is not None:
            try:
                self._readiness.check(
@@ -91,6 +109,14 @@ class StreamingLifecycle:
                    now_monotonic_ns=self._clock_ns(),
                    require_warmup=require_warmup,
                )
+            except WorkerTelemetryUnavailable as exc:
+                if self.continuity is None:
+                    self.request_stop("worker-not-ready")
+                    raise
+                self.continuity.pause("worker-telemetry")
+                self.mailbox.pause()
+                if not allow_unavailable:
+                    raise StreamSuspended(str(exc)) from exc
            except WorkerReadinessError:
                self.request_stop("worker-not-ready")
                raise
@@ -102,7 +128,7 @@ class StreamingLifecycle:
        heartbeat are independent requirements. No host operations under lock.
        """
        with self._lock:
-            self._check(start, starting=True)
+            self._check(start, starting=True, allow_unavailable=True)
            if self._readiness is None:
                raise WorkerReadinessError("worker readiness monitoring is not configured")
            try:
@@ -112,13 +138,19 @@ class StreamingLifecycle:
                    now_monotonic_ns=self._clock_ns(),
                    require_warmup=self.state == GraphState.RUNNING,
                )
+            except WorkerTelemetryUnavailable:
+                if self.continuity is None:
+                    self.request_stop("worker-not-ready")
+                    raise
+                self.continuity.pause("worker-telemetry")
+                self.mailbox.pause()
            except WorkerReadinessError:
                self.request_stop("worker-not-ready")
                raise

    def renew(self, start: StreamStart) -> None:
        with self._lock:
-            self._check(start, starting=True)
+            self._check(start, starting=True, allow_unavailable=True)
            self.lease.renew(start)

    def ready(self) -> None:
@@ -148,32 +180,127 @@ class StreamingLifecycle:
    def track_thread(self, thread: threading.Thread) -> None:
        with self._lock:
            self._check(self.start, starting=True)
+            self._threads = [t for t in self._threads if t.ident is None or t.is_alive()]
            if len(self._threads) >= 8 or thread in self._threads:
                raise WorkerLeaseError("runtime thread registration outside bound")
            self._threads.append(thread)

-    def admit(self, start: StreamStart, bundle: StreamBundle) -> bool:
+    def register_ingress(self, thread: threading.Thread, epoch: StreamStart) -> None:
+        with self._lock:
+            if self.continuity is not None:
+                self.check_input(epoch, synchronizing=True)
+                if self._ingress_thread is not None and (
+                    self._ingress_thread.ident is None or self._ingress_thread.is_alive()
+                ):
+                    raise StreamSuspended("one input connection per resident profile")
+            self.track_thread(thread)
+            self._ingress_thread = thread
+
+    def check_input(
+        self, epoch: StreamStart, *, synchronizing: bool = False, event_ns: int | None = None
+    ) -> None:
+        with self._lock:
+            if self.continuity is None:
+                self._check(epoch)
+                return
+            self._check(self.start)
+            self.continuity.check(epoch, synchronizing=synchronizing)
+            if event_ns is not None:
+                assert self._source_clock_ns is not None
+                now = self._source_clock_ns()
+                if event_ns < self.continuity.cutoff_ns or not 0 <= now - event_ns <= 250_000_000:
+                    raise StreamSuspended("obsolete or future input, no backlog replay")
+
+    def pause_input(self, epoch: StreamStart, reason: str) -> None:
+        with self._lock:
+            if self.continuity is None:
+                raise ValueError("resumable input was not enabled")
+            # A disconnected old socket must not pause the replacement epoch.
+            if epoch != self.continuity.epoch:
+                raise StreamSuspended("obsolete input epoch")
+            self._check(self.start, allow_unavailable=True)
+            self.continuity.pause(reason)
+            self.mailbox.pause()
+
+    def begin_input(self, start: StreamStart) -> StreamStart:
        with self._lock:
            self._check(start)
+            if self.continuity is None or self._source_clock_ns is None:
+                raise ValueError("resumable input was not enabled")
+            if any(self._active.values()) or (
+                self._ingress_thread is not None and self._ingress_thread.is_alive()
+            ):
+                raise StreamSuspended("old epoch callbacks or connection still active")
+            if self.continuity.phase != "waiting":
+                raise StreamSuspended("pause required before a new input epoch")
+            self.mailbox.begin_epoch()
+            if self._ingress_thread is not None:
+                self._threads = [t for t in self._threads if t is not self._ingress_thread]
+                self._ingress_thread = None
+            return self.continuity.begin(self._source_clock_ns())
+
+    def resume_input(
+        self, epoch: StreamStart, evidence: ResumeEvidence, reset_temporal: Callable[[], None]
+    ) -> None:
+        """Trusted adapter proves a new decoded keyframe + current sensor pair.
+
+        Reset only temporal/rolling outputs, outside the lock, after old compute
+        has drained. No model restart. A failed reset is an actual runtime fault.
+        """
+        with self._lock:
+            self.check_input(epoch, synchronizing=True)
+            if (
+                self.continuity is None
+                or self.continuity.phase != "synchronizing"
+                or any(self._active.values())
+            ):
+                raise StreamSuspended("resynchronization is not quiescent")
+            assert self._source_clock_ns is not None
+            evidence.validate(
+                cutoff_ns=self.continuity.cutoff_ns, source_now_ns=self._source_clock_ns()
+            )
+            self._active["resync"] = 1
+        try:
+            reset_temporal()
+            with self._lock:
+                self.check_input(epoch, synchronizing=True)
+                evidence.validate(
+                    cutoff_ns=self.continuity.cutoff_ns, source_now_ns=self._source_clock_ns()
+                )
+                self.continuity.phase, self.continuity.reason = "active", None
+        except StreamSuspended:
+            raise
+        except BaseException:
+            self.request_stop("failed")
+            raise
+        finally:
+            with self._lock:
+                self._active["resync"] = 0
+
+    def admit(self, start: StreamStart, bundle: StreamBundle) -> bool:
+        with self._lock:
+            self.check_input(start)
            return self.mailbox.put(bundle)

    def admit_reserved(
        self, start: StreamStart, bundle: StreamBundle, reservation: IngressReservation
    ) -> bool:
        with self._lock:
-            self._check(start)
+            self.check_input(start)
            return self.mailbox.put_reserved(bundle, reservation)

    @contextmanager
    def work(self, start: StreamStart, lane: Literal["gpu", "cpu"]) -> Iterator[None]:
        with self._lock:
-            self._check(start)
+            self.check_input(start)
            if lane not in self._active or self._active[lane]:
                raise WorkerLeaseError("only one active call per compute lane")
            self._active[lane] += 1
        try:
            yield
-            self.check_current(start)
+            self.check_input(start)
+        except StreamSuspended:
+            raise  # The caller discards this epoch's result, not its models.
        except BaseException:
            self.request_stop("failed")
            raise
@@ -182,14 +309,17 @@ class StreamingLifecycle:
                self._active[lane] -= 1

    def check_current(self, start: StreamStart, *, starting: bool = False) -> None:
-        """Only child startup handshakes may opt into the warmup state."""
+        """Resource/IPC guard: drain active RPC on pause; work/result gates fence it.
+
+        Only child startup handshakes may opt into the warmup state.
+        """
        with self._lock:
-            self._check(start, starting=starting)
+            self._check(start, starting=starting, allow_unavailable=self.continuity is not None)

    def validate_result_binding(self, value: object) -> None:
        """Recheck at receipt/use; accepting a hash alone cannot renew authority."""
        start = StreamStart.from_dict(value)
-        self.check_current(start)
+        self.check_input(start)

    def request_stop(self, reason: str = "cancelled") -> None:
        if reason not in (
@@ -217,7 +347,7 @@ class StreamingLifecycle:
            try:
                with self._lock:
                    if self.state in (GraphState.STARTING, GraphState.RUNNING):
-                        self._check(self.start, starting=True)
+                        self._check(self.start, starting=True, allow_unavailable=True)
                        if any(p.poll() is not None for p in self._children):
                            self.request_stop("child-exited")
                    stopping = self.state == GraphState.STOPPING
@@ -304,6 +434,9 @@ class StreamingLifecycle:
                    _wire_integer(self.retired_ns) if self.retired_ns is not None else None
                ),
                "event_clock": "worker-process-monotonic",
+                "input_continuity": self.continuity.snapshot()
+                if self.continuity is not None
+                else {"enabled": False},
                "worker_readiness": (
                    self._readiness.snapshot()
                    if self._readiness is not None