feat(perception): gate streaming lifecycle on controller readiness
This commit is contained in:
@@ -23,6 +23,8 @@ from .realtime_contract import StreamStart
|
||||
from .realtime_scene import _wire_integer
|
||||
from .streaming_queue import IngressReservation, StreamBundle, StreamMailbox
|
||||
from .worker_lease import WorkerLease, WorkerLeaseError
|
||||
from .worker_operating_envelope import WorkerSnapshot
|
||||
from .worker_readiness import WorkerReadinessError, WorkerReadinessMonitor
|
||||
|
||||
|
||||
def _group_exists(pgid: int) -> bool:
|
||||
@@ -45,8 +47,12 @@ class StreamingLifecycle:
|
||||
*,
|
||||
ttl_seconds: float = 2.0,
|
||||
clock_ns: Callable[[], int] = time.monotonic_ns,
|
||||
readiness: WorkerReadinessMonitor | None = None,
|
||||
) -> None:
|
||||
self.start, self.mailbox, self.stop_event = start, mailbox, stop
|
||||
self._clock_ns, self._readiness = clock_ns, readiness
|
||||
if readiness is not None:
|
||||
readiness.check(start, now_monotonic_ns=clock_ns(), require_warmup=False)
|
||||
self._lock = threading.RLock()
|
||||
self._cleanup_lock = threading.Lock()
|
||||
self._children: list[subprocess.Popen[bytes]] = []
|
||||
@@ -75,6 +81,40 @@ class StreamingLifecycle:
|
||||
if start == self.start:
|
||||
self.request_stop("lease-lost")
|
||||
raise
|
||||
self._check_readiness(require_warmup=self.state == GraphState.RUNNING)
|
||||
|
||||
def _check_readiness(self, *, require_warmup: bool) -> None:
|
||||
if self._readiness is not None:
|
||||
try:
|
||||
self._readiness.check(
|
||||
self.start,
|
||||
now_monotonic_ns=self._clock_ns(),
|
||||
require_warmup=require_warmup,
|
||||
)
|
||||
except WorkerReadinessError:
|
||||
self.request_stop("worker-not-ready")
|
||||
raise
|
||||
|
||||
def observe_worker(self, start: StreamStart, observed: WorkerSnapshot) -> None:
|
||||
"""Trusted control plane only; sensor payloads never call this method.
|
||||
|
||||
This is not a lease renewal. A fresh inventory and an active controller
|
||||
heartbeat are independent requirements. No host operations under lock.
|
||||
"""
|
||||
with self._lock:
|
||||
self._check(start, starting=True)
|
||||
if self._readiness is None:
|
||||
raise WorkerReadinessError("worker readiness monitoring is not configured")
|
||||
try:
|
||||
self._readiness.observe(
|
||||
start,
|
||||
observed,
|
||||
now_monotonic_ns=self._clock_ns(),
|
||||
require_warmup=self.state == GraphState.RUNNING,
|
||||
)
|
||||
except WorkerReadinessError:
|
||||
self.request_stop("worker-not-ready")
|
||||
raise
|
||||
|
||||
def renew(self, start: StreamStart) -> None:
|
||||
with self._lock:
|
||||
@@ -89,6 +129,7 @@ class StreamingLifecycle:
|
||||
if any(p.poll() is not None for p in self._children):
|
||||
self.request_stop("child-exited")
|
||||
raise WorkerLeaseError("owned child exited during warmup")
|
||||
self._check_readiness(require_warmup=True)
|
||||
self.state = GraphState.RUNNING
|
||||
|
||||
def spawn(self, factory: Callable[[], subprocess.Popen[bytes]]) -> subprocess.Popen[bytes]:
|
||||
@@ -157,6 +198,7 @@ class StreamingLifecycle:
|
||||
"lease-lost",
|
||||
"child-exited",
|
||||
"invalid-process-group",
|
||||
"worker-not-ready",
|
||||
"failed",
|
||||
):
|
||||
raise ValueError("unknown runtime stop reason")
|
||||
@@ -262,4 +304,9 @@ class StreamingLifecycle:
|
||||
_wire_integer(self.retired_ns) if self.retired_ns is not None else None
|
||||
),
|
||||
"event_clock": "worker-process-monotonic",
|
||||
"worker_readiness": (
|
||||
self._readiness.snapshot()
|
||||
if self._readiness is not None
|
||||
else {"enabled": False, "realtime_qualified": False}
|
||||
),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user