feat(perception): fence streaming lifecycle with worker-local ownership
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,257 @@
|
||||
"""Lifecycle/fencing for one subprocess-backed full perception profile.
|
||||
|
||||
Uses existing GraphState, StreamStart and bounded scheduler primitives. The
|
||||
trusted controller supplies child commands and one Worker-wide lease directory;
|
||||
neither is accepted from stream payloads. No backend queue or model is created
|
||||
here. External-client inventory and real network authentication remain separate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager, suppress
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from .graph_contracts import GraphState
|
||||
from .realtime_contract import StreamStart
|
||||
from .realtime_scene import _wire_integer
|
||||
from .streaming_queue import StreamBundle, StreamMailbox
|
||||
from .worker_lease import WorkerLease, WorkerLeaseError
|
||||
|
||||
|
||||
def _group_exists(pgid: int) -> bool:
|
||||
try:
|
||||
os.killpg(pgid, 0)
|
||||
return True
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True # Unknown is not a release proof.
|
||||
|
||||
|
||||
class StreamingLifecycle:
|
||||
def __init__(
|
||||
self,
|
||||
start: StreamStart,
|
||||
lease_root: Path,
|
||||
mailbox: StreamMailbox,
|
||||
stop: threading.Event,
|
||||
*,
|
||||
ttl_seconds: float = 2.0,
|
||||
clock_ns: Callable[[], int] = time.monotonic_ns,
|
||||
) -> None:
|
||||
self.start, self.mailbox, self.stop_event = start, mailbox, stop
|
||||
self._lock = threading.RLock()
|
||||
self._cleanup_lock = threading.Lock()
|
||||
self._children: list[subprocess.Popen[bytes]] = []
|
||||
self._threads: list[threading.Thread] = []
|
||||
self._active = {"gpu": 0, "cpu": 0}
|
||||
self.state = GraphState.CREATED
|
||||
self.reason: str | None = None
|
||||
self.stop_requested_ns: int | None = None
|
||||
self.retired_ns: int | None = None
|
||||
self.lease = WorkerLease(lease_root, start, ttl_seconds=ttl_seconds, clock_ns=clock_ns)
|
||||
self.state = GraphState.STARTING
|
||||
self._watchdog_stop = threading.Event()
|
||||
self._watchdog = threading.Thread(
|
||||
target=self._watch, name="perception-lease-watchdog", daemon=True
|
||||
)
|
||||
self._watchdog.start()
|
||||
|
||||
def _check(self, start: StreamStart, *, starting: bool = False) -> None:
|
||||
allowed = (GraphState.STARTING, GraphState.RUNNING) if starting else (GraphState.RUNNING,)
|
||||
if self.state not in allowed or self.stop_event.is_set():
|
||||
raise WorkerLeaseError("runtime is not accepting work")
|
||||
try:
|
||||
self.lease.check(start)
|
||||
except WorkerLeaseError:
|
||||
# A stale client is rejected but cannot cancel the current owner.
|
||||
if start == self.start:
|
||||
self.request_stop("lease-lost")
|
||||
raise
|
||||
|
||||
def renew(self, start: StreamStart) -> None:
|
||||
with self._lock:
|
||||
self._check(start, starting=True)
|
||||
self.lease.renew(start)
|
||||
|
||||
def ready(self) -> None:
|
||||
with self._lock:
|
||||
self._check(self.start, starting=True)
|
||||
if self.state != GraphState.STARTING:
|
||||
raise WorkerLeaseError("warmup completion already consumed")
|
||||
if any(p.poll() is not None for p in self._children):
|
||||
self.request_stop("child-exited")
|
||||
raise WorkerLeaseError("owned child exited during warmup")
|
||||
self.state = GraphState.RUNNING
|
||||
|
||||
def spawn(self, factory: Callable[[], subprocess.Popen[bytes]]) -> subprocess.Popen[bytes]:
|
||||
"""Trusted, bounded process creation + registration, atomic with stop."""
|
||||
with self._lock:
|
||||
self._check(self.start, starting=True)
|
||||
if self.state != GraphState.STARTING or len(self._children) >= 8:
|
||||
raise WorkerLeaseError("model processes may start only once during warmup")
|
||||
process = factory()
|
||||
self._children.append(process)
|
||||
if process.poll() is None and os.getpgid(process.pid) != process.pid:
|
||||
self.request_stop("invalid-process-group")
|
||||
raise WorkerLeaseError("runtime children need dedicated process groups")
|
||||
return process
|
||||
|
||||
def track_thread(self, thread: threading.Thread) -> None:
|
||||
with self._lock:
|
||||
self._check(self.start, starting=True)
|
||||
if len(self._threads) >= 8 or thread in self._threads:
|
||||
raise WorkerLeaseError("runtime thread registration outside bound")
|
||||
self._threads.append(thread)
|
||||
|
||||
def admit(self, start: StreamStart, bundle: StreamBundle) -> bool:
|
||||
with self._lock:
|
||||
self._check(start)
|
||||
return self.mailbox.put(bundle)
|
||||
|
||||
@contextmanager
|
||||
def work(self, start: StreamStart, lane: Literal["gpu", "cpu"]) -> Iterator[None]:
|
||||
with self._lock:
|
||||
self._check(start)
|
||||
if lane not in self._active or self._active[lane]:
|
||||
raise WorkerLeaseError("only one active call per compute lane")
|
||||
self._active[lane] += 1
|
||||
try:
|
||||
yield
|
||||
self.check_current(start)
|
||||
except BaseException:
|
||||
self.request_stop("failed")
|
||||
raise
|
||||
finally:
|
||||
with self._lock:
|
||||
self._active[lane] -= 1
|
||||
|
||||
def check_current(self, start: StreamStart) -> None:
|
||||
with self._lock:
|
||||
self._check(start)
|
||||
|
||||
def validate_result_binding(self, value: object) -> None:
|
||||
"""Recheck at receipt/use; accepting a hash alone cannot renew authority."""
|
||||
start = StreamStart.from_dict(value)
|
||||
self.check_current(start)
|
||||
|
||||
def request_stop(self, reason: str = "cancelled") -> None:
|
||||
if reason not in (
|
||||
"completed",
|
||||
"cancelled",
|
||||
"lease-lost",
|
||||
"child-exited",
|
||||
"invalid-process-group",
|
||||
"failed",
|
||||
):
|
||||
raise ValueError("unknown runtime stop reason")
|
||||
with self._lock:
|
||||
if self.state in (GraphState.STOPPED, GraphState.CANCELLED, GraphState.FAILED):
|
||||
return
|
||||
self.reason = self.reason or reason
|
||||
if self.stop_requested_ns is None:
|
||||
self.stop_requested_ns = time.monotonic_ns()
|
||||
self.state = GraphState.STOPPING
|
||||
self.stop_event.set()
|
||||
self.mailbox.cancel()
|
||||
|
||||
def _watch(self) -> None:
|
||||
while not self._watchdog_stop.wait(0.05):
|
||||
try:
|
||||
with self._lock:
|
||||
if self.state in (GraphState.STARTING, GraphState.RUNNING):
|
||||
self._check(self.start, starting=True)
|
||||
if any(p.poll() is not None for p in self._children):
|
||||
self.request_stop("child-exited")
|
||||
stopping = self.state == GraphState.STOPPING
|
||||
if stopping:
|
||||
self.stop_children()
|
||||
return
|
||||
except (WorkerLeaseError, OSError):
|
||||
self.request_stop("lease-lost")
|
||||
self.stop_children()
|
||||
return
|
||||
|
||||
def stop_children(self) -> bool:
|
||||
"""Stop only owned groups. No Boolean resource-release attestation input."""
|
||||
with self._cleanup_lock:
|
||||
with self._lock:
|
||||
children = tuple(self._children)
|
||||
for sent_signal, grace in ((signal.SIGTERM, 4.0), (signal.SIGKILL, 1.0)):
|
||||
for process in reversed(children):
|
||||
if _group_exists(process.pid):
|
||||
with suppress(ProcessLookupError):
|
||||
os.killpg(process.pid, sent_signal)
|
||||
deadline = time.monotonic() + grace
|
||||
while True:
|
||||
complete = all(
|
||||
p.poll() is not None and not _group_exists(p.pid) for p in children
|
||||
)
|
||||
if complete or time.monotonic() >= deadline:
|
||||
break
|
||||
time.sleep(0.01)
|
||||
if complete:
|
||||
return True
|
||||
return False
|
||||
|
||||
def close(self, reason: str = "completed") -> bool:
|
||||
self.request_stop(reason)
|
||||
self._watchdog_stop.set()
|
||||
children_stopped = self.stop_children()
|
||||
if self._watchdog is not threading.current_thread():
|
||||
self._watchdog.join(timeout=0.1)
|
||||
with self._lock:
|
||||
if self.lease.released:
|
||||
return True
|
||||
# Caller joins/finishes its adapters; active callbacks and payloads
|
||||
# continue to fence even if all GPU subprocesses have died already.
|
||||
if (
|
||||
not children_stopped
|
||||
or any(self._active.values())
|
||||
or any(t.is_alive() for t in self._threads)
|
||||
or self._watchdog.is_alive()
|
||||
or not self.mailbox.quiescent
|
||||
):
|
||||
return False
|
||||
self.lease._retire_after_verified_stop()
|
||||
self.retired_ns = time.monotonic_ns()
|
||||
self.state = (
|
||||
GraphState.STOPPED
|
||||
if self.reason == "completed"
|
||||
else GraphState.CANCELLED
|
||||
if self.reason == "cancelled"
|
||||
else GraphState.FAILED
|
||||
)
|
||||
return True
|
||||
|
||||
def snapshot(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
return {
|
||||
"state": self.state.value,
|
||||
"reason": self.reason,
|
||||
"start": self.start.to_dict(),
|
||||
"active_calls": dict(self._active),
|
||||
"lease_released": self.lease.released,
|
||||
"input_payloads_released": self.mailbox.quiescent,
|
||||
"owned_children": len(self._children),
|
||||
"live_children": sum(p.poll() is None for p in self._children),
|
||||
"actuation_allowed": False,
|
||||
"lease_renewals": self.lease.renewals,
|
||||
"lease_deadline_monotonic_ns": _wire_integer(self.lease.deadline_ns),
|
||||
"stop_requested_monotonic_ns": (
|
||||
_wire_integer(self.stop_requested_ns)
|
||||
if self.stop_requested_ns is not None
|
||||
else None
|
||||
),
|
||||
"retired_monotonic_ns": (
|
||||
_wire_integer(self.retired_ns) if self.retired_ns is not None else None
|
||||
),
|
||||
"event_clock": "worker-process-monotonic",
|
||||
}
|
||||
Reference in New Issue
Block a user