SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): connect scoped host telemetry to recoverable profile lifecycle

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-09-02 16:50:57 +03:00
parent e91721fe6a
commit 61cbdb30a0
9 changed files with 895 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/perception/streaming_lifecycle.py
+22 -4
View File
@@ -113,8 +113,9 @@ class StreamingLifecycle:
if self.continuity is None:
self.request_stop("worker-not-ready")
raise
self.continuity.pause("worker-telemetry")
self.mailbox.pause()
if self.state == GraphState.RUNNING:
self.continuity.pause("worker-telemetry")
self.mailbox.pause()
if not allow_unavailable:
raise StreamSuspended(str(exc)) from exc
except WorkerReadinessError:
@@ -142,12 +143,29 @@ class StreamingLifecycle:
if self.continuity is None:
self.request_stop("worker-not-ready")
raise
self.continuity.pause("worker-telemetry")
self.mailbox.pause()
if self.state == GraphState.RUNNING:
self.continuity.pause("worker-telemetry")
self.mailbox.pause()
except WorkerReadinessError:
self.request_stop("worker-not-ready")
raise
def attach_readiness(self, monitor: WorkerReadinessMonitor) -> None:
"""Trusted bootstrap after acquiring the local lease, BEFORE any child.
Allows a real collector to report the acquired owner, not a fabricated
pre-lease claim. No GPU work may be spawned by this bootstrap adapter
before this call. Installation is one-shot and cannot weaken a monitor.
"""
with self._lock:
self._check(self.start, starting=True)
if self.state != GraphState.STARTING or self._children or self._readiness is not None:
raise WorkerReadinessError("readiness must be attached once before model startup")
if self.continuity is not None and not monitor.recoverable:
raise WorkerReadinessError("recoverable input requires recoverable readiness")
monitor.check(self.start, now_monotonic_ns=self._clock_ns(), require_warmup=False)
self._readiness = monitor
def renew(self, start: StreamStart) -> None:
with self._lock:
self._check(start, starting=True, allow_unavailable=True)
src/k1link/perception/worker_control.py
+206
View File
@@ -0,0 +1,206 @@
"""Bounded Worker-local host observation channel, separate from sensor ingress.
The launcher gives the controller a private request directory and a SEPARATE
read-only response mount. Only the trusted host collector writes responses;
neither a Docker socket nor host commands are exposed to the AI container.
This filesystem boundary is not authentication for a network/GCS connection.
One outstanding nonce requires a new host read after each request. Observation
time is the request's LOCAL monotonic start (a conservative lower bound), never
the reply's arrival or the foreign Windows clock. Delays cannot rejuvenate data.
"""
from __future__ import annotations
import hashlib
import json
import os
import secrets
import time
from collections.abc import Callable
from pathlib import Path
from typing import Any
from .realtime_contract import RealtimeContractError, StreamStart, _digest, _integer
from .worker_operating_envelope import WorkerSnapshot
CONTROL_SCHEMA = "missioncore.worker-host-observation/v1"
MAX_CONTROL_BYTES = 16384
INVENTORY_SCOPE = "docker-gpu-access"
_FACT_FIELDS = {
"worker_id",
"container_id",
"image_sha256",
"cpu_limit_millicores",
"memory_limit_mib",
"gpu_name",
"driver_version",
"sm_clock_mhz",
"memory_clock_mhz",
"gpu_telemetry_available",
"inventory_complete",
"competing_gpu_clients",
"inventory_scope",
"host_process_inventory",
}
def activation_digest(start: StreamStart) -> str:
return hashlib.sha256(
json.dumps(start.to_dict(), sort_keys=True, separators=(",", ":")).encode()
).hexdigest()
class WorkerControlChannel:
"""Single-reader/single-writer adapter owned by the local controller thread.
Paths and target container prefix come from the launcher, never a source
packet. Container identity is pinned on the first response and then exact.
Only 12..64 lowercase hex Docker IDs are accepted (not names).
"""
def __init__(
self,
start: StreamStart,
requests: Path,
responses: Path,
*,
container_id: str,
clock_domain_id: str,
maximum_age_ms: int = 1000,
clock_ns: Callable[[], int] = time.monotonic_ns,
) -> None:
if not 12 <= len(container_id) <= 64 or any(
c not in "0123456789abcdef" for c in container_id
):
raise ValueError("launcher must select a Docker ID, not a name")
if requests.resolve() == responses.resolve():
raise ValueError("request and read-only response mounts must be separate")
_integer(maximum_age_ms, "control maximum age", minimum=1)
self.start, self.activation = start, activation_digest(start)
self.requests, self.responses = requests, responses
self.container_id, self.clock_domain_id = container_id, clock_domain_id
self.clock_ns, self.maximum_age_ns = clock_ns, maximum_age_ms * 1_000_000
self.pending: dict[str, Any] | None = None
self.requested_ns = 0
self.sequence = self.accepted = self.expired = 0
self.last_error: str | None = None
self.last_roundtrip_ms: float | None = None
def request(self) -> None:
if self.pending is not None:
return
self.sequence += 1
self.requested_ns = self.clock_ns()
self.pending = {
"schema_version": CONTROL_SCHEMA,
"activation_sha256": self.activation,
"nonce": secrets.token_hex(32),
"sequence": self.sequence,
}
# Two fixed files; no queue or per-sample history. Requests contain no
# executable command, secret, source data or caller-selected target.
path = self.requests / "request.json"
temporary = self.requests / "request.tmp"
temporary.write_text(json.dumps(self.pending), encoding="utf-8")
os.replace(temporary, path)
def poll(self, *, owner: tuple[str, int], warmup_complete: bool) -> WorkerSnapshot | None:
if self.pending is None:
return None
now = self.clock_ns()
if now < self.requested_ns:
raise RealtimeContractError("controller clock moved backwards")
try:
with (self.responses / "response.json").open("rb") as stream:
raw = stream.read(MAX_CONTROL_BYTES + 1)
if len(raw) > MAX_CONTROL_BYTES:
raise ValueError("size")
reply = json.loads(raw.decode("utf-8-sig"))
if not isinstance(reply, dict) or set(reply) != set(self.pending) | {"facts"}:
raise ValueError("schema")
if any(reply[key] != value for key, value in self.pending.items()):
# Old/different activation cannot poison or refresh this owner.
raise ValueError("binding")
snapshot = self._snapshot(reply["facts"], owner, warmup_complete)
except (OSError, ValueError, TypeError, KeyError):
self.last_error = "response-unavailable-or-invalid"
if now - self.requested_ns > self.maximum_age_ns:
self.expired += 1
self.pending = None
return None
self.pending = None
self.last_roundtrip_ms = (now - self.requested_ns) / 1_000_000
if now - self.requested_ns > self.maximum_age_ns:
self.expired += 1
self.last_error = "response-expired"
# Preserve known conflicts even in a delayed reply; the monitor
# separately rejects its OLD age. It must never become fresh.
else:
self.last_error = None
self.accepted += 1
return snapshot
def _snapshot(self, facts: object, owner: tuple[str, int], warmup: bool) -> WorkerSnapshot:
if not isinstance(facts, dict) or set(facts) != _FACT_FIELDS:
raise ValueError("facts schema")
target = facts["container_id"]
if target is not None:
_digest(target, "container ID")
if not target.startswith(self.container_id):
# Do not disguise a changed container as a missing observation.
raise ContainerIdentityConflict("host observed another container")
if (
facts["inventory_scope"] != INVENTORY_SCOPE
or facts["host_process_inventory"] != "unproved"
):
raise ValueError("unsupported inventory claim")
if type(facts["inventory_complete"]) is not bool:
raise ValueError("inventory completeness")
clients = facts["competing_gpu_clients"]
if not isinstance(clients, list) or len(clients) > 64:
raise ValueError("inventory bound")
# Partial inventory may prove a conflict, but cannot prove its absence.
competitors = tuple(clients) if clients or facts["inventory_complete"] else None
snapshot = WorkerSnapshot(
worker_id=facts["worker_id"],
clock_domain_id=self.clock_domain_id,
observed_monotonic_ns=self.requested_ns,
gpu_name=facts["gpu_name"],
driver_version=facts["driver_version"],
image_sha256=facts["image_sha256"] if target else None,
effective_config_sha256=self.start.effective_config_sha256,
cpu_limit_millicores=facts["cpu_limit_millicores"],
memory_limit_mib=facts["memory_limit_mib"],
sm_clock_mhz=facts["sm_clock_mhz"],
memory_clock_mhz=facts["memory_clock_mhz"],
gpu_owner_run_id=owner[0],
lease_generation=owner[1],
competing_gpu_clients=competitors,
warmup_complete=warmup,
inventory_scope=INVENTORY_SCOPE,
gpu_telemetry_available=facts["gpu_telemetry_available"],
)
if target is not None:
self.container_id = target
return snapshot
def snapshot(self) -> dict[str, object]:
return {
"schema_version": CONTROL_SCHEMA,
"requests": self.sequence,
"accepted": self.accepted,
"expired": self.expired,
"pending": self.pending is not None,
"last_error": self.last_error,
"last_roundtrip_ms": self.last_roundtrip_ms,
"container_id": self.container_id,
"inventory_scope": INVENTORY_SCOPE,
"host_process_inventory": "unproved",
"network_authenticated": False,
"realtime_qualified": False,
}
class ContainerIdentityConflict(RuntimeError):
"""Trusted host returned a known different container; terminal, not a lag."""
src/k1link/perception/worker_control_pump.py
+107
View File
@@ -0,0 +1,107 @@
"""Local controller adapter: host I/O never holds lifecycle locks or renews lease."""
from __future__ import annotations
import threading
import time
from .streaming_continuity import StreamSuspended
from .streaming_lifecycle import StreamingLifecycle
from .worker_control import ContainerIdentityConflict, WorkerControlChannel
from .worker_operating_envelope import WorkerOperatingEnvelope, WorkerSnapshot
from .worker_readiness import ReadinessMode, WorkerReadinessMonitor, WorkerTelemetryUnavailable
class WorkerControlPump:
def __init__(
self,
runtime: StreamingLifecycle,
channel: WorkerControlChannel,
envelope: WorkerOperatingEnvelope,
*,
mode: ReadinessMode,
bootstrap_seconds: float = 5.0,
) -> None:
self.runtime, self.channel = runtime, channel
self.stop_event, self.warmed, self.warm_observed = (
threading.Event(),
threading.Event(),
threading.Event(),
)
self.io_failures = 0
self.error: str | None = None
deadline = time.monotonic() + bootstrap_seconds
initial = None
while initial is None and time.monotonic() < deadline:
initial = self._poll()
if initial is None:
self.stop_event.wait(0.025)
if initial is None:
raise WorkerTelemetryUnavailable(
"host collector bootstrap timed out; no models started"
)
runtime.attach_readiness(
WorkerReadinessMonitor(
runtime.start,
envelope,
initial,
mode=mode,
clock_domain_id=channel.clock_domain_id,
now_monotonic_ns=channel.clock_ns(),
recoverable=True,
)
)
self.thread = threading.Thread(target=self._run, name="worker-host-observer", daemon=True)
runtime.track_thread(self.thread)
self.thread.start()
def _poll(self) -> WorkerSnapshot | None:
# Owner/config/warmup come from THIS controller, not a host JSON reply
# or a source heartbeat. The lease is checked again on observe_worker.
self.runtime.check_current(self.runtime.start, starting=True)
try:
self.channel.request()
return self.channel.poll(
owner=(self.runtime.start.run_id, self.runtime.start.lease_generation),
warmup_complete=self.warmed.is_set(),
)
except OSError:
self.io_failures += 1
return None # The unchanged observation expires independently.
def _run(self) -> None:
try:
while not self.stop_event.wait(0.025) and not self.runtime.stop_event.is_set():
observed = self._poll()
if observed is not None:
self.runtime.observe_worker(self.runtime.start, observed)
if observed.warmup_complete:
self.warm_observed.set()
self.stop_event.wait(0.2)
except Exception as exc:
self.error = type(exc).__name__
self.runtime.request_stop(
"worker-container-conflict"
if isinstance(exc, ContainerIdentityConflict)
else "worker-control-failed"
)
def ready(self, seconds: float = 5.0) -> None:
self.warmed.set()
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
if self.warm_observed.wait(0.025):
try:
self.runtime.ready()
return
except StreamSuspended:
self.stop_event.wait(0.025)
raise WorkerTelemetryUnavailable("post-warmup host readiness not established")
def close(self) -> bool:
self.stop_event.set()
self.thread.join(timeout=1)
return not self.thread.is_alive()
def snapshot(self) -> dict[str, object]:
return {**self.channel.snapshot(), "io_failures": self.io_failures, "error": self.error}
src/k1link/perception/worker_operating_envelope.py
+14
View File
@@ -31,9 +31,13 @@ class WorkerOperatingEnvelope:
minimum_sm_clock_mhz: int
minimum_memory_clock_mhz: int
maximum_snapshot_age_ms: int = 1000
# A Docker inventory is NOT an audit of native host/WSL GPU processes.
inventory_scope: str = "host-compute"
def __post_init__(self) -> None:
_identifier(self.envelope_id, "envelope_id")
if self.inventory_scope not in ("host-compute", "docker-gpu-access"):
raise RealtimeContractError("unsupported inventory scope")
for field in ("gpu_name", "driver_version"):
value = getattr(self, field)
if not isinstance(value, str) or not value.strip() or len(value) > 160:
@@ -66,9 +70,15 @@ class WorkerSnapshot:
lease_generation: int | None
competing_gpu_clients: tuple[str, ...] | None
warmup_complete: bool | None
inventory_scope: str = "host-compute"
gpu_telemetry_available: bool = True
def __post_init__(self) -> None:
_identifier(self.worker_id, "worker_id")
if self.inventory_scope not in ("host-compute", "docker-gpu-access"):
raise RealtimeContractError("unsupported inventory scope")
if type(self.gpu_telemetry_available) is not bool:
raise RealtimeContractError("GPU telemetry availability must be boolean")
_identifier(self.clock_domain_id, "clock_domain_id")
_integer(self.observed_monotonic_ns, "observed_monotonic_ns")
for field in ("image_sha256", "effective_config_sha256"):
@@ -129,6 +139,10 @@ def operating_envelope_failures(
if age < 0:
raise RealtimeContractError("worker snapshot is from the future")
failures = []
if observed.inventory_scope != expected.inventory_scope:
failures.append("inventory-scope-mismatch")
if not observed.gpu_telemetry_available:
failures.append("gpu-telemetry-unavailable")
if age > expected.maximum_snapshot_age_ms * 1_000_000:
failures.append("worker-snapshot-expired")
for field in ("worker_id", "image_sha256", "effective_config_sha256"):
src/k1link/perception/worker_readiness.py
+5 -1
View File
@@ -107,7 +107,11 @@ class WorkerReadinessMonitor:
if self.recoverable:
# Missing metrics are not a proof that another process owns GPU.
# Known identity/conflict/clock failures still fence immediately.
uncertain = {"worker-snapshot-expired", "warmup-not-complete"}
uncertain = {
"worker-snapshot-expired",
"warmup-not-complete",
"gpu-telemetry-unavailable",
}
for field in ("image_sha256", "effective_config_sha256"):
if getattr(self._observed, field) is None:
uncertain.add(f"{field}-mismatch-or-unknown")