Files
NODEDC_MISSION_CORE/src/k1link/web/runtime_readiness.py
T

197 lines
6.9 KiB
Python

"""Bounded, non-secret runtime readiness for the Mission Core control plane."""
from __future__ import annotations
import threading
from collections.abc import Iterable, Mapping
from datetime import UTC, datetime
from typing import Any
from k1link.artifact_gateway import ArtifactGateway
from k1link.sessions.recording import SessionRecordingMaterializer
class BackgroundReconcilerReadiness:
"""Track repeated catalog-reconciliation failures without exposing details."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._completed_runs = 0
self._consecutive_failures = 0
self._last_success_at_utc: str | None = None
self._last_failure_at_utc: str | None = None
self._last_failure_type: str | None = None
def record_success(self) -> None:
with self._lock:
self._completed_runs += 1
self._consecutive_failures = 0
self._last_success_at_utc = _utc_now_iso()
def record_failure(self, error: BaseException) -> None:
with self._lock:
self._completed_runs += 1
self._consecutive_failures += 1
self._last_failure_at_utc = _utc_now_iso()
self._last_failure_type = type(error).__name__
def snapshot(self) -> dict[str, object]:
with self._lock:
if self._completed_runs == 0:
status = "starting"
elif self._consecutive_failures == 0:
status = "ready"
else:
status = "degraded"
return {
"status": status,
"completed_runs": self._completed_runs,
"consecutive_failures": self._consecutive_failures,
"last_success_at_utc": self._last_success_at_utc,
"last_failure_at_utc": self._last_failure_at_utc,
"last_failure_type": self._last_failure_type,
}
def build_runtime_readiness(
*,
version: str,
plugin_runtime_health: Iterable[Mapping[str, Any]],
recording_materializer: SessionRecordingMaterializer,
artifact_gateway: ArtifactGateway | None,
map_gateway_configured: bool,
ffmpeg_available: bool,
ffprobe_available: bool,
reconciler: BackgroundReconcilerReadiness,
) -> dict[str, Any]:
"""Compose readiness from local components without network fan-out."""
runtime_health = tuple(plugin_runtime_health)
ready_runtime_count = sum(
item.get("status") == "ready" for item in runtime_health
)
plugin_ready = ready_runtime_count == len(runtime_health)
try:
recording = recording_materializer.cache_status()
recording_status = "ready"
recording_document: dict[str, object] = {
"status": recording_status,
"entries": recording.entry_count,
"bytes": recording.total_bytes,
"max_bytes": recording.cache_max_bytes,
"free_bytes": recording.free_bytes,
"free_space_reserve_bytes": recording.free_space_reserve_bytes,
}
recording_ready = (
recording.total_bytes <= recording.cache_max_bytes
and recording.free_bytes >= recording.free_space_reserve_bytes
)
if not recording_ready:
recording_document["status"] = "capacity-pressure"
except (OSError, RuntimeError):
recording_ready = False
recording_document = {"status": "unavailable"}
if artifact_gateway is None:
artifact_document: dict[str, object] = {
"mode": "standalone",
"status": "not-configured",
"central_available": None,
}
shared_artifacts_ready = False
shared_artifacts_configured = False
else:
shared_artifacts_configured = True
try:
artifact = artifact_gateway.status()
cache = artifact.cache
artifact_document = {
"mode": "shared",
"status": (
"ready"
if artifact.central_status == "ready"
else (
"offline-cache-only"
if artifact.central_status == "unavailable"
else "invalid"
)
),
"central_available": artifact.central_status == "ready",
"cache": {
"objects": cache.object_count,
"bytes": cache.total_bytes,
"pinned_objects": cache.pinned_object_count,
"pinned_bytes": cache.pinned_bytes,
"max_bytes": cache.cache_max_bytes,
"free_space_reserve_bytes": cache.free_space_reserve_bytes,
},
}
shared_artifacts_ready = artifact.central_status == "ready"
except (OSError, RuntimeError):
artifact_document = {
"mode": "shared",
"status": "unavailable",
"central_available": False,
}
shared_artifacts_ready = False
media_tools_ready = ffmpeg_available and ffprobe_available
reconciler_document = reconciler.snapshot()
failure_count = reconciler_document["consecutive_failures"]
reconciler_failures = (
failure_count
if isinstance(failure_count, int) and not isinstance(failure_count, bool)
else 3
)
operational_ready = (
plugin_ready
and recording_ready
and media_tools_ready
and reconciler_failures < 3
)
degraded = reconciler_failures > 0 or not operational_ready
status = (
"unavailable"
if not operational_ready
else "degraded"
if degraded
else "ok"
)
return {
"ok": operational_ready,
"status": status,
"service": "mission-core-control-plane",
"version": version,
"readiness": {
"operational": operational_ready,
"optional_shared_artifacts": {
"required": False,
"configured": shared_artifacts_configured,
"ready": shared_artifacts_ready,
},
},
"plugin_runtimes": {
"ready": ready_runtime_count,
"total": len(runtime_health),
},
"components": {
"recording_cache": recording_document,
"artifact_store": artifact_document,
"background_reconciler": reconciler_document,
"media_tools": {
"status": "ready" if media_tools_ready else "unavailable",
"ffmpeg": ffmpeg_available,
"ffprobe": ffprobe_available,
},
"map_gateway": {
"status": "configured" if map_gateway_configured else "optional",
"configured": map_gateway_configured,
},
},
}
def _utc_now_iso() -> str:
return datetime.now(UTC).isoformat().replace("+00:00", "Z")