197 lines
6.9 KiB
Python
197 lines
6.9 KiB
Python
"""Bounded, non-secret runtime readiness for the Mission Core control plane."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
from collections.abc import Iterable, Mapping
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from k1link.artifact_gateway import ArtifactGateway
|
|
from k1link.sessions.recording import SessionRecordingMaterializer
|
|
|
|
|
|
class BackgroundReconcilerReadiness:
|
|
"""Track repeated catalog-reconciliation failures without exposing details."""
|
|
|
|
def __init__(self) -> None:
|
|
self._lock = threading.Lock()
|
|
self._completed_runs = 0
|
|
self._consecutive_failures = 0
|
|
self._last_success_at_utc: str | None = None
|
|
self._last_failure_at_utc: str | None = None
|
|
self._last_failure_type: str | None = None
|
|
|
|
def record_success(self) -> None:
|
|
with self._lock:
|
|
self._completed_runs += 1
|
|
self._consecutive_failures = 0
|
|
self._last_success_at_utc = _utc_now_iso()
|
|
|
|
def record_failure(self, error: BaseException) -> None:
|
|
with self._lock:
|
|
self._completed_runs += 1
|
|
self._consecutive_failures += 1
|
|
self._last_failure_at_utc = _utc_now_iso()
|
|
self._last_failure_type = type(error).__name__
|
|
|
|
def snapshot(self) -> dict[str, object]:
|
|
with self._lock:
|
|
if self._completed_runs == 0:
|
|
status = "starting"
|
|
elif self._consecutive_failures == 0:
|
|
status = "ready"
|
|
else:
|
|
status = "degraded"
|
|
return {
|
|
"status": status,
|
|
"completed_runs": self._completed_runs,
|
|
"consecutive_failures": self._consecutive_failures,
|
|
"last_success_at_utc": self._last_success_at_utc,
|
|
"last_failure_at_utc": self._last_failure_at_utc,
|
|
"last_failure_type": self._last_failure_type,
|
|
}
|
|
|
|
|
|
def build_runtime_readiness(
|
|
*,
|
|
version: str,
|
|
plugin_runtime_health: Iterable[Mapping[str, Any]],
|
|
recording_materializer: SessionRecordingMaterializer,
|
|
artifact_gateway: ArtifactGateway | None,
|
|
map_gateway_configured: bool,
|
|
ffmpeg_available: bool,
|
|
ffprobe_available: bool,
|
|
reconciler: BackgroundReconcilerReadiness,
|
|
) -> dict[str, Any]:
|
|
"""Compose readiness from local components without network fan-out."""
|
|
|
|
runtime_health = tuple(plugin_runtime_health)
|
|
ready_runtime_count = sum(
|
|
item.get("status") == "ready" for item in runtime_health
|
|
)
|
|
plugin_ready = ready_runtime_count == len(runtime_health)
|
|
|
|
try:
|
|
recording = recording_materializer.cache_status()
|
|
recording_status = "ready"
|
|
recording_document: dict[str, object] = {
|
|
"status": recording_status,
|
|
"entries": recording.entry_count,
|
|
"bytes": recording.total_bytes,
|
|
"max_bytes": recording.cache_max_bytes,
|
|
"free_bytes": recording.free_bytes,
|
|
"free_space_reserve_bytes": recording.free_space_reserve_bytes,
|
|
}
|
|
recording_ready = (
|
|
recording.total_bytes <= recording.cache_max_bytes
|
|
and recording.free_bytes >= recording.free_space_reserve_bytes
|
|
)
|
|
if not recording_ready:
|
|
recording_document["status"] = "capacity-pressure"
|
|
except (OSError, RuntimeError):
|
|
recording_ready = False
|
|
recording_document = {"status": "unavailable"}
|
|
|
|
if artifact_gateway is None:
|
|
artifact_document: dict[str, object] = {
|
|
"mode": "standalone",
|
|
"status": "not-configured",
|
|
"central_available": None,
|
|
}
|
|
shared_artifacts_ready = False
|
|
shared_artifacts_configured = False
|
|
else:
|
|
shared_artifacts_configured = True
|
|
try:
|
|
artifact = artifact_gateway.status()
|
|
cache = artifact.cache
|
|
artifact_document = {
|
|
"mode": "shared",
|
|
"status": (
|
|
"ready"
|
|
if artifact.central_status == "ready"
|
|
else (
|
|
"offline-cache-only"
|
|
if artifact.central_status == "unavailable"
|
|
else "invalid"
|
|
)
|
|
),
|
|
"central_available": artifact.central_status == "ready",
|
|
"cache": {
|
|
"objects": cache.object_count,
|
|
"bytes": cache.total_bytes,
|
|
"pinned_objects": cache.pinned_object_count,
|
|
"pinned_bytes": cache.pinned_bytes,
|
|
"max_bytes": cache.cache_max_bytes,
|
|
"free_space_reserve_bytes": cache.free_space_reserve_bytes,
|
|
},
|
|
}
|
|
shared_artifacts_ready = artifact.central_status == "ready"
|
|
except (OSError, RuntimeError):
|
|
artifact_document = {
|
|
"mode": "shared",
|
|
"status": "unavailable",
|
|
"central_available": False,
|
|
}
|
|
shared_artifacts_ready = False
|
|
|
|
media_tools_ready = ffmpeg_available and ffprobe_available
|
|
reconciler_document = reconciler.snapshot()
|
|
failure_count = reconciler_document["consecutive_failures"]
|
|
reconciler_failures = (
|
|
failure_count
|
|
if isinstance(failure_count, int) and not isinstance(failure_count, bool)
|
|
else 3
|
|
)
|
|
operational_ready = (
|
|
plugin_ready
|
|
and recording_ready
|
|
and media_tools_ready
|
|
and reconciler_failures < 3
|
|
)
|
|
degraded = reconciler_failures > 0 or not operational_ready
|
|
status = (
|
|
"unavailable"
|
|
if not operational_ready
|
|
else "degraded"
|
|
if degraded
|
|
else "ok"
|
|
)
|
|
return {
|
|
"ok": operational_ready,
|
|
"status": status,
|
|
"service": "mission-core-control-plane",
|
|
"version": version,
|
|
"readiness": {
|
|
"operational": operational_ready,
|
|
"optional_shared_artifacts": {
|
|
"required": False,
|
|
"configured": shared_artifacts_configured,
|
|
"ready": shared_artifacts_ready,
|
|
},
|
|
},
|
|
"plugin_runtimes": {
|
|
"ready": ready_runtime_count,
|
|
"total": len(runtime_health),
|
|
},
|
|
"components": {
|
|
"recording_cache": recording_document,
|
|
"artifact_store": artifact_document,
|
|
"background_reconciler": reconciler_document,
|
|
"media_tools": {
|
|
"status": "ready" if media_tools_ready else "unavailable",
|
|
"ffmpeg": ffmpeg_available,
|
|
"ffprobe": ffprobe_available,
|
|
},
|
|
"map_gateway": {
|
|
"status": "configured" if map_gateway_configured else "optional",
|
|
"configured": map_gateway_configured,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def _utc_now_iso() -> str:
|
|
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|