SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(runtime): fail closed on offline artifact misses

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-30 22:19:01 +03:00
parent 81d7ad1a77
commit 8c91d40e2e
12 changed files with 510 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/web/app.py
+18 -14
View File
@@ -63,6 +63,10 @@ from k1link.web.plugin_runtime import (
)
from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root
from k1link.web.runtime_diagnostics import configure_scanner_diagnostics
from k1link.web.runtime_readiness import (
BackgroundReconcilerReadiness,
build_runtime_readiness,
)
from k1link.web.session_api import build_session_router
from k1link.web.system_telemetry_api import build_system_telemetry_router
from k1link.web.viewer_diagnostics_api import build_viewer_diagnostics_router
@@ -167,6 +171,7 @@ session_perception_epoch_store = (
else None
)
map_gateway_proxy = MapGatewayProxy(MapGatewayConfiguration.from_environment())
recording_reconciler_readiness = BackgroundReconcilerReadiness()
def _prepare_recorded_media_for_launch(
@@ -281,10 +286,11 @@ async def _recording_preparation_reconciler() -> None:
newly_finalized,
)
known_finalized = finalized
except Exception:
recording_reconciler_readiness.record_success()
except Exception as exc:
# A transient filesystem/catalog failure must not permanently
# disable preparation of sessions completed later in the run.
pass
recording_reconciler_readiness.record_failure(exc)
await asyncio.sleep(2.0)
@@ -339,18 +345,16 @@ async def request_validation_error_handler(
@app.get("/api/health")
def health() -> dict[str, Any]:
runtime_health = plugin_environment.runtime_health
runtimes_ready = all(item["status"] == "ready" for item in runtime_health)
return {
"ok": runtimes_ready,
"status": "ok" if runtimes_ready else "degraded",
"service": "mission-core-control-plane",
"version": __version__,
"plugin_runtimes": {
"ready": sum(item["status"] == "ready" for item in runtime_health),
"total": len(runtime_health),
},
}
return build_runtime_readiness(
version=__version__,
plugin_runtime_health=plugin_environment.runtime_health,
recording_materializer=session_recording_materializer,
artifact_gateway=session_artifact_gateway,
map_gateway_configured=map_gateway_proxy.configured,
ffmpeg_available=_ffmpeg is not None,
ffprobe_available=_ffprobe is not None,
reconciler=recording_reconciler_readiness,
)
@app.get("/api/v1/device-plugins")
src/k1link/web/runtime_readiness.py
+199
View File
@@ -0,0 +1,199 @@
"""Bounded, non-secret runtime readiness for the Mission Core control plane."""
from __future__ import annotations
import threading
from collections.abc import Iterable, Mapping
from datetime import UTC, datetime
from typing import Any
from k1link.artifact_gateway import ArtifactGateway
from k1link.sessions.recording import SessionRecordingMaterializer
class BackgroundReconcilerReadiness:
"""Track repeated catalog-reconciliation failures without exposing details."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._completed_runs = 0
self._consecutive_failures = 0
self._last_success_at_utc: str | None = None
self._last_failure_at_utc: str | None = None
self._last_failure_type: str | None = None
def record_success(self) -> None:
with self._lock:
self._completed_runs += 1
self._consecutive_failures = 0
self._last_success_at_utc = _utc_now_iso()
def record_failure(self, error: BaseException) -> None:
with self._lock:
self._completed_runs += 1
self._consecutive_failures += 1
self._last_failure_at_utc = _utc_now_iso()
self._last_failure_type = type(error).__name__
def snapshot(self) -> dict[str, object]:
with self._lock:
if self._completed_runs == 0:
status = "starting"
elif self._consecutive_failures == 0:
status = "ready"
else:
status = "degraded"
return {
"status": status,
"completed_runs": self._completed_runs,
"consecutive_failures": self._consecutive_failures,
"last_success_at_utc": self._last_success_at_utc,
"last_failure_at_utc": self._last_failure_at_utc,
"last_failure_type": self._last_failure_type,
}
def build_runtime_readiness(
*,
version: str,
plugin_runtime_health: Iterable[Mapping[str, Any]],
recording_materializer: SessionRecordingMaterializer,
artifact_gateway: ArtifactGateway | None,
map_gateway_configured: bool,
ffmpeg_available: bool,
ffprobe_available: bool,
reconciler: BackgroundReconcilerReadiness,
) -> dict[str, Any]:
"""Compose readiness from local components without network fan-out."""
runtime_health = tuple(plugin_runtime_health)
ready_runtime_count = sum(
item.get("status") == "ready" for item in runtime_health
)
plugin_ready = ready_runtime_count == len(runtime_health)
try:
recording = recording_materializer.cache_status()
recording_status = "ready"
recording_document: dict[str, object] = {
"status": recording_status,
"entries": recording.entry_count,
"bytes": recording.total_bytes,
"max_bytes": recording.cache_max_bytes,
"free_bytes": recording.free_bytes,
"free_space_reserve_bytes": recording.free_space_reserve_bytes,
}
recording_ready = (
recording.total_bytes <= recording.cache_max_bytes
and recording.free_bytes >= recording.free_space_reserve_bytes
)
if not recording_ready:
recording_document["status"] = "capacity-pressure"
except (OSError, RuntimeError):
recording_ready = False
recording_document = {"status": "unavailable"}
if artifact_gateway is None:
artifact_document: dict[str, object] = {
"mode": "standalone",
"status": "not-configured",
"central_available": None,
}
shared_artifacts_ready = False
shared_artifact_degraded = False
else:
try:
artifact = artifact_gateway.status()
cache = artifact.cache
artifact_document = {
"mode": "shared",
"status": (
"ready"
if artifact.central_status == "ready"
else (
"offline-cache-only"
if artifact.central_status == "unavailable"
else "invalid"
)
),
"central_available": artifact.central_status == "ready",
"cache": {
"objects": cache.object_count,
"bytes": cache.total_bytes,
"pinned_objects": cache.pinned_object_count,
"pinned_bytes": cache.pinned_bytes,
"max_bytes": cache.cache_max_bytes,
"free_space_reserve_bytes": cache.free_space_reserve_bytes,
},
}
shared_artifacts_ready = artifact.central_status == "ready"
shared_artifact_degraded = not shared_artifacts_ready
except (OSError, RuntimeError):
artifact_document = {
"mode": "shared",
"status": "unavailable",
"central_available": False,
}
shared_artifacts_ready = False
shared_artifact_degraded = True
media_tools_ready = ffmpeg_available and ffprobe_available
reconciler_document = reconciler.snapshot()
failure_count = reconciler_document["consecutive_failures"]
reconciler_failures = (
failure_count
if isinstance(failure_count, int) and not isinstance(failure_count, bool)
else 3
)
operational_ready = (
plugin_ready
and recording_ready
and media_tools_ready
and reconciler_failures < 3
)
degraded = (
shared_artifact_degraded
or reconciler_failures > 0
or not operational_ready
)
status = (
"unavailable"
if not operational_ready
else "degraded"
if degraded
else "ok"
)
return {
"ok": operational_ready,
"status": status,
"service": "mission-core-control-plane",
"version": version,
"readiness": {
"operational": operational_ready,
"production_shared_artifacts": (
operational_ready and shared_artifacts_ready
),
},
"plugin_runtimes": {
"ready": ready_runtime_count,
"total": len(runtime_health),
},
"components": {
"recording_cache": recording_document,
"artifact_store": artifact_document,
"background_reconciler": reconciler_document,
"media_tools": {
"status": "ready" if media_tools_ready else "unavailable",
"ffmpeg": ffmpeg_available,
"ffprobe": ffprobe_available,
},
"map_gateway": {
"status": "configured" if map_gateway_configured else "optional",
"configured": map_gateway_configured,
},
},
}
def _utc_now_iso() -> str:
return datetime.now(UTC).isoformat().replace("+00:00", "Z")