SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(runtime): fail closed on offline artifact misses

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-30 22:19:01 +03:00
parent 81d7ad1a77
commit 8c91d40e2e
12 changed files with 510 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/artifact_gateway.py
+23 -1
View File
@@ -16,7 +16,7 @@ from collections.abc import Iterator, Mapping, Sequence
from contextlib import contextmanager, suppress
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Final
from typing import Any, Final, Literal
from uuid import uuid4
from k1link.artifacts import utc_now_iso
@@ -98,6 +98,12 @@ class ArtifactCacheStatus:
free_space_reserve_bytes: int
@dataclass(frozen=True, slots=True)
class ArtifactGatewayStatus:
central_status: Literal["ready", "unavailable", "invalid"]
cache: ArtifactCacheStatus
class CentralArtifactStore:
"""Immutable SHA-256 objects/manifests plus atomically replaceable named refs."""
@@ -617,6 +623,22 @@ class ArtifactGateway:
self.store = store
self.cache = cache
def status(self) -> ArtifactGatewayStatus:
"""Return a lightweight readiness snapshot without resolving an artifact."""
try:
self.store._require_root()
except ArtifactStoreUnavailable:
central_status: Literal["ready", "unavailable", "invalid"] = "unavailable"
except ArtifactIntegrityError:
central_status = "invalid"
else:
central_status = "ready"
return ArtifactGatewayStatus(
central_status=central_status,
cache=self.cache.status(),
)
def publish(
self,
*,
src/k1link/compute/integrated_perception.py
+6 -1
View File
@@ -560,8 +560,13 @@ class IntegratedPerceptionOverlayStore:
role = f"integrated-overlay:{recording_id}"
try:
resolved = self.artifact_gateway.resolve_role("sessions", session_id, role)
except (ArtifactNotFound, ArtifactStoreUnavailable):
except ArtifactNotFound:
return None
except ArtifactStoreUnavailable as exc:
raise RecordedPerceptionOverlayError(
"central integrated perception artifact is unavailable "
"and is not present in the local artifact cache"
) from exc
expected_result_id = resolved.manifest.metadata.get("integrated-result-id")
if (
resolved.member.media_type != "application/vnd.rerun.rrd"
src/k1link/sessions/recording.py
+36 -2
View File
@@ -43,6 +43,7 @@ RERUN_RECORDING_MEDIA_TYPE = "application/vnd.rerun.rrd"
RERUN_SESSION_TIMELINE = "session_time"
SESSION_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
DEFAULT_FREE_SPACE_RESERVE_BYTES = 2 * 1024 * 1024 * 1024
DEFAULT_CACHE_MAX_BYTES = 8 * 1024 * 1024 * 1024
RrdExporter = Callable[..., Mapping[str, object]]
RecordingProgressCallback = Callable[[str, float], None]
@@ -71,6 +72,15 @@ class MaterializedRecording:
timeline_end_ns: int
@dataclass(frozen=True, slots=True)
class RecordingCacheStatus:
entry_count: int
total_bytes: int
cache_max_bytes: int
free_space_reserve_bytes: int
free_bytes: int
@dataclass(frozen=True, slots=True)
class _ValidatedMemoryEntry:
source_identity: tuple[object, ...]
@@ -165,10 +175,15 @@ class SessionRecordingMaterializer:
self._exporters = dict(exporters or {})
if len(self._exporters) != len(set(self._exporters)):
raise ValueError("recording exporter plugin ids must be unique")
self.cache_max_bytes = _optional_positive_configuration(
configured_cache_max_bytes = _optional_positive_configuration(
cache_max_bytes,
environment_name="MISSIONCORE_RRD_CACHE_MAX_BYTES",
)
self.cache_max_bytes = (
configured_cache_max_bytes
if configured_cache_max_bytes is not None
else DEFAULT_CACHE_MAX_BYTES
)
self.free_space_reserve_bytes = _non_negative_configuration(
free_space_reserve_bytes,
environment_name="MISSIONCORE_RRD_FREE_SPACE_RESERVE_BYTES",
@@ -199,6 +214,20 @@ class SessionRecordingMaterializer:
_callable_accepts_keyword(exporter, "cancel_event") for exporter in exporters
)
def cache_status(self) -> RecordingCacheStatus:
"""Return bounded cache capacity and occupancy for runtime readiness."""
with self._cache_guard:
total_bytes, entries = _cache_entries(self.recordings_root)
free_bytes = shutil.disk_usage(self.recordings_root).free
return RecordingCacheStatus(
entry_count=len(entries),
total_bytes=total_bytes,
cache_max_bytes=self.cache_max_bytes,
free_space_reserve_bytes=self.free_space_reserve_bytes,
free_bytes=free_bytes,
)
def is_recording_available(self, recording: MaterializedRecording) -> bool:
"""Cheap no-follow check for a previously validated ready handle."""
@@ -409,8 +438,13 @@ class SessionRecordingMaterializer:
session_id,
"base-rrd",
)
except (ArtifactNotFound, ArtifactStoreUnavailable):
except ArtifactNotFound:
return None
except ArtifactStoreUnavailable as exc:
raise RecordingMaterializationError(
"central recording is unavailable and is not present "
"in the local artifact cache"
) from exc
except ArtifactGatewayError as exc:
raise RecordingMaterializationError(
"central recording artifact failed validation"
src/k1link/web/app.py
+18 -14
View File
@@ -63,6 +63,10 @@ from k1link.web.plugin_runtime import (
)
from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root
from k1link.web.runtime_diagnostics import configure_scanner_diagnostics
from k1link.web.runtime_readiness import (
BackgroundReconcilerReadiness,
build_runtime_readiness,
)
from k1link.web.session_api import build_session_router
from k1link.web.system_telemetry_api import build_system_telemetry_router
from k1link.web.viewer_diagnostics_api import build_viewer_diagnostics_router
@@ -167,6 +171,7 @@ session_perception_epoch_store = (
else None
)
map_gateway_proxy = MapGatewayProxy(MapGatewayConfiguration.from_environment())
recording_reconciler_readiness = BackgroundReconcilerReadiness()
def _prepare_recorded_media_for_launch(
@@ -281,10 +286,11 @@ async def _recording_preparation_reconciler() -> None:
newly_finalized,
)
known_finalized = finalized
except Exception:
recording_reconciler_readiness.record_success()
except Exception as exc:
# A transient filesystem/catalog failure must not permanently
# disable preparation of sessions completed later in the run.
pass
recording_reconciler_readiness.record_failure(exc)
await asyncio.sleep(2.0)
@@ -339,18 +345,16 @@ async def request_validation_error_handler(
@app.get("/api/health")
def health() -> dict[str, Any]:
runtime_health = plugin_environment.runtime_health
runtimes_ready = all(item["status"] == "ready" for item in runtime_health)
return {
"ok": runtimes_ready,
"status": "ok" if runtimes_ready else "degraded",
"service": "mission-core-control-plane",
"version": __version__,
"plugin_runtimes": {
"ready": sum(item["status"] == "ready" for item in runtime_health),
"total": len(runtime_health),
},
}
return build_runtime_readiness(
version=__version__,
plugin_runtime_health=plugin_environment.runtime_health,
recording_materializer=session_recording_materializer,
artifact_gateway=session_artifact_gateway,
map_gateway_configured=map_gateway_proxy.configured,
ffmpeg_available=_ffmpeg is not None,
ffprobe_available=_ffprobe is not None,
reconciler=recording_reconciler_readiness,
)
@app.get("/api/v1/device-plugins")
src/k1link/web/runtime_readiness.py
+199
View File
@@ -0,0 +1,199 @@
"""Bounded, non-secret runtime readiness for the Mission Core control plane."""
from __future__ import annotations
import threading
from collections.abc import Iterable, Mapping
from datetime import UTC, datetime
from typing import Any
from k1link.artifact_gateway import ArtifactGateway
from k1link.sessions.recording import SessionRecordingMaterializer
class BackgroundReconcilerReadiness:
"""Track repeated catalog-reconciliation failures without exposing details."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._completed_runs = 0
self._consecutive_failures = 0
self._last_success_at_utc: str | None = None
self._last_failure_at_utc: str | None = None
self._last_failure_type: str | None = None
def record_success(self) -> None:
with self._lock:
self._completed_runs += 1
self._consecutive_failures = 0
self._last_success_at_utc = _utc_now_iso()
def record_failure(self, error: BaseException) -> None:
with self._lock:
self._completed_runs += 1
self._consecutive_failures += 1
self._last_failure_at_utc = _utc_now_iso()
self._last_failure_type = type(error).__name__
def snapshot(self) -> dict[str, object]:
with self._lock:
if self._completed_runs == 0:
status = "starting"
elif self._consecutive_failures == 0:
status = "ready"
else:
status = "degraded"
return {
"status": status,
"completed_runs": self._completed_runs,
"consecutive_failures": self._consecutive_failures,
"last_success_at_utc": self._last_success_at_utc,
"last_failure_at_utc": self._last_failure_at_utc,
"last_failure_type": self._last_failure_type,
}
def build_runtime_readiness(
*,
version: str,
plugin_runtime_health: Iterable[Mapping[str, Any]],
recording_materializer: SessionRecordingMaterializer,
artifact_gateway: ArtifactGateway | None,
map_gateway_configured: bool,
ffmpeg_available: bool,
ffprobe_available: bool,
reconciler: BackgroundReconcilerReadiness,
) -> dict[str, Any]:
"""Compose readiness from local components without network fan-out."""
runtime_health = tuple(plugin_runtime_health)
ready_runtime_count = sum(
item.get("status") == "ready" for item in runtime_health
)
plugin_ready = ready_runtime_count == len(runtime_health)
try:
recording = recording_materializer.cache_status()
recording_status = "ready"
recording_document: dict[str, object] = {
"status": recording_status,
"entries": recording.entry_count,
"bytes": recording.total_bytes,
"max_bytes": recording.cache_max_bytes,
"free_bytes": recording.free_bytes,
"free_space_reserve_bytes": recording.free_space_reserve_bytes,
}
recording_ready = (
recording.total_bytes <= recording.cache_max_bytes
and recording.free_bytes >= recording.free_space_reserve_bytes
)
if not recording_ready:
recording_document["status"] = "capacity-pressure"
except (OSError, RuntimeError):
recording_ready = False
recording_document = {"status": "unavailable"}
if artifact_gateway is None:
artifact_document: dict[str, object] = {
"mode": "standalone",
"status": "not-configured",
"central_available": None,
}
shared_artifacts_ready = False
shared_artifact_degraded = False
else:
try:
artifact = artifact_gateway.status()
cache = artifact.cache
artifact_document = {
"mode": "shared",
"status": (
"ready"
if artifact.central_status == "ready"
else (
"offline-cache-only"
if artifact.central_status == "unavailable"
else "invalid"
)
),
"central_available": artifact.central_status == "ready",
"cache": {
"objects": cache.object_count,
"bytes": cache.total_bytes,
"pinned_objects": cache.pinned_object_count,
"pinned_bytes": cache.pinned_bytes,
"max_bytes": cache.cache_max_bytes,
"free_space_reserve_bytes": cache.free_space_reserve_bytes,
},
}
shared_artifacts_ready = artifact.central_status == "ready"
shared_artifact_degraded = not shared_artifacts_ready
except (OSError, RuntimeError):
artifact_document = {
"mode": "shared",
"status": "unavailable",
"central_available": False,
}
shared_artifacts_ready = False
shared_artifact_degraded = True
media_tools_ready = ffmpeg_available and ffprobe_available
reconciler_document = reconciler.snapshot()
failure_count = reconciler_document["consecutive_failures"]
reconciler_failures = (
failure_count
if isinstance(failure_count, int) and not isinstance(failure_count, bool)
else 3
)
operational_ready = (
plugin_ready
and recording_ready
and media_tools_ready
and reconciler_failures < 3
)
degraded = (
shared_artifact_degraded
or reconciler_failures > 0
or not operational_ready
)
status = (
"unavailable"
if not operational_ready
else "degraded"
if degraded
else "ok"
)
return {
"ok": operational_ready,
"status": status,
"service": "mission-core-control-plane",
"version": version,
"readiness": {
"operational": operational_ready,
"production_shared_artifacts": (
operational_ready and shared_artifacts_ready
),
},
"plugin_runtimes": {
"ready": ready_runtime_count,
"total": len(runtime_health),
},
"components": {
"recording_cache": recording_document,
"artifact_store": artifact_document,
"background_reconciler": reconciler_document,
"media_tools": {
"status": "ready" if media_tools_ready else "unavailable",
"ffmpeg": ffmpeg_available,
"ffprobe": ffprobe_available,
},
"map_gateway": {
"status": "configured" if map_gateway_configured else "optional",
"configured": map_gateway_configured,
},
},
}
def _utc_now_iso() -> str:
return datetime.now(UTC).isoformat().replace("+00:00", "Z")