fix(runtime): fail closed on offline artifact misses
This commit is contained in:
@@ -224,7 +224,7 @@ a separately attested storage root:
|
|||||||
export MISSIONCORE_DATA_DIR=/absolute/private/path/mission-core
|
export MISSIONCORE_DATA_DIR=/absolute/private/path/mission-core
|
||||||
# Full K1 point-plus-camera evidence must currently remain below the checkout.
|
# Full K1 point-plus-camera evidence must currently remain below the checkout.
|
||||||
export MISSIONCORE_EVIDENCE_DIR=/absolute/path/to/NODEDC_MISSION_CORE/.runtime/mission-core/evidence/sessions
|
export MISSIONCORE_EVIDENCE_DIR=/absolute/path/to/NODEDC_MISSION_CORE/.runtime/mission-core/evidence/sessions
|
||||||
# Optional operator retention quota; unset means no application byte quota.
|
# Optional operator override; the application default is 8 GiB.
|
||||||
# export MISSIONCORE_RRD_CACHE_MAX_BYTES=8589934592
|
# export MISSIONCORE_RRD_CACHE_MAX_BYTES=8589934592
|
||||||
export MISSIONCORE_RRD_FREE_SPACE_RESERVE_BYTES=2147483648
|
export MISSIONCORE_RRD_FREE_SPACE_RESERVE_BYTES=2147483648
|
||||||
uv run k1link serve
|
uv run k1link serve
|
||||||
@@ -298,10 +298,11 @@ global cross-process export gate to cap concurrent RAM, CPU and temporary-disk
|
|||||||
use. Crash leftovers from candidates, exporter temporary files and staged replay
|
use. Crash leftovers from candidates, exporter temporary files and staged replay
|
||||||
prefixes are scavenged under that lock before capacity accounting. Ready cache
|
prefixes are scavenged under that lock before capacity accounting. Ready cache
|
||||||
hits and active response leases do not wait behind that gate. The derived cache
|
hits and active response leases do not wait behind that gate. The derived cache
|
||||||
has no application byte quota by default, so a single multi-hour RRD is not
|
has an 8 GiB application byte quota and a 2 GiB filesystem reserve by default.
|
||||||
rejected at 8 GiB. It still preserves a 2 GiB default filesystem reserve. An
|
`MISSIONCORE_RRD_CACHE_MAX_BYTES` can override the quota; LRU eviction still
|
||||||
operator may set `MISSIONCORE_RRD_CACHE_MAX_BYTES` to enable LRU eviction of
|
affects derived RRDs only, and native evidence is never deleted. A single
|
||||||
derived RRDs only; native evidence is never deleted.
|
derived RRD larger than the configured quota fails explicitly instead of
|
||||||
|
consuming the operator host without a bound.
|
||||||
|
|
||||||
## Capture-clock envelope
|
## Capture-clock envelope
|
||||||
|
|
||||||
|
|||||||
@@ -63,9 +63,10 @@ gateway remains a same-process data-plane route; portable stream/media IPC is a
|
|||||||
separate future gate.
|
separate future gate.
|
||||||
|
|
||||||
`GET /api/v1/device-plugin-runtimes` exposes the small lifecycle snapshots. The
|
`GET /api/v1/device-plugin-runtimes` exposes the small lifecycle snapshots. The
|
||||||
main health response reports ready/total runtime counts. These values do not
|
main health response includes ready/total runtime counts as one component beside
|
||||||
claim device connectivity, sensor health, process containment or durable
|
recording-cache, shared-artifact, media-tool and catalog-reconciler readiness.
|
||||||
supervision.
|
The plugin values still do not claim device connectivity, sensor health,
|
||||||
|
process containment or durable supervision.
|
||||||
|
|
||||||
## Acceptance
|
## Acceptance
|
||||||
|
|
||||||
|
|||||||
@@ -77,6 +77,11 @@ Contour health reads only live contracts:
|
|||||||
- `GET /api/v1/polygon/worker`;
|
- `GET /api/v1/polygon/worker`;
|
||||||
- current Mission Runtime state for active device and perception metrics.
|
- current Mission Runtime state for active device and perception metrics.
|
||||||
|
|
||||||
|
The base health contract now separates local operational readiness from
|
||||||
|
production shared-artifact readiness and reports bounded cache, catalog
|
||||||
|
reconciler, media-tool and Map Gateway configuration states. It performs no
|
||||||
|
remote fan-out and never invents Worker 006 or device health.
|
||||||
|
|
||||||
Unknown or unavailable values remain empty/offline. The UI does not invent
|
Unknown or unavailable values remain empty/offline. The UI does not invent
|
||||||
load, latency, activity or connected devices.
|
load, latency, activity or connected devices.
|
||||||
|
|
||||||
|
|||||||
@@ -84,6 +84,10 @@ The integrated-perception endpoint resolves
|
|||||||
results. A sealed central/local CAS hit therefore needs neither Worker 006 nor
|
results. A sealed central/local CAS hit therefore needs neither Worker 006 nor
|
||||||
the original E10 result tree. Invalid central metadata fails closed; a genuinely
|
the original E10 result tree. Invalid central metadata fails closed; a genuinely
|
||||||
absent central role can still use the existing local materialization path.
|
absent central role can still use the existing local materialization path.
|
||||||
|
When the central store is unavailable, however, a missing local CAS role is not
|
||||||
|
treated as an absent role: base RRD export and overlay rendering both fail with
|
||||||
|
an explicit offline-cache error. This prevents one disconnected operator host
|
||||||
|
from silently producing a second derived truth.
|
||||||
|
|
||||||
## Persistent pins and offline use
|
## Persistent pins and offline use
|
||||||
|
|
||||||
@@ -105,7 +109,15 @@ When the central root is unavailable:
|
|||||||
|
|
||||||
- a sealed local reference, manifest, and object resolve normally;
|
- a sealed local reference, manifest, and object resolve normally;
|
||||||
- an absent local object returns an explicit offline-cache error;
|
- an absent local object returns an explicit offline-cache error;
|
||||||
- Mission Core does not silently fetch from Worker 006 or start a heavy AI run.
|
- Mission Core does not invoke the local RRD exporter, scan stale worker
|
||||||
|
results, render an overlay, fetch from Worker 006 or start a heavy AI run.
|
||||||
|
|
||||||
|
`GET /api/health` exposes the effective shared/standalone mode, central-store
|
||||||
|
availability, bounded CAS and RRD-cache occupancy, media-tool availability and
|
||||||
|
the background catalog reconciler state without disclosing filesystem paths.
|
||||||
|
Operational readiness remains distinct from production shared-artifact
|
||||||
|
readiness so an offline, already pinned field host stays observable rather than
|
||||||
|
being mistaken for a fully connected host.
|
||||||
|
|
||||||
## RAVNOVES00 migration acceptance
|
## RAVNOVES00 migration acceptance
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from collections.abc import Iterator, Mapping, Sequence
|
|||||||
from contextlib import contextmanager, suppress
|
from contextlib import contextmanager, suppress
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Final
|
from typing import Any, Final, Literal
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from k1link.artifacts import utc_now_iso
|
from k1link.artifacts import utc_now_iso
|
||||||
@@ -98,6 +98,12 @@ class ArtifactCacheStatus:
|
|||||||
free_space_reserve_bytes: int
|
free_space_reserve_bytes: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ArtifactGatewayStatus:
|
||||||
|
central_status: Literal["ready", "unavailable", "invalid"]
|
||||||
|
cache: ArtifactCacheStatus
|
||||||
|
|
||||||
|
|
||||||
class CentralArtifactStore:
|
class CentralArtifactStore:
|
||||||
"""Immutable SHA-256 objects/manifests plus atomically replaceable named refs."""
|
"""Immutable SHA-256 objects/manifests plus atomically replaceable named refs."""
|
||||||
|
|
||||||
@@ -617,6 +623,22 @@ class ArtifactGateway:
|
|||||||
self.store = store
|
self.store = store
|
||||||
self.cache = cache
|
self.cache = cache
|
||||||
|
|
||||||
|
def status(self) -> ArtifactGatewayStatus:
|
||||||
|
"""Return a lightweight readiness snapshot without resolving an artifact."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.store._require_root()
|
||||||
|
except ArtifactStoreUnavailable:
|
||||||
|
central_status: Literal["ready", "unavailable", "invalid"] = "unavailable"
|
||||||
|
except ArtifactIntegrityError:
|
||||||
|
central_status = "invalid"
|
||||||
|
else:
|
||||||
|
central_status = "ready"
|
||||||
|
return ArtifactGatewayStatus(
|
||||||
|
central_status=central_status,
|
||||||
|
cache=self.cache.status(),
|
||||||
|
)
|
||||||
|
|
||||||
def publish(
|
def publish(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
|
|||||||
@@ -560,8 +560,13 @@ class IntegratedPerceptionOverlayStore:
|
|||||||
role = f"integrated-overlay:{recording_id}"
|
role = f"integrated-overlay:{recording_id}"
|
||||||
try:
|
try:
|
||||||
resolved = self.artifact_gateway.resolve_role("sessions", session_id, role)
|
resolved = self.artifact_gateway.resolve_role("sessions", session_id, role)
|
||||||
except (ArtifactNotFound, ArtifactStoreUnavailable):
|
except ArtifactNotFound:
|
||||||
return None
|
return None
|
||||||
|
except ArtifactStoreUnavailable as exc:
|
||||||
|
raise RecordedPerceptionOverlayError(
|
||||||
|
"central integrated perception artifact is unavailable "
|
||||||
|
"and is not present in the local artifact cache"
|
||||||
|
) from exc
|
||||||
expected_result_id = resolved.manifest.metadata.get("integrated-result-id")
|
expected_result_id = resolved.manifest.metadata.get("integrated-result-id")
|
||||||
if (
|
if (
|
||||||
resolved.member.media_type != "application/vnd.rerun.rrd"
|
resolved.member.media_type != "application/vnd.rerun.rrd"
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ RERUN_RECORDING_MEDIA_TYPE = "application/vnd.rerun.rrd"
|
|||||||
RERUN_SESSION_TIMELINE = "session_time"
|
RERUN_SESSION_TIMELINE = "session_time"
|
||||||
SESSION_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
SESSION_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||||
DEFAULT_FREE_SPACE_RESERVE_BYTES = 2 * 1024 * 1024 * 1024
|
DEFAULT_FREE_SPACE_RESERVE_BYTES = 2 * 1024 * 1024 * 1024
|
||||||
|
DEFAULT_CACHE_MAX_BYTES = 8 * 1024 * 1024 * 1024
|
||||||
|
|
||||||
RrdExporter = Callable[..., Mapping[str, object]]
|
RrdExporter = Callable[..., Mapping[str, object]]
|
||||||
RecordingProgressCallback = Callable[[str, float], None]
|
RecordingProgressCallback = Callable[[str, float], None]
|
||||||
@@ -71,6 +72,15 @@ class MaterializedRecording:
|
|||||||
timeline_end_ns: int
|
timeline_end_ns: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class RecordingCacheStatus:
|
||||||
|
entry_count: int
|
||||||
|
total_bytes: int
|
||||||
|
cache_max_bytes: int
|
||||||
|
free_space_reserve_bytes: int
|
||||||
|
free_bytes: int
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class _ValidatedMemoryEntry:
|
class _ValidatedMemoryEntry:
|
||||||
source_identity: tuple[object, ...]
|
source_identity: tuple[object, ...]
|
||||||
@@ -165,10 +175,15 @@ class SessionRecordingMaterializer:
|
|||||||
self._exporters = dict(exporters or {})
|
self._exporters = dict(exporters or {})
|
||||||
if len(self._exporters) != len(set(self._exporters)):
|
if len(self._exporters) != len(set(self._exporters)):
|
||||||
raise ValueError("recording exporter plugin ids must be unique")
|
raise ValueError("recording exporter plugin ids must be unique")
|
||||||
self.cache_max_bytes = _optional_positive_configuration(
|
configured_cache_max_bytes = _optional_positive_configuration(
|
||||||
cache_max_bytes,
|
cache_max_bytes,
|
||||||
environment_name="MISSIONCORE_RRD_CACHE_MAX_BYTES",
|
environment_name="MISSIONCORE_RRD_CACHE_MAX_BYTES",
|
||||||
)
|
)
|
||||||
|
self.cache_max_bytes = (
|
||||||
|
configured_cache_max_bytes
|
||||||
|
if configured_cache_max_bytes is not None
|
||||||
|
else DEFAULT_CACHE_MAX_BYTES
|
||||||
|
)
|
||||||
self.free_space_reserve_bytes = _non_negative_configuration(
|
self.free_space_reserve_bytes = _non_negative_configuration(
|
||||||
free_space_reserve_bytes,
|
free_space_reserve_bytes,
|
||||||
environment_name="MISSIONCORE_RRD_FREE_SPACE_RESERVE_BYTES",
|
environment_name="MISSIONCORE_RRD_FREE_SPACE_RESERVE_BYTES",
|
||||||
@@ -199,6 +214,20 @@ class SessionRecordingMaterializer:
|
|||||||
_callable_accepts_keyword(exporter, "cancel_event") for exporter in exporters
|
_callable_accepts_keyword(exporter, "cancel_event") for exporter in exporters
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def cache_status(self) -> RecordingCacheStatus:
|
||||||
|
"""Return bounded cache capacity and occupancy for runtime readiness."""
|
||||||
|
|
||||||
|
with self._cache_guard:
|
||||||
|
total_bytes, entries = _cache_entries(self.recordings_root)
|
||||||
|
free_bytes = shutil.disk_usage(self.recordings_root).free
|
||||||
|
return RecordingCacheStatus(
|
||||||
|
entry_count=len(entries),
|
||||||
|
total_bytes=total_bytes,
|
||||||
|
cache_max_bytes=self.cache_max_bytes,
|
||||||
|
free_space_reserve_bytes=self.free_space_reserve_bytes,
|
||||||
|
free_bytes=free_bytes,
|
||||||
|
)
|
||||||
|
|
||||||
def is_recording_available(self, recording: MaterializedRecording) -> bool:
|
def is_recording_available(self, recording: MaterializedRecording) -> bool:
|
||||||
"""Cheap no-follow check for a previously validated ready handle."""
|
"""Cheap no-follow check for a previously validated ready handle."""
|
||||||
|
|
||||||
@@ -409,8 +438,13 @@ class SessionRecordingMaterializer:
|
|||||||
session_id,
|
session_id,
|
||||||
"base-rrd",
|
"base-rrd",
|
||||||
)
|
)
|
||||||
except (ArtifactNotFound, ArtifactStoreUnavailable):
|
except ArtifactNotFound:
|
||||||
return None
|
return None
|
||||||
|
except ArtifactStoreUnavailable as exc:
|
||||||
|
raise RecordingMaterializationError(
|
||||||
|
"central recording is unavailable and is not present "
|
||||||
|
"in the local artifact cache"
|
||||||
|
) from exc
|
||||||
except ArtifactGatewayError as exc:
|
except ArtifactGatewayError as exc:
|
||||||
raise RecordingMaterializationError(
|
raise RecordingMaterializationError(
|
||||||
"central recording artifact failed validation"
|
"central recording artifact failed validation"
|
||||||
|
|||||||
+18
-14
@@ -63,6 +63,10 @@ from k1link.web.plugin_runtime import (
|
|||||||
)
|
)
|
||||||
from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root
|
from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root
|
||||||
from k1link.web.runtime_diagnostics import configure_scanner_diagnostics
|
from k1link.web.runtime_diagnostics import configure_scanner_diagnostics
|
||||||
|
from k1link.web.runtime_readiness import (
|
||||||
|
BackgroundReconcilerReadiness,
|
||||||
|
build_runtime_readiness,
|
||||||
|
)
|
||||||
from k1link.web.session_api import build_session_router
|
from k1link.web.session_api import build_session_router
|
||||||
from k1link.web.system_telemetry_api import build_system_telemetry_router
|
from k1link.web.system_telemetry_api import build_system_telemetry_router
|
||||||
from k1link.web.viewer_diagnostics_api import build_viewer_diagnostics_router
|
from k1link.web.viewer_diagnostics_api import build_viewer_diagnostics_router
|
||||||
@@ -167,6 +171,7 @@ session_perception_epoch_store = (
|
|||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
map_gateway_proxy = MapGatewayProxy(MapGatewayConfiguration.from_environment())
|
map_gateway_proxy = MapGatewayProxy(MapGatewayConfiguration.from_environment())
|
||||||
|
recording_reconciler_readiness = BackgroundReconcilerReadiness()
|
||||||
|
|
||||||
|
|
||||||
def _prepare_recorded_media_for_launch(
|
def _prepare_recorded_media_for_launch(
|
||||||
@@ -281,10 +286,11 @@ async def _recording_preparation_reconciler() -> None:
|
|||||||
newly_finalized,
|
newly_finalized,
|
||||||
)
|
)
|
||||||
known_finalized = finalized
|
known_finalized = finalized
|
||||||
except Exception:
|
recording_reconciler_readiness.record_success()
|
||||||
|
except Exception as exc:
|
||||||
# A transient filesystem/catalog failure must not permanently
|
# A transient filesystem/catalog failure must not permanently
|
||||||
# disable preparation of sessions completed later in the run.
|
# disable preparation of sessions completed later in the run.
|
||||||
pass
|
recording_reconciler_readiness.record_failure(exc)
|
||||||
await asyncio.sleep(2.0)
|
await asyncio.sleep(2.0)
|
||||||
|
|
||||||
|
|
||||||
@@ -339,18 +345,16 @@ async def request_validation_error_handler(
|
|||||||
|
|
||||||
@app.get("/api/health")
|
@app.get("/api/health")
|
||||||
def health() -> dict[str, Any]:
|
def health() -> dict[str, Any]:
|
||||||
runtime_health = plugin_environment.runtime_health
|
return build_runtime_readiness(
|
||||||
runtimes_ready = all(item["status"] == "ready" for item in runtime_health)
|
version=__version__,
|
||||||
return {
|
plugin_runtime_health=plugin_environment.runtime_health,
|
||||||
"ok": runtimes_ready,
|
recording_materializer=session_recording_materializer,
|
||||||
"status": "ok" if runtimes_ready else "degraded",
|
artifact_gateway=session_artifact_gateway,
|
||||||
"service": "mission-core-control-plane",
|
map_gateway_configured=map_gateway_proxy.configured,
|
||||||
"version": __version__,
|
ffmpeg_available=_ffmpeg is not None,
|
||||||
"plugin_runtimes": {
|
ffprobe_available=_ffprobe is not None,
|
||||||
"ready": sum(item["status"] == "ready" for item in runtime_health),
|
reconciler=recording_reconciler_readiness,
|
||||||
"total": len(runtime_health),
|
)
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@app.get("/api/v1/device-plugins")
|
@app.get("/api/v1/device-plugins")
|
||||||
|
|||||||
@@ -0,0 +1,199 @@
|
|||||||
|
"""Bounded, non-secret runtime readiness for the Mission Core control plane."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import threading
|
||||||
|
from collections.abc import Iterable, Mapping
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from k1link.artifact_gateway import ArtifactGateway
|
||||||
|
from k1link.sessions.recording import SessionRecordingMaterializer
|
||||||
|
|
||||||
|
|
||||||
|
class BackgroundReconcilerReadiness:
|
||||||
|
"""Track repeated catalog-reconciliation failures without exposing details."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._completed_runs = 0
|
||||||
|
self._consecutive_failures = 0
|
||||||
|
self._last_success_at_utc: str | None = None
|
||||||
|
self._last_failure_at_utc: str | None = None
|
||||||
|
self._last_failure_type: str | None = None
|
||||||
|
|
||||||
|
def record_success(self) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._completed_runs += 1
|
||||||
|
self._consecutive_failures = 0
|
||||||
|
self._last_success_at_utc = _utc_now_iso()
|
||||||
|
|
||||||
|
def record_failure(self, error: BaseException) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._completed_runs += 1
|
||||||
|
self._consecutive_failures += 1
|
||||||
|
self._last_failure_at_utc = _utc_now_iso()
|
||||||
|
self._last_failure_type = type(error).__name__
|
||||||
|
|
||||||
|
def snapshot(self) -> dict[str, object]:
|
||||||
|
with self._lock:
|
||||||
|
if self._completed_runs == 0:
|
||||||
|
status = "starting"
|
||||||
|
elif self._consecutive_failures == 0:
|
||||||
|
status = "ready"
|
||||||
|
else:
|
||||||
|
status = "degraded"
|
||||||
|
return {
|
||||||
|
"status": status,
|
||||||
|
"completed_runs": self._completed_runs,
|
||||||
|
"consecutive_failures": self._consecutive_failures,
|
||||||
|
"last_success_at_utc": self._last_success_at_utc,
|
||||||
|
"last_failure_at_utc": self._last_failure_at_utc,
|
||||||
|
"last_failure_type": self._last_failure_type,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def build_runtime_readiness(
|
||||||
|
*,
|
||||||
|
version: str,
|
||||||
|
plugin_runtime_health: Iterable[Mapping[str, Any]],
|
||||||
|
recording_materializer: SessionRecordingMaterializer,
|
||||||
|
artifact_gateway: ArtifactGateway | None,
|
||||||
|
map_gateway_configured: bool,
|
||||||
|
ffmpeg_available: bool,
|
||||||
|
ffprobe_available: bool,
|
||||||
|
reconciler: BackgroundReconcilerReadiness,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Compose readiness from local components without network fan-out."""
|
||||||
|
|
||||||
|
runtime_health = tuple(plugin_runtime_health)
|
||||||
|
ready_runtime_count = sum(
|
||||||
|
item.get("status") == "ready" for item in runtime_health
|
||||||
|
)
|
||||||
|
plugin_ready = ready_runtime_count == len(runtime_health)
|
||||||
|
|
||||||
|
try:
|
||||||
|
recording = recording_materializer.cache_status()
|
||||||
|
recording_status = "ready"
|
||||||
|
recording_document: dict[str, object] = {
|
||||||
|
"status": recording_status,
|
||||||
|
"entries": recording.entry_count,
|
||||||
|
"bytes": recording.total_bytes,
|
||||||
|
"max_bytes": recording.cache_max_bytes,
|
||||||
|
"free_bytes": recording.free_bytes,
|
||||||
|
"free_space_reserve_bytes": recording.free_space_reserve_bytes,
|
||||||
|
}
|
||||||
|
recording_ready = (
|
||||||
|
recording.total_bytes <= recording.cache_max_bytes
|
||||||
|
and recording.free_bytes >= recording.free_space_reserve_bytes
|
||||||
|
)
|
||||||
|
if not recording_ready:
|
||||||
|
recording_document["status"] = "capacity-pressure"
|
||||||
|
except (OSError, RuntimeError):
|
||||||
|
recording_ready = False
|
||||||
|
recording_document = {"status": "unavailable"}
|
||||||
|
|
||||||
|
if artifact_gateway is None:
|
||||||
|
artifact_document: dict[str, object] = {
|
||||||
|
"mode": "standalone",
|
||||||
|
"status": "not-configured",
|
||||||
|
"central_available": None,
|
||||||
|
}
|
||||||
|
shared_artifacts_ready = False
|
||||||
|
shared_artifact_degraded = False
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
artifact = artifact_gateway.status()
|
||||||
|
cache = artifact.cache
|
||||||
|
artifact_document = {
|
||||||
|
"mode": "shared",
|
||||||
|
"status": (
|
||||||
|
"ready"
|
||||||
|
if artifact.central_status == "ready"
|
||||||
|
else (
|
||||||
|
"offline-cache-only"
|
||||||
|
if artifact.central_status == "unavailable"
|
||||||
|
else "invalid"
|
||||||
|
)
|
||||||
|
),
|
||||||
|
"central_available": artifact.central_status == "ready",
|
||||||
|
"cache": {
|
||||||
|
"objects": cache.object_count,
|
||||||
|
"bytes": cache.total_bytes,
|
||||||
|
"pinned_objects": cache.pinned_object_count,
|
||||||
|
"pinned_bytes": cache.pinned_bytes,
|
||||||
|
"max_bytes": cache.cache_max_bytes,
|
||||||
|
"free_space_reserve_bytes": cache.free_space_reserve_bytes,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
shared_artifacts_ready = artifact.central_status == "ready"
|
||||||
|
shared_artifact_degraded = not shared_artifacts_ready
|
||||||
|
except (OSError, RuntimeError):
|
||||||
|
artifact_document = {
|
||||||
|
"mode": "shared",
|
||||||
|
"status": "unavailable",
|
||||||
|
"central_available": False,
|
||||||
|
}
|
||||||
|
shared_artifacts_ready = False
|
||||||
|
shared_artifact_degraded = True
|
||||||
|
|
||||||
|
media_tools_ready = ffmpeg_available and ffprobe_available
|
||||||
|
reconciler_document = reconciler.snapshot()
|
||||||
|
failure_count = reconciler_document["consecutive_failures"]
|
||||||
|
reconciler_failures = (
|
||||||
|
failure_count
|
||||||
|
if isinstance(failure_count, int) and not isinstance(failure_count, bool)
|
||||||
|
else 3
|
||||||
|
)
|
||||||
|
operational_ready = (
|
||||||
|
plugin_ready
|
||||||
|
and recording_ready
|
||||||
|
and media_tools_ready
|
||||||
|
and reconciler_failures < 3
|
||||||
|
)
|
||||||
|
degraded = (
|
||||||
|
shared_artifact_degraded
|
||||||
|
or reconciler_failures > 0
|
||||||
|
or not operational_ready
|
||||||
|
)
|
||||||
|
status = (
|
||||||
|
"unavailable"
|
||||||
|
if not operational_ready
|
||||||
|
else "degraded"
|
||||||
|
if degraded
|
||||||
|
else "ok"
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"ok": operational_ready,
|
||||||
|
"status": status,
|
||||||
|
"service": "mission-core-control-plane",
|
||||||
|
"version": version,
|
||||||
|
"readiness": {
|
||||||
|
"operational": operational_ready,
|
||||||
|
"production_shared_artifacts": (
|
||||||
|
operational_ready and shared_artifacts_ready
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"plugin_runtimes": {
|
||||||
|
"ready": ready_runtime_count,
|
||||||
|
"total": len(runtime_health),
|
||||||
|
},
|
||||||
|
"components": {
|
||||||
|
"recording_cache": recording_document,
|
||||||
|
"artifact_store": artifact_document,
|
||||||
|
"background_reconciler": reconciler_document,
|
||||||
|
"media_tools": {
|
||||||
|
"status": "ready" if media_tools_ready else "unavailable",
|
||||||
|
"ffmpeg": ffmpeg_available,
|
||||||
|
"ffprobe": ffprobe_available,
|
||||||
|
},
|
||||||
|
"map_gateway": {
|
||||||
|
"status": "configured" if map_gateway_configured else "optional",
|
||||||
|
"configured": map_gateway_configured,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _utc_now_iso() -> str:
|
||||||
|
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||||
@@ -19,12 +19,14 @@ from k1link.artifact_gateway import (
|
|||||||
ArtifactGateway,
|
ArtifactGateway,
|
||||||
ArtifactManifest,
|
ArtifactManifest,
|
||||||
ArtifactMember,
|
ArtifactMember,
|
||||||
|
ArtifactStoreUnavailable,
|
||||||
ResolvedArtifact,
|
ResolvedArtifact,
|
||||||
)
|
)
|
||||||
from k1link.compute.integrated_perception import (
|
from k1link.compute.integrated_perception import (
|
||||||
IntegratedPerceptionOverlayStore,
|
IntegratedPerceptionOverlayStore,
|
||||||
_CuboidPresentationState,
|
_CuboidPresentationState,
|
||||||
)
|
)
|
||||||
|
from k1link.compute.results import RecordedPerceptionOverlayError
|
||||||
|
|
||||||
|
|
||||||
def _write_result_descriptor(
|
def _write_result_descriptor(
|
||||||
@@ -303,6 +305,54 @@ def test_integrated_overlay_uses_verified_central_cache_before_local_result_scan
|
|||||||
assert artifact.sha256 == member.sha256
|
assert artifact.sha256 == member.sha256
|
||||||
|
|
||||||
|
|
||||||
|
def test_integrated_overlay_does_not_render_when_central_store_is_offline(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
for name in ("jobs", "results", "lidar-packs"):
|
||||||
|
(tmp_path / name).mkdir()
|
||||||
|
|
||||||
|
class OfflineGateway:
|
||||||
|
def resolve_role(
|
||||||
|
self,
|
||||||
|
namespace: str,
|
||||||
|
key: str,
|
||||||
|
role: str,
|
||||||
|
) -> object:
|
||||||
|
assert (namespace, key, role) == (
|
||||||
|
"sessions",
|
||||||
|
"session-1",
|
||||||
|
"integrated-overlay:recording-1",
|
||||||
|
)
|
||||||
|
raise ArtifactStoreUnavailable("offline cache miss")
|
||||||
|
|
||||||
|
store = IntegratedPerceptionOverlayStore(
|
||||||
|
jobs_root=tmp_path / "jobs",
|
||||||
|
results_root=tmp_path / "results",
|
||||||
|
lidar_packs_root=tmp_path / "lidar-packs",
|
||||||
|
cache_root=tmp_path / "cache",
|
||||||
|
ffmpeg_path=tmp_path / "ffmpeg",
|
||||||
|
artifact_gateway=OfflineGateway(), # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
store,
|
||||||
|
"_latest",
|
||||||
|
lambda _session_id: (_ for _ in ()).throw(
|
||||||
|
AssertionError("offline CAS miss must not scan local results")
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
RecordedPerceptionOverlayError,
|
||||||
|
match="not present in the local artifact cache",
|
||||||
|
):
|
||||||
|
store.materialize(
|
||||||
|
"session-1",
|
||||||
|
application_id="nodedc_mission_core_recorded",
|
||||||
|
recording_id="recording-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_e10_profile_pins_integrated_realtime_budget() -> None:
|
def test_e10_profile_pins_integrated_realtime_budget() -> None:
|
||||||
_fusion, runner = _worker_modules()
|
_fusion, runner = _worker_modules()
|
||||||
profile_path = (
|
profile_path = (
|
||||||
|
|||||||
@@ -0,0 +1,112 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from k1link.artifact_gateway import (
|
||||||
|
ArtifactGateway,
|
||||||
|
CentralArtifactStore,
|
||||||
|
LocalArtifactCache,
|
||||||
|
)
|
||||||
|
from k1link.sessions.recording import SessionRecordingMaterializer
|
||||||
|
from k1link.web.runtime_readiness import (
|
||||||
|
BackgroundReconcilerReadiness,
|
||||||
|
build_runtime_readiness,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _document(
|
||||||
|
tmp_path: Path,
|
||||||
|
*,
|
||||||
|
gateway: ArtifactGateway | None,
|
||||||
|
reconciler: BackgroundReconcilerReadiness,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return build_runtime_readiness(
|
||||||
|
version="test",
|
||||||
|
plugin_runtime_health=({"status": "ready"},),
|
||||||
|
recording_materializer=SessionRecordingMaterializer(
|
||||||
|
tmp_path / "data",
|
||||||
|
exporter=lambda _source, _destination: {},
|
||||||
|
free_space_reserve_bytes=0,
|
||||||
|
),
|
||||||
|
artifact_gateway=gateway,
|
||||||
|
map_gateway_configured=False,
|
||||||
|
ffmpeg_available=True,
|
||||||
|
ffprobe_available=True,
|
||||||
|
reconciler=reconciler,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_readiness_reports_shared_store_and_bounded_cache(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
central_root = tmp_path / "central"
|
||||||
|
gateway = ArtifactGateway(
|
||||||
|
CentralArtifactStore(central_root, create=True),
|
||||||
|
LocalArtifactCache(
|
||||||
|
tmp_path / "cache",
|
||||||
|
max_bytes=1024 * 1024,
|
||||||
|
free_space_reserve_bytes=0,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
reconciler = BackgroundReconcilerReadiness()
|
||||||
|
reconciler.record_success()
|
||||||
|
|
||||||
|
document = _document(tmp_path, gateway=gateway, reconciler=reconciler)
|
||||||
|
|
||||||
|
assert document["ok"] is True
|
||||||
|
assert document["status"] == "ok"
|
||||||
|
assert document["readiness"] == {
|
||||||
|
"operational": True,
|
||||||
|
"production_shared_artifacts": True,
|
||||||
|
}
|
||||||
|
components = document["components"]
|
||||||
|
assert isinstance(components, dict)
|
||||||
|
assert components["recording_cache"]["max_bytes"] == 8 * 1024 * 1024 * 1024
|
||||||
|
assert components["artifact_store"]["status"] == "ready"
|
||||||
|
assert components["map_gateway"]["status"] == "optional"
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_readiness_exposes_offline_cache_only_mode(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
central_root = tmp_path / "central"
|
||||||
|
gateway = ArtifactGateway(
|
||||||
|
CentralArtifactStore(central_root, create=True),
|
||||||
|
LocalArtifactCache(
|
||||||
|
tmp_path / "cache",
|
||||||
|
max_bytes=1024 * 1024,
|
||||||
|
free_space_reserve_bytes=0,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
central_root.rename(tmp_path / "central-offline")
|
||||||
|
reconciler = BackgroundReconcilerReadiness()
|
||||||
|
reconciler.record_success()
|
||||||
|
|
||||||
|
document = _document(tmp_path, gateway=gateway, reconciler=reconciler)
|
||||||
|
|
||||||
|
assert document["ok"] is True
|
||||||
|
assert document["status"] == "degraded"
|
||||||
|
assert document["readiness"] == {
|
||||||
|
"operational": True,
|
||||||
|
"production_shared_artifacts": False,
|
||||||
|
}
|
||||||
|
components = document["components"]
|
||||||
|
assert isinstance(components, dict)
|
||||||
|
assert components["artifact_store"]["status"] == "offline-cache-only"
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_readiness_fails_after_repeated_reconciler_errors(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
reconciler = BackgroundReconcilerReadiness()
|
||||||
|
for _ in range(3):
|
||||||
|
reconciler.record_failure(OSError("private detail"))
|
||||||
|
|
||||||
|
document = _document(tmp_path, gateway=None, reconciler=reconciler)
|
||||||
|
|
||||||
|
assert document["ok"] is False
|
||||||
|
assert document["status"] == "unavailable"
|
||||||
|
components = document["components"]
|
||||||
|
assert isinstance(components, dict)
|
||||||
|
assert components["background_reconciler"]["last_failure_type"] == "OSError"
|
||||||
|
assert "private detail" not in str(document)
|
||||||
@@ -15,6 +15,7 @@ import pytest
|
|||||||
import k1link.sessions.recording as recording_module
|
import k1link.sessions.recording as recording_module
|
||||||
from k1link.artifact_gateway import (
|
from k1link.artifact_gateway import (
|
||||||
ArtifactGateway,
|
ArtifactGateway,
|
||||||
|
ArtifactStoreUnavailable,
|
||||||
CentralArtifactStore,
|
CentralArtifactStore,
|
||||||
LocalArtifactCache,
|
LocalArtifactCache,
|
||||||
)
|
)
|
||||||
@@ -184,7 +185,7 @@ def test_materializer_restores_exact_central_recording_without_export(
|
|||||||
assert exporter.calls == 0
|
assert exporter.calls == 0
|
||||||
|
|
||||||
|
|
||||||
def test_default_recording_cache_has_no_application_byte_quota(
|
def test_default_recording_cache_has_a_bounded_application_quota(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -195,7 +196,42 @@ def test_default_recording_cache_has_no_application_byte_quota(
|
|||||||
exporter=FakeExporter(),
|
exporter=FakeExporter(),
|
||||||
)
|
)
|
||||||
|
|
||||||
assert materializer.cache_max_bytes is None
|
assert materializer.cache_max_bytes == 8 * 1024 * 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
def test_materializer_does_not_rebuild_when_central_store_is_offline(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
command = _command(tmp_path / "source")
|
||||||
|
exporter = FakeExporter()
|
||||||
|
|
||||||
|
class OfflineGateway:
|
||||||
|
def resolve_role(
|
||||||
|
self,
|
||||||
|
namespace: str,
|
||||||
|
key: str,
|
||||||
|
role: str,
|
||||||
|
) -> object:
|
||||||
|
assert (namespace, key, role) == (
|
||||||
|
"sessions",
|
||||||
|
command.session_id,
|
||||||
|
"base-rrd",
|
||||||
|
)
|
||||||
|
raise ArtifactStoreUnavailable("offline cache miss")
|
||||||
|
|
||||||
|
materializer = SessionRecordingMaterializer(
|
||||||
|
tmp_path / "private",
|
||||||
|
exporter=exporter,
|
||||||
|
artifact_gateway=OfflineGateway(), # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
RecordingMaterializationError,
|
||||||
|
match="not present in the local artifact cache",
|
||||||
|
):
|
||||||
|
materializer.materialize(command)
|
||||||
|
|
||||||
|
assert exporter.calls == 0
|
||||||
|
|
||||||
|
|
||||||
def test_delete_cached_refuses_a_live_lease_then_removes_the_exact_cache(
|
def test_delete_cached_refuses_a_live_lease_then_removes_the_exact_cache(
|
||||||
|
|||||||
Reference in New Issue
Block a user