fix(runtime): fail closed on offline artifact misses

This commit is contained in:
DCCONSTRUCTIONS
2026-07-30 22:19:01 +03:00
parent 81d7ad1a77
commit 8c91d40e2e
12 changed files with 510 additions and 29 deletions
+36 -2
View File
@@ -43,6 +43,7 @@ RERUN_RECORDING_MEDIA_TYPE = "application/vnd.rerun.rrd"
RERUN_SESSION_TIMELINE = "session_time"
SESSION_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
DEFAULT_FREE_SPACE_RESERVE_BYTES = 2 * 1024 * 1024 * 1024
DEFAULT_CACHE_MAX_BYTES = 8 * 1024 * 1024 * 1024
RrdExporter = Callable[..., Mapping[str, object]]
RecordingProgressCallback = Callable[[str, float], None]
@@ -71,6 +72,15 @@ class MaterializedRecording:
timeline_end_ns: int
@dataclass(frozen=True, slots=True)
class RecordingCacheStatus:
entry_count: int
total_bytes: int
cache_max_bytes: int
free_space_reserve_bytes: int
free_bytes: int
@dataclass(frozen=True, slots=True)
class _ValidatedMemoryEntry:
source_identity: tuple[object, ...]
@@ -165,10 +175,15 @@ class SessionRecordingMaterializer:
self._exporters = dict(exporters or {})
if len(self._exporters) != len(set(self._exporters)):
raise ValueError("recording exporter plugin ids must be unique")
self.cache_max_bytes = _optional_positive_configuration(
configured_cache_max_bytes = _optional_positive_configuration(
cache_max_bytes,
environment_name="MISSIONCORE_RRD_CACHE_MAX_BYTES",
)
self.cache_max_bytes = (
configured_cache_max_bytes
if configured_cache_max_bytes is not None
else DEFAULT_CACHE_MAX_BYTES
)
self.free_space_reserve_bytes = _non_negative_configuration(
free_space_reserve_bytes,
environment_name="MISSIONCORE_RRD_FREE_SPACE_RESERVE_BYTES",
@@ -199,6 +214,20 @@ class SessionRecordingMaterializer:
_callable_accepts_keyword(exporter, "cancel_event") for exporter in exporters
)
def cache_status(self) -> RecordingCacheStatus:
"""Return bounded cache capacity and occupancy for runtime readiness."""
with self._cache_guard:
total_bytes, entries = _cache_entries(self.recordings_root)
free_bytes = shutil.disk_usage(self.recordings_root).free
return RecordingCacheStatus(
entry_count=len(entries),
total_bytes=total_bytes,
cache_max_bytes=self.cache_max_bytes,
free_space_reserve_bytes=self.free_space_reserve_bytes,
free_bytes=free_bytes,
)
def is_recording_available(self, recording: MaterializedRecording) -> bool:
"""Cheap no-follow check for a previously validated ready handle."""
@@ -409,8 +438,13 @@ class SessionRecordingMaterializer:
session_id,
"base-rrd",
)
except (ArtifactNotFound, ArtifactStoreUnavailable):
except ArtifactNotFound:
return None
except ArtifactStoreUnavailable as exc:
raise RecordingMaterializationError(
"central recording is unavailable and is not present "
"in the local artifact cache"
) from exc
except ArtifactGatewayError as exc:
raise RecordingMaterializationError(
"central recording artifact failed validation"