fix(runtime): fail closed on offline artifact misses

This commit is contained in:
DCCONSTRUCTIONS
2026-07-30 22:19:01 +03:00
parent 81d7ad1a77
commit 8c91d40e2e
12 changed files with 510 additions and 29 deletions
+50
View File
@@ -19,12 +19,14 @@ from k1link.artifact_gateway import (
ArtifactGateway,
ArtifactManifest,
ArtifactMember,
ArtifactStoreUnavailable,
ResolvedArtifact,
)
from k1link.compute.integrated_perception import (
IntegratedPerceptionOverlayStore,
_CuboidPresentationState,
)
from k1link.compute.results import RecordedPerceptionOverlayError
def _write_result_descriptor(
@@ -303,6 +305,54 @@ def test_integrated_overlay_uses_verified_central_cache_before_local_result_scan
assert artifact.sha256 == member.sha256
def test_integrated_overlay_does_not_render_when_central_store_is_offline(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
for name in ("jobs", "results", "lidar-packs"):
(tmp_path / name).mkdir()
class OfflineGateway:
def resolve_role(
self,
namespace: str,
key: str,
role: str,
) -> object:
assert (namespace, key, role) == (
"sessions",
"session-1",
"integrated-overlay:recording-1",
)
raise ArtifactStoreUnavailable("offline cache miss")
store = IntegratedPerceptionOverlayStore(
jobs_root=tmp_path / "jobs",
results_root=tmp_path / "results",
lidar_packs_root=tmp_path / "lidar-packs",
cache_root=tmp_path / "cache",
ffmpeg_path=tmp_path / "ffmpeg",
artifact_gateway=OfflineGateway(), # type: ignore[arg-type]
)
monkeypatch.setattr(
store,
"_latest",
lambda _session_id: (_ for _ in ()).throw(
AssertionError("offline CAS miss must not scan local results")
),
)
with pytest.raises(
RecordedPerceptionOverlayError,
match="not present in the local artifact cache",
):
store.materialize(
"session-1",
application_id="nodedc_mission_core_recorded",
recording_id="recording-1",
)
def test_e10_profile_pins_integrated_realtime_budget() -> None:
_fusion, runner = _worker_modules()
profile_path = (
+112
View File
@@ -0,0 +1,112 @@
from __future__ import annotations
from pathlib import Path
from k1link.artifact_gateway import (
ArtifactGateway,
CentralArtifactStore,
LocalArtifactCache,
)
from k1link.sessions.recording import SessionRecordingMaterializer
from k1link.web.runtime_readiness import (
BackgroundReconcilerReadiness,
build_runtime_readiness,
)
def _document(
tmp_path: Path,
*,
gateway: ArtifactGateway | None,
reconciler: BackgroundReconcilerReadiness,
) -> dict[str, object]:
return build_runtime_readiness(
version="test",
plugin_runtime_health=({"status": "ready"},),
recording_materializer=SessionRecordingMaterializer(
tmp_path / "data",
exporter=lambda _source, _destination: {},
free_space_reserve_bytes=0,
),
artifact_gateway=gateway,
map_gateway_configured=False,
ffmpeg_available=True,
ffprobe_available=True,
reconciler=reconciler,
)
def test_runtime_readiness_reports_shared_store_and_bounded_cache(
tmp_path: Path,
) -> None:
central_root = tmp_path / "central"
gateway = ArtifactGateway(
CentralArtifactStore(central_root, create=True),
LocalArtifactCache(
tmp_path / "cache",
max_bytes=1024 * 1024,
free_space_reserve_bytes=0,
),
)
reconciler = BackgroundReconcilerReadiness()
reconciler.record_success()
document = _document(tmp_path, gateway=gateway, reconciler=reconciler)
assert document["ok"] is True
assert document["status"] == "ok"
assert document["readiness"] == {
"operational": True,
"production_shared_artifacts": True,
}
components = document["components"]
assert isinstance(components, dict)
assert components["recording_cache"]["max_bytes"] == 8 * 1024 * 1024 * 1024
assert components["artifact_store"]["status"] == "ready"
assert components["map_gateway"]["status"] == "optional"
def test_runtime_readiness_exposes_offline_cache_only_mode(
tmp_path: Path,
) -> None:
central_root = tmp_path / "central"
gateway = ArtifactGateway(
CentralArtifactStore(central_root, create=True),
LocalArtifactCache(
tmp_path / "cache",
max_bytes=1024 * 1024,
free_space_reserve_bytes=0,
),
)
central_root.rename(tmp_path / "central-offline")
reconciler = BackgroundReconcilerReadiness()
reconciler.record_success()
document = _document(tmp_path, gateway=gateway, reconciler=reconciler)
assert document["ok"] is True
assert document["status"] == "degraded"
assert document["readiness"] == {
"operational": True,
"production_shared_artifacts": False,
}
components = document["components"]
assert isinstance(components, dict)
assert components["artifact_store"]["status"] == "offline-cache-only"
def test_runtime_readiness_fails_after_repeated_reconciler_errors(
tmp_path: Path,
) -> None:
reconciler = BackgroundReconcilerReadiness()
for _ in range(3):
reconciler.record_failure(OSError("private detail"))
document = _document(tmp_path, gateway=None, reconciler=reconciler)
assert document["ok"] is False
assert document["status"] == "unavailable"
components = document["components"]
assert isinstance(components, dict)
assert components["background_reconciler"]["last_failure_type"] == "OSError"
assert "private detail" not in str(document)
+38 -2
View File
@@ -15,6 +15,7 @@ import pytest
import k1link.sessions.recording as recording_module
from k1link.artifact_gateway import (
ArtifactGateway,
ArtifactStoreUnavailable,
CentralArtifactStore,
LocalArtifactCache,
)
@@ -184,7 +185,7 @@ def test_materializer_restores_exact_central_recording_without_export(
assert exporter.calls == 0
def test_default_recording_cache_has_no_application_byte_quota(
def test_default_recording_cache_has_a_bounded_application_quota(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -195,7 +196,42 @@ def test_default_recording_cache_has_no_application_byte_quota(
exporter=FakeExporter(),
)
assert materializer.cache_max_bytes is None
assert materializer.cache_max_bytes == 8 * 1024 * 1024 * 1024
def test_materializer_does_not_rebuild_when_central_store_is_offline(
tmp_path: Path,
) -> None:
command = _command(tmp_path / "source")
exporter = FakeExporter()
class OfflineGateway:
def resolve_role(
self,
namespace: str,
key: str,
role: str,
) -> object:
assert (namespace, key, role) == (
"sessions",
command.session_id,
"base-rrd",
)
raise ArtifactStoreUnavailable("offline cache miss")
materializer = SessionRecordingMaterializer(
tmp_path / "private",
exporter=exporter,
artifact_gateway=OfflineGateway(), # type: ignore[arg-type]
)
with pytest.raises(
RecordingMaterializationError,
match="not present in the local artifact cache",
):
materializer.materialize(command)
assert exporter.calls == 0
def test_delete_cached_refuses_a_live_lease_then_removes_the_exact_cache(