fix(runtime): fail closed on offline artifact misses
This commit is contained in:
@@ -19,12 +19,14 @@ from k1link.artifact_gateway import (
|
||||
ArtifactGateway,
|
||||
ArtifactManifest,
|
||||
ArtifactMember,
|
||||
ArtifactStoreUnavailable,
|
||||
ResolvedArtifact,
|
||||
)
|
||||
from k1link.compute.integrated_perception import (
|
||||
IntegratedPerceptionOverlayStore,
|
||||
_CuboidPresentationState,
|
||||
)
|
||||
from k1link.compute.results import RecordedPerceptionOverlayError
|
||||
|
||||
|
||||
def _write_result_descriptor(
|
||||
@@ -303,6 +305,54 @@ def test_integrated_overlay_uses_verified_central_cache_before_local_result_scan
|
||||
assert artifact.sha256 == member.sha256
|
||||
|
||||
|
||||
def test_integrated_overlay_does_not_render_when_central_store_is_offline(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
for name in ("jobs", "results", "lidar-packs"):
|
||||
(tmp_path / name).mkdir()
|
||||
|
||||
class OfflineGateway:
|
||||
def resolve_role(
|
||||
self,
|
||||
namespace: str,
|
||||
key: str,
|
||||
role: str,
|
||||
) -> object:
|
||||
assert (namespace, key, role) == (
|
||||
"sessions",
|
||||
"session-1",
|
||||
"integrated-overlay:recording-1",
|
||||
)
|
||||
raise ArtifactStoreUnavailable("offline cache miss")
|
||||
|
||||
store = IntegratedPerceptionOverlayStore(
|
||||
jobs_root=tmp_path / "jobs",
|
||||
results_root=tmp_path / "results",
|
||||
lidar_packs_root=tmp_path / "lidar-packs",
|
||||
cache_root=tmp_path / "cache",
|
||||
ffmpeg_path=tmp_path / "ffmpeg",
|
||||
artifact_gateway=OfflineGateway(), # type: ignore[arg-type]
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
store,
|
||||
"_latest",
|
||||
lambda _session_id: (_ for _ in ()).throw(
|
||||
AssertionError("offline CAS miss must not scan local results")
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
RecordedPerceptionOverlayError,
|
||||
match="not present in the local artifact cache",
|
||||
):
|
||||
store.materialize(
|
||||
"session-1",
|
||||
application_id="nodedc_mission_core_recorded",
|
||||
recording_id="recording-1",
|
||||
)
|
||||
|
||||
|
||||
def test_e10_profile_pins_integrated_realtime_budget() -> None:
|
||||
_fusion, runner = _worker_modules()
|
||||
profile_path = (
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.artifact_gateway import (
|
||||
ArtifactGateway,
|
||||
CentralArtifactStore,
|
||||
LocalArtifactCache,
|
||||
)
|
||||
from k1link.sessions.recording import SessionRecordingMaterializer
|
||||
from k1link.web.runtime_readiness import (
|
||||
BackgroundReconcilerReadiness,
|
||||
build_runtime_readiness,
|
||||
)
|
||||
|
||||
|
||||
def _document(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
gateway: ArtifactGateway | None,
|
||||
reconciler: BackgroundReconcilerReadiness,
|
||||
) -> dict[str, object]:
|
||||
return build_runtime_readiness(
|
||||
version="test",
|
||||
plugin_runtime_health=({"status": "ready"},),
|
||||
recording_materializer=SessionRecordingMaterializer(
|
||||
tmp_path / "data",
|
||||
exporter=lambda _source, _destination: {},
|
||||
free_space_reserve_bytes=0,
|
||||
),
|
||||
artifact_gateway=gateway,
|
||||
map_gateway_configured=False,
|
||||
ffmpeg_available=True,
|
||||
ffprobe_available=True,
|
||||
reconciler=reconciler,
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_readiness_reports_shared_store_and_bounded_cache(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
central_root = tmp_path / "central"
|
||||
gateway = ArtifactGateway(
|
||||
CentralArtifactStore(central_root, create=True),
|
||||
LocalArtifactCache(
|
||||
tmp_path / "cache",
|
||||
max_bytes=1024 * 1024,
|
||||
free_space_reserve_bytes=0,
|
||||
),
|
||||
)
|
||||
reconciler = BackgroundReconcilerReadiness()
|
||||
reconciler.record_success()
|
||||
|
||||
document = _document(tmp_path, gateway=gateway, reconciler=reconciler)
|
||||
|
||||
assert document["ok"] is True
|
||||
assert document["status"] == "ok"
|
||||
assert document["readiness"] == {
|
||||
"operational": True,
|
||||
"production_shared_artifacts": True,
|
||||
}
|
||||
components = document["components"]
|
||||
assert isinstance(components, dict)
|
||||
assert components["recording_cache"]["max_bytes"] == 8 * 1024 * 1024 * 1024
|
||||
assert components["artifact_store"]["status"] == "ready"
|
||||
assert components["map_gateway"]["status"] == "optional"
|
||||
|
||||
|
||||
def test_runtime_readiness_exposes_offline_cache_only_mode(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
central_root = tmp_path / "central"
|
||||
gateway = ArtifactGateway(
|
||||
CentralArtifactStore(central_root, create=True),
|
||||
LocalArtifactCache(
|
||||
tmp_path / "cache",
|
||||
max_bytes=1024 * 1024,
|
||||
free_space_reserve_bytes=0,
|
||||
),
|
||||
)
|
||||
central_root.rename(tmp_path / "central-offline")
|
||||
reconciler = BackgroundReconcilerReadiness()
|
||||
reconciler.record_success()
|
||||
|
||||
document = _document(tmp_path, gateway=gateway, reconciler=reconciler)
|
||||
|
||||
assert document["ok"] is True
|
||||
assert document["status"] == "degraded"
|
||||
assert document["readiness"] == {
|
||||
"operational": True,
|
||||
"production_shared_artifacts": False,
|
||||
}
|
||||
components = document["components"]
|
||||
assert isinstance(components, dict)
|
||||
assert components["artifact_store"]["status"] == "offline-cache-only"
|
||||
|
||||
|
||||
def test_runtime_readiness_fails_after_repeated_reconciler_errors(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
reconciler = BackgroundReconcilerReadiness()
|
||||
for _ in range(3):
|
||||
reconciler.record_failure(OSError("private detail"))
|
||||
|
||||
document = _document(tmp_path, gateway=None, reconciler=reconciler)
|
||||
|
||||
assert document["ok"] is False
|
||||
assert document["status"] == "unavailable"
|
||||
components = document["components"]
|
||||
assert isinstance(components, dict)
|
||||
assert components["background_reconciler"]["last_failure_type"] == "OSError"
|
||||
assert "private detail" not in str(document)
|
||||
@@ -15,6 +15,7 @@ import pytest
|
||||
import k1link.sessions.recording as recording_module
|
||||
from k1link.artifact_gateway import (
|
||||
ArtifactGateway,
|
||||
ArtifactStoreUnavailable,
|
||||
CentralArtifactStore,
|
||||
LocalArtifactCache,
|
||||
)
|
||||
@@ -184,7 +185,7 @@ def test_materializer_restores_exact_central_recording_without_export(
|
||||
assert exporter.calls == 0
|
||||
|
||||
|
||||
def test_default_recording_cache_has_no_application_byte_quota(
|
||||
def test_default_recording_cache_has_a_bounded_application_quota(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
@@ -195,7 +196,42 @@ def test_default_recording_cache_has_no_application_byte_quota(
|
||||
exporter=FakeExporter(),
|
||||
)
|
||||
|
||||
assert materializer.cache_max_bytes is None
|
||||
assert materializer.cache_max_bytes == 8 * 1024 * 1024 * 1024
|
||||
|
||||
|
||||
def test_materializer_does_not_rebuild_when_central_store_is_offline(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
command = _command(tmp_path / "source")
|
||||
exporter = FakeExporter()
|
||||
|
||||
class OfflineGateway:
|
||||
def resolve_role(
|
||||
self,
|
||||
namespace: str,
|
||||
key: str,
|
||||
role: str,
|
||||
) -> object:
|
||||
assert (namespace, key, role) == (
|
||||
"sessions",
|
||||
command.session_id,
|
||||
"base-rrd",
|
||||
)
|
||||
raise ArtifactStoreUnavailable("offline cache miss")
|
||||
|
||||
materializer = SessionRecordingMaterializer(
|
||||
tmp_path / "private",
|
||||
exporter=exporter,
|
||||
artifact_gateway=OfflineGateway(), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
RecordingMaterializationError,
|
||||
match="not present in the local artifact cache",
|
||||
):
|
||||
materializer.materialize(command)
|
||||
|
||||
assert exporter.calls == 0
|
||||
|
||||
|
||||
def test_delete_cached_refuses_a_live_lease_then_removes_the_exact_cache(
|
||||
|
||||
Reference in New Issue
Block a user