SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(storage): add portable session artifact gateway

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-30 10:05:48 +03:00
parent 2ab9e45548
commit e56fa0c074
11 changed files with 1889 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_artifact_gateway.py
+209
View File
@@ -0,0 +1,209 @@
from __future__ import annotations
import os
from pathlib import Path
import pytest
from k1link.artifact_gateway import (
ArtifactCacheCapacityError,
ArtifactGateway,
ArtifactIntegrityError,
ArtifactStoreUnavailable,
CentralArtifactStore,
LocalArtifactCache,
)
def _gateway(
tmp_path: Path,
*,
max_bytes: int = 1024,
) -> tuple[ArtifactGateway, Path, Path]:
central_root = tmp_path / "central"
local_root = tmp_path / "local"
return (
ArtifactGateway(
CentralArtifactStore(central_root, create=True),
LocalArtifactCache(
local_root,
max_bytes=max_bytes,
free_space_reserve_bytes=0,
),
),
central_root,
local_root,
)
def test_gateway_publishes_deduplicated_objects_and_resolves_through_local_cache(
tmp_path: Path,
) -> None:
gateway, central_root, _local_root = _gateway(tmp_path)
first = tmp_path / "first.rrd"
duplicate = tmp_path / "duplicate.rrd"
first.write_bytes(b"RRF2same-content")
duplicate.write_bytes(first.read_bytes())
manifest = gateway.publish(
namespace="sessions",
key="session-1",
artifact_type="recorded-session",
subject_id="session-1",
sources=(
("base-rrd", "application/vnd.rerun.rrd", first),
("overlay-rrd", "application/vnd.rerun.rrd", duplicate),
),
metadata={"display-name": "Session 1"},
)
assert len(manifest.members) == 2
assert manifest.members[0].sha256 == manifest.members[1].sha256
objects = tuple((central_root / "objects" / "sha256").glob("*/*"))
assert len(objects) == 1
cold = gateway.resolve_role("sessions", "session-1", "overlay-rrd")
assert cold.path.read_bytes() == first.read_bytes()
assert cold.cache_hit is False
assert cold.central_available is True
warm = gateway.resolve_role("sessions", "session-1", "overlay-rrd")
assert warm.path == cold.path
assert warm.cache_hit is True
def test_persistent_pin_survives_restart_and_resolves_with_central_offline(
tmp_path: Path,
) -> None:
gateway, central_root, local_root = _gateway(tmp_path)
source = tmp_path / "overlay.rrd"
source.write_bytes(b"RRF2offline-ready")
gateway.publish(
namespace="sessions",
key="session-1",
artifact_type="recorded-session",
subject_id="session-1",
sources=(("overlay-rrd", "application/vnd.rerun.rrd", source),),
)
gateway.pin_reference("sessions", "session-1", pin_id="session-1")
offline_root = tmp_path / "central-offline"
os.replace(central_root, offline_root)
restarted = ArtifactGateway(
gateway.store,
LocalArtifactCache(
local_root,
max_bytes=1024,
free_space_reserve_bytes=0,
),
)
resolved = restarted.resolve_role("sessions", "session-1", "overlay-rrd")
assert resolved.path.read_bytes() == source.read_bytes()
assert resolved.cache_hit is True
assert resolved.central_available is False
status = restarted.cache.status()
assert status.pinned_object_count == 1
assert status.pinned_bytes == len(source.read_bytes())
def test_offline_cache_miss_is_explicit(tmp_path: Path) -> None:
gateway, central_root, local_root = _gateway(tmp_path)
source = tmp_path / "overlay.rrd"
source.write_bytes(b"RRF2not-fetched")
gateway.publish(
namespace="sessions",
key="session-1",
artifact_type="recorded-session",
subject_id="session-1",
sources=(("overlay-rrd", "application/vnd.rerun.rrd", source),),
)
os.replace(central_root, tmp_path / "central-offline")
restarted = ArtifactGateway(
gateway.store,
LocalArtifactCache(local_root, max_bytes=1024, free_space_reserve_bytes=0),
)
with pytest.raises(ArtifactStoreUnavailable, match="offline cache"):
restarted.resolve_role("sessions", "session-1", "overlay-rrd")
def test_unpinned_lru_is_evicted_but_pinned_object_is_preserved(tmp_path: Path) -> None:
gateway, _central_root, _local_root = _gateway(tmp_path, max_bytes=18)
first = tmp_path / "first.bin"
second = tmp_path / "second.bin"
third = tmp_path / "third.bin"
first.write_bytes(b"a" * 9)
second.write_bytes(b"b" * 9)
third.write_bytes(b"c" * 9)
for key, source in (("first", first), ("second", second), ("third", third)):
gateway.publish(
namespace="objects",
key=key,
artifact_type="test-object",
subject_id=key,
sources=(("payload", "application/octet-stream", source),),
)
gateway.pin_reference("objects", "first", pin_id="keep-first")
second_resolved = gateway.resolve_role("objects", "second", "payload")
assert second_resolved.path.is_file()
gateway.resolve_role("objects", "third", "payload")
first_member = gateway.store.resolve_reference("objects", "first").member("payload")
second_member = gateway.store.resolve_reference("objects", "second").member("payload")
third_member = gateway.store.resolve_reference("objects", "third").member("payload")
assert gateway.cache.get(first_member) is not None
assert gateway.cache.get(second_member) is None
assert gateway.cache.get(third_member) is not None
def test_cache_refuses_object_larger_than_its_quota(tmp_path: Path) -> None:
gateway, _central_root, _local_root = _gateway(tmp_path, max_bytes=8)
source = tmp_path / "large.bin"
source.write_bytes(b"x" * 9)
gateway.publish(
namespace="objects",
key="large",
artifact_type="test-object",
subject_id="large",
sources=(("payload", "application/octet-stream", source),),
)
with pytest.raises(ArtifactCacheCapacityError, match="exceeds"):
gateway.resolve_role("objects", "large", "payload")
def test_same_size_local_corruption_is_detected_after_stat_identity_changes(
tmp_path: Path,
) -> None:
gateway, _central_root, _local_root = _gateway(tmp_path)
source = tmp_path / "payload.bin"
source.write_bytes(b"original")
gateway.publish(
namespace="objects",
key="payload",
artifact_type="test-object",
subject_id="payload",
sources=(("payload", "application/octet-stream", source),),
)
resolved = gateway.resolve_role("objects", "payload", "payload")
resolved.path.write_bytes(b"tampered")
os.utime(resolved.path, ns=(resolved.path.stat().st_atime_ns, 1))
member = gateway.store.resolve_reference("objects", "payload").member("payload")
assert gateway.cache.get(member) is None
repaired = gateway.resolve_role("objects", "payload", "payload")
assert repaired.path.read_bytes() == b"original"
def test_store_rejects_corrupt_existing_content_addressed_object(tmp_path: Path) -> None:
store = CentralArtifactStore(tmp_path / "central", create=True)
source = tmp_path / "source.bin"
source.write_bytes(b"original")
published = store.publish_file(source)
store.object_path(published.sha256).write_bytes(b"tampered")
with pytest.raises(ArtifactIntegrityError, match="corrupt"):
store.publish_file(source)
tests/test_e10_integrated_perception.py
+76
View File
@@ -9,11 +9,18 @@ import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace
from typing import cast
import numpy as np
import pytest
import k1link.compute.integrated_perception as integrated_module
from k1link.artifact_gateway import (
ArtifactGateway,
ArtifactManifest,
ArtifactMember,
ResolvedArtifact,
)
from k1link.compute.integrated_perception import (
IntegratedPerceptionOverlayStore,
_CuboidPresentationState,
@@ -227,6 +234,75 @@ def test_integrated_overlay_serializes_heavy_materialization_across_sessions(
assert maximum_active == 1
def test_integrated_overlay_uses_verified_central_cache_before_local_result_scan(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
for name in ("jobs", "results", "lidar-packs"):
(tmp_path / name).mkdir()
recording_id = "recording-1"
result_id = f"e10-integrated-perception-{'a' * 64}"
payload = b"RRF2central-overlay"
artifact_path = tmp_path / "artifact-cache" / "overlay.rrd"
artifact_path.parent.mkdir()
artifact_path.write_bytes(payload)
member = ArtifactMember(
role=f"integrated-overlay:{recording_id}",
media_type="application/vnd.rerun.rrd",
sha256=hashlib.sha256(payload).hexdigest(),
byte_length=len(payload),
)
manifest = ArtifactManifest(
manifest_id="b" * 64,
artifact_type="recorded-session",
subject_id="session-1",
created_at_utc="2026-07-30T00:00:00Z",
members=(member,),
metadata={"integrated-result-id": result_id},
)
class FakeGateway:
def resolve_role(self, namespace: str, key: str, role: str) -> ResolvedArtifact:
assert (namespace, key, role) == (
"sessions",
"session-1",
f"integrated-overlay:{recording_id}",
)
return ResolvedArtifact(
manifest=manifest,
member=member,
path=artifact_path,
cache_hit=False,
central_available=True,
)
store = IntegratedPerceptionOverlayStore(
jobs_root=tmp_path / "jobs",
results_root=tmp_path / "results",
lidar_packs_root=tmp_path / "lidar-packs",
cache_root=tmp_path / "cache",
ffmpeg_path=tmp_path / "ffmpeg",
artifact_gateway=cast(ArtifactGateway, FakeGateway()),
)
monkeypatch.setattr(
store,
"_latest_descriptor",
lambda _session_id: (_ for _ in ()).throw(
AssertionError("central cache hit must not scan local results")
),
)
artifact = store.materialize(
"session-1",
application_id="nodedc_mission_core_recorded",
recording_id=recording_id,
)
assert artifact is not None
assert artifact.path == artifact_path
assert artifact.sha256 == member.sha256
def test_e10_profile_pins_integrated_realtime_budget() -> None:
_fusion, runner = _worker_modules()
profile_path = (
tests/test_session_recording.py
+53
View File
@@ -13,6 +13,11 @@ from types import SimpleNamespace
import pytest
import k1link.sessions.recording as recording_module
from k1link.artifact_gateway import (
ArtifactGateway,
CentralArtifactStore,
LocalArtifactCache,
)
from k1link.sessions.models import ReplayArtifact, ReplayCommand
from k1link.sessions.recording import (
CACHE_SCHEMA,
@@ -131,6 +136,54 @@ def test_materializer_reuses_only_a_digest_validated_private_cache(tmp_path: Pat
assert exporter.calls == 1
def test_materializer_restores_exact_central_recording_without_export(
tmp_path: Path,
) -> None:
command = _command(tmp_path / "source")
central_recording = tmp_path / "central.rrd"
central_recording.write_bytes(b"RRF2centrally-prepared-recording")
gateway = ArtifactGateway(
CentralArtifactStore(tmp_path / "central-store", create=True),
LocalArtifactCache(
tmp_path / "artifact-cache",
max_bytes=1024 * 1024,
free_space_reserve_bytes=0,
),
)
gateway.publish(
namespace="sessions",
key=command.session_id,
artifact_type="recorded-session",
subject_id=command.session_id,
sources=(
("base-rrd", RERUN_RECORDING_MEDIA_TYPE, central_recording),
),
metadata={
"base-source-sha256": _sha256(command.primary_artifact.path),
"base-timeline": "session_time",
"base-timeline-start-ns": "0",
"base-timeline-end-ns": "2500000000",
},
)
exporter = FakeExporter()
materializer = SessionRecordingMaterializer(
tmp_path / "private",
exporter=exporter,
artifact_gateway=gateway,
)
restored = materializer.materialize(command)
assert exporter.calls == 0
assert restored.path.is_relative_to(materializer.recordings_root)
assert restored.path.read_bytes() == central_recording.read_bytes()
assert restored.sha256 == _sha256(central_recording)
assert restored.source_sha256 == _sha256(command.primary_artifact.path)
assert restored.timeline_end_ns == 2_500_000_000
assert materializer.materialize(command) == restored
assert exporter.calls == 0
def test_default_recording_cache_has_no_application_byte_quota(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,