feat(storage): add portable session artifact gateway

This commit is contained in:
DCCONSTRUCTIONS
2026-07-30 10:05:48 +03:00
parent 2ab9e45548
commit e56fa0c074
11 changed files with 1889 additions and 2 deletions
@@ -22,6 +22,11 @@ import numpy as np
import rerun as rr
from rerun.components import FillMode
from k1link.artifact_gateway import (
ArtifactGateway,
ArtifactNotFound,
ArtifactStoreUnavailable,
)
from k1link.artifacts import write_json_atomic
from k1link.sessions import SessionIntegrityError
@@ -377,12 +382,14 @@ class IntegratedPerceptionOverlayStore:
lidar_packs_root: Path,
cache_root: Path,
ffmpeg_path: Path,
artifact_gateway: ArtifactGateway | None = None,
) -> None:
self.jobs_root = jobs_root.expanduser().absolute()
self.results_root = results_root.expanduser().absolute()
self.lidar_packs_root = lidar_packs_root.expanduser().absolute()
self.cache_root = cache_root.expanduser().absolute()
self.ffmpeg_path = ffmpeg_path.expanduser().absolute()
self.artifact_gateway = artifact_gateway
self._catalog_lock = threading.Lock()
self._descriptor_catalog_generation: tuple[int, int] | None = None
self._descriptors_by_session: dict[str, tuple[_ResultDescriptor, ...]] = {}
@@ -444,6 +451,15 @@ class IntegratedPerceptionOverlayStore:
self._set_status(key, state="preparing", phase="cache-lookup")
try:
with self._single_flight(key):
central = self._read_gateway_cache(session_id, recording_id)
if central is not None:
self._set_status(
key,
state="ready",
phase="ready",
byte_length=central.byte_length,
)
return central
cached = self._read_admitted_cache(session_id, recording_id)
if cached is not None:
self._set_status(
@@ -534,6 +550,50 @@ class IntegratedPerceptionOverlayStore:
self._set_status(key, state="error", phase="error")
raise
def _read_gateway_cache(
self,
session_id: str,
recording_id: str,
) -> RecordedPerceptionOverlayArtifact | None:
if self.artifact_gateway is None:
return None
role = f"integrated-overlay:{recording_id}"
try:
resolved = self.artifact_gateway.resolve_role("sessions", session_id, role)
except (ArtifactNotFound, ArtifactStoreUnavailable):
return None
expected_result_id = resolved.manifest.metadata.get("integrated-result-id")
if (
resolved.member.media_type != "application/vnd.rerun.rrd"
or expected_result_id is None
or _SAFE_RESULT_ID.fullmatch(expected_result_id) is None
):
raise RecordedPerceptionOverlayError(
"central integrated perception artifact metadata is invalid"
)
try:
metadata = resolved.path.lstat()
with resolved.path.open("rb") as stream:
magic = stream.read(4)
except OSError as exc:
raise RecordedPerceptionOverlayError(
"central integrated perception artifact became unavailable"
) from exc
if (
stat.S_ISLNK(metadata.st_mode)
or not stat.S_ISREG(metadata.st_mode)
or metadata.st_size != resolved.member.byte_length
or magic != b"RRF2"
):
raise RecordedPerceptionOverlayError(
"central integrated perception artifact is invalid"
)
return RecordedPerceptionOverlayArtifact(
path=resolved.path.resolve(strict=True),
byte_length=resolved.member.byte_length,
sha256=resolved.member.sha256,
)
def status(self, session_id: str, *, recording_id: str) -> dict[str, Any]:
if (
_SAFE_RECORDING_ID.fullmatch(session_id) is None