SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(archive): preserve prepared sessions across restarts

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-18 11:08:27 +03:00
parent aa3680948f
commit 574a494759
13 changed files with 398 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/sessions/media.py
+36
View File
@@ -175,6 +175,42 @@ class RecordedMediaInspector:
self._cache[key] = _CachedManifest(identity=identity, manifest=manifest)
return manifest
def restore_prepared(
self,
artifact: RecordedMediaArtifact,
replay: ReplayCommand,
) -> RecordedMediaManifest | None:
"""Restore one previously prepared manifest without rebuilding it.
A catalog read or process restart must never turn an old camera archive
into fresh preparation work. The durable sidecar is accepted only when
its complete source stat identity still matches the sealed archive.
Missing or stale sidecars remain cold until the operator explicitly
requests that session.
"""
if artifact.session_id != replay.session_id:
raise SessionIntegrityError("recorded media does not belong to replay session")
epoch_paths = _epoch_paths(artifact.source_path)
key = (artifact.session_id, artifact.artifact_id)
with self._lock:
cached = self._cache.get(key)
if cached is not None:
identity = _prepared_source_identity(
replay,
epoch_paths,
cached.manifest.epochs,
)
if cached.identity == identity:
return cached.manifest
prepared = self._load_prepared_sidecar(artifact, replay, epoch_paths)
if prepared is None:
return None
manifest, identity = prepared
with self._lock:
self._cache[key] = _CachedManifest(identity=identity, manifest=manifest)
return manifest
def _load_prepared_sidecar(
self,
artifact: RecordedMediaArtifact,
src/k1link/sessions/preparation.py
+76 -7
View File
@@ -111,6 +111,11 @@ class SessionRecordingPreparationManager:
tuple[RecordedMediaManifest, ...],
]
| None = None,
ready_restorer: Callable[
[ReplayCommand, MaterializedRecording],
tuple[RecordedMediaManifest, ...] | None,
]
| None = None,
) -> None:
if queue_capacity < 1:
raise ValueError("recording preparation queue capacity must be positive")
@@ -123,6 +128,7 @@ class SessionRecordingPreparationManager:
# hung exporter must become observable to the browser stall detector.
self._heartbeat_interval_seconds = heartbeat_interval_seconds
self._ready_preparer = ready_preparer
self._ready_restorer = ready_restorer
self._guard = threading.RLock()
self._current_by_session: dict[str, _PreparationJob] = {}
self._closed = True
@@ -237,6 +243,74 @@ class SessionRecordingPreparationManager:
self._current_by_session[command.session_id] = ready
return self._snapshot_locked(ready)
def restore_published(
self,
command: ReplayCommand,
) -> RecordingPreparationSnapshot | None:
"""Restore a complete durable package without enqueueing preparation.
The RRD and every recorded-media descriptor must already have been
published by an earlier successful job. Missing/stale pieces return a
cold miss; this method never invokes an exporter or media parser.
"""
source_command = _source_command(command)
identity = _source_identity(source_command)
with self._guard:
current = self._current_by_session.get(command.session_id)
if (
current is not None
and current.source_identity == identity
and current.state in ACTIVE_PREPARATION_STATES | {"ready"}
):
return self._snapshot_locked(current)
recording = self.materializer.restore_published(source_command)
if recording is None:
with self._guard:
current = self._current_by_session.get(command.session_id)
if current is not None and current.state == "ready":
self._current_by_session.pop(command.session_id, None)
return None
if self._ready_preparer is None:
recorded_media: tuple[RecordedMediaManifest, ...] = ()
else:
if self._ready_restorer is None:
return None
restored_media = self._ready_restorer(source_command, recording)
if restored_media is None:
return None
recorded_media = restored_media
validate_recorded_media_timeline(
recorded_media,
recording_start_seconds=recording.timeline_start_ns / 1_000_000_000,
recording_end_seconds=recording.timeline_end_ns / 1_000_000_000,
)
with self._guard:
current = self._current_by_session.get(command.session_id)
if (
current is not None
and current.source_identity == identity
and current.state in ACTIVE_PREPARATION_STATES | {"ready"}
):
if current.state == "ready":
current.recording = recording
current.recorded_media = recorded_media
return self._snapshot_locked(current)
ready = _PreparationJob(
preparation_id=uuid4().hex,
source_identity=identity,
command=source_command,
state="ready",
progress=1.0,
recording=recording,
recorded_media=recorded_media,
)
self._current_by_session[command.session_id] = ready
return self._snapshot_locked(ready)
def resolve_cached_pinned(
self,
command: ReplayCommand,
@@ -325,10 +399,7 @@ class SessionRecordingPreparationManager:
job is None
or job.state != "ready"
or job.recording is None
or (
preparation_id is not None
and job.preparation_id != preparation_id
)
or (preparation_id is not None and job.preparation_id != preparation_id)
):
return None
recording = job.recording
@@ -456,9 +527,7 @@ class SessionRecordingPreparationManager:
recording_start_seconds=(
recording.timeline_start_ns / 1_000_000_000
),
recording_end_seconds=(
recording.timeline_end_ns / 1_000_000_000
),
recording_end_seconds=(recording.timeline_end_ns / 1_000_000_000),
)
except Exception:
with self._guard:
src/k1link/sessions/recording.py
+40 -8
View File
@@ -232,6 +232,26 @@ class SessionRecordingMaterializer:
with self._lock_for(session_id):
return self._load_cached_recording(session_id, _validate_source(command))
def restore_published(self, command: ReplayCommand) -> MaterializedRecording | None:
"""Restore an immutable published cache using bounded metadata checks.
Full source/output digests are proved before the cache is published.
On a later process start the private cache is restored only when its
schema, source stat identities and derived-file stat identity still
match the durable sidecar. This keeps a prepared recording durable
across restarts without rereading gigabytes or starting an exporter.
"""
session_id = _validate_command_shape(command)
if not self._has_compatible_cache_candidate(session_id):
return None
with self._lock_for(session_id):
return self._load_cached_recording(
session_id,
_validate_source(command),
verify_digests=False,
)
def get_cached_pinned(
self,
command: ReplayCommand,
@@ -430,6 +450,7 @@ class SessionRecordingMaterializer:
source: _ValidatedSource,
*,
pin: bool = False,
verify_digests: bool = True,
) -> MaterializedRecording | None:
# Cache validation and the optional lease are one transaction with
# eviction. This makes it safe for FileResponse to open the path after
@@ -451,6 +472,7 @@ class SessionRecordingMaterializer:
source=source,
recording_path=recording_path,
sidecar_path=sidecar_path,
verify_digests=verify_digests,
)
if cached is None:
return None
@@ -661,6 +683,7 @@ class SessionRecordingMaterializer:
source: _ValidatedSource,
recording_path: Path,
sidecar_path: Path,
verify_digests: bool = True,
) -> MaterializedRecording | None:
if recording_path.is_symlink() or sidecar_path.is_symlink():
return None
@@ -698,12 +721,19 @@ class SessionRecordingMaterializer:
source_sha256: str | None = None
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
digest = _sha256_prefix_stable(
artifact.path,
artifact.file_stat,
artifact.replay_byte_length,
digest = (
_sha256_prefix_stable(
artifact.path,
artifact.file_stat,
artifact.replay_byte_length,
)
if verify_digests
else cached["sha256"]
)
if artifact.expected_sha256 is not None and digest != artifact.expected_sha256:
if (
artifact.expected_sha256 is not None
and cached["sha256"] != artifact.expected_sha256
):
raise RecordingMaterializationError(
"recording artifact digest no longer matches catalog"
)
@@ -713,9 +743,11 @@ class SessionRecordingMaterializer:
source_sha256 = digest
if source_sha256 is None or source_sha256 != document["source_sha256"]:
return None
recording_sha256 = _sha256_stable(recording_path, recording_stat)
if recording_sha256 != document["recording_sha256"]:
return None
recording_sha256 = document["recording_sha256"]
if verify_digests:
recording_sha256 = _sha256_stable(recording_path, recording_stat)
if recording_sha256 != document["recording_sha256"]:
return None
_chmod_best_effort(recording_path, 0o600)
_chmod_best_effort(sidecar_path, 0o600)
if document["schema_version"] != CACHE_SCHEMA:
src/k1link/web/app.py
+70 -28
View File
@@ -1,7 +1,7 @@
from __future__ import annotations
import asyncio
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Iterable
from contextlib import asynccontextmanager, suppress
from pathlib import Path
from typing import Any
@@ -65,9 +65,25 @@ def _prepare_recorded_media_for_launch(
)
def _restore_recorded_media_for_launch(
command: ReplayCommand,
_: MaterializedRecording,
) -> tuple[RecordedMediaManifest, ...] | None:
"""Restore only previously published media descriptors."""
restored: list[RecordedMediaManifest] = []
for artifact in session_store.list_recorded_media(command.session_id):
manifest = session_recorded_media_inspector.restore_prepared(artifact, command)
if manifest is None:
return None
restored.append(manifest)
return tuple(restored)
session_recording_preparation_manager = SessionRecordingPreparationManager(
session_recording_materializer,
ready_preparer=_prepare_recorded_media_for_launch,
ready_restorer=_restore_recorded_media_for_launch,
)
@@ -82,47 +98,73 @@ def refresh_observation_catalog() -> tuple[str, ...]:
return tuple(dict.fromkeys(imported))
def enqueue_replayable_recordings() -> tuple[str, ...]:
"""Reconcile finalized catalog sessions into the durable RRD work queue."""
def finalized_replayable_recording_ids() -> tuple[str, ...]:
"""List finalized replayable catalog identities without scheduling work."""
enqueued: list[str] = []
finalized: list[str] = []
cursor: str | None = None
while True:
page = session_store.list_recent(limit=100, cursor=cursor)
for summary in page.items:
if not summary.replayable or summary.status not in {
"ready",
"interrupted",
"failed",
}:
continue
try:
command = session_store.prepare_replay(summary.session_id)
session_recording_preparation_manager.enqueue(
command,
retry_interrupted=True,
)
except RecordingPreparationQueueFull:
return tuple(enqueued)
except Exception:
# One stale/corrupt catalog row must not starve every valid
# finalized session that follows it in the reconciliation
# page. The row remains visible as failed/interrupted evidence
# and can be repaired independently.
continue
enqueued.append(summary.session_id)
finalized.extend(
summary.session_id
for summary in page.items
if summary.replayable and summary.status in {"ready", "interrupted", "failed"}
)
cursor = page.next_cursor
if cursor is None:
return tuple(finalized)
def enqueue_replayable_recordings(session_ids: Iterable[str]) -> tuple[str, ...]:
"""Schedule only explicitly selected newly finalized sessions."""
enqueued: list[str] = []
for session_id in dict.fromkeys(session_ids):
try:
command = session_store.prepare_replay(session_id)
session_recording_preparation_manager.enqueue(
command,
retry_interrupted=True,
)
except RecordingPreparationQueueFull:
return tuple(enqueued)
except Exception:
# One stale/corrupt row must not starve a later newly completed
# session. Historical cold caches remain operator-triggered.
continue
enqueued.append(session_id)
return tuple(enqueued)
def newly_finalized_recording_ids(
known_finalized: set[str] | None,
current_finalized: Iterable[str],
) -> tuple[str, ...]:
"""Return only post-startup completions; the first scan is a baseline."""
if known_finalized is None:
return ()
return tuple(sorted(set(current_finalized) - known_finalized))
async def _recording_preparation_reconciler() -> None:
"""Discover newly completed/recovered captures without blocking requests."""
"""Prepare sessions finalized during this process, never historical rows."""
known_finalized: set[str] | None = None
while True:
try:
await asyncio.to_thread(refresh_observation_catalog)
await asyncio.to_thread(enqueue_replayable_recordings)
finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids))
newly_finalized = newly_finalized_recording_ids(
known_finalized,
finalized,
)
if newly_finalized:
await asyncio.to_thread(
enqueue_replayable_recordings,
newly_finalized,
)
known_finalized = finalized
except Exception:
# A transient filesystem/catalog failure must not permanently
# disable preparation of sessions completed later in the run.
src/k1link/web/session_api.py
+24 -25
View File
@@ -45,9 +45,7 @@ SAFE_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$")
SAFE_SHA256 = re.compile(r"^[a-f0-9]{64}$")
MAX_SAFE_INTEGER = 9_007_199_254_740_991
REVALIDATED_RECORDING_CACHE_CONTROL = "private, no-cache, no-transform"
IMMUTABLE_RECORDING_CACHE_CONTROL = (
"private, max-age=31536000, immutable, no-transform"
)
IMMUTABLE_RECORDING_CACHE_CONTROL = "private, max-age=31536000, immutable, no-transform"
class _ReleasingFileResponse(FileResponse):
@@ -321,13 +319,17 @@ def build_session_router(
if recording_preparation_manager is not None:
try:
# Enqueue performs only catalog identity/lstat checks. A cold
# RRD cache and every camera index are validated exclusively
# by the process-owned worker.
snapshot = recording_preparation_manager.enqueue(
command,
retry_failed=True,
)
snapshot = recording_preparation_manager.status(command.session_id)
if snapshot is None:
snapshot = await run_in_threadpool(
recording_preparation_manager.restore_published,
command,
)
if snapshot is None:
snapshot = recording_preparation_manager.enqueue(
command,
retry_failed=True,
)
except RecordingPreparationQueueFull as exc:
raise HTTPException(
status_code=503,
@@ -504,7 +506,12 @@ def build_session_router(
False,
False,
)
snapshot = recording_preparation_manager.enqueue(command)
snapshot = await run_in_threadpool(
recording_preparation_manager.restore_published,
command,
)
if snapshot is None:
snapshot = recording_preparation_manager.enqueue(command)
except SessionNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except (SessionNotReplayableError, SessionIntegrityError) as exc:
@@ -1036,15 +1043,16 @@ def _catalog_preparation_document(
snapshot = manager.status(session_id)
if snapshot is None:
try:
snapshot = manager.enqueue(store.prepare_replay(session_id))
snapshot = manager.restore_published(store.prepare_replay(session_id))
except (
RecordingPreparationQueueFull,
SessionNotFoundError,
SessionNotReplayableError,
SessionIntegrityError,
ValueError,
):
return None
if snapshot is None:
return None
document: dict[str, Any] = {
"preparation_id": snapshot.preparation_id,
"state": snapshot.state,
@@ -1165,10 +1173,7 @@ def _recorded_media_manifest_document(
"segments": [
{
"sequence": segment.sequence,
"url": (
f"{base}/epochs/{epoch.ordinal}/segments/"
f"{segment.sequence}.m4s"
),
"url": (f"{base}/epochs/{epoch.ordinal}/segments/{segment.sequence}.m4s"),
"byte_length": segment.byte_length,
"sha256": segment.sha256,
}
@@ -1198,19 +1203,13 @@ def _resolve_recorded_media_manifest(
detail="Некорректный идентификатор записанного медиаканала.",
) from exc
snapshot = manager.status(session_id)
if (
snapshot is None
or snapshot.state != "ready"
or snapshot.recorded_media is None
):
if snapshot is None or snapshot.state != "ready" or snapshot.recorded_media is None:
raise HTTPException(
status_code=409,
detail="Медиаканал ещё не подготовлен.",
)
matches = tuple(
manifest
for manifest in snapshot.recorded_media
if manifest.artifact_id == artifact_id
manifest for manifest in snapshot.recorded_media if manifest.artifact_id == artifact_id
)
if len(matches) != 1:
raise HTTPException(status_code=404, detail="Записанный медиаканал не найден.")