fix(archive): preserve prepared sessions across restarts

This commit is contained in:
DCCONSTRUCTIONS
2026-07-18 11:08:27 +03:00
parent aa3680948f
commit 574a494759
13 changed files with 398 additions and 132 deletions
+70 -28
View File
@@ -1,7 +1,7 @@
from __future__ import annotations
import asyncio
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Iterable
from contextlib import asynccontextmanager, suppress
from pathlib import Path
from typing import Any
@@ -65,9 +65,25 @@ def _prepare_recorded_media_for_launch(
)
def _restore_recorded_media_for_launch(
command: ReplayCommand,
_: MaterializedRecording,
) -> tuple[RecordedMediaManifest, ...] | None:
"""Restore only previously published media descriptors."""
restored: list[RecordedMediaManifest] = []
for artifact in session_store.list_recorded_media(command.session_id):
manifest = session_recorded_media_inspector.restore_prepared(artifact, command)
if manifest is None:
return None
restored.append(manifest)
return tuple(restored)
session_recording_preparation_manager = SessionRecordingPreparationManager(
session_recording_materializer,
ready_preparer=_prepare_recorded_media_for_launch,
ready_restorer=_restore_recorded_media_for_launch,
)
@@ -82,47 +98,73 @@ def refresh_observation_catalog() -> tuple[str, ...]:
return tuple(dict.fromkeys(imported))
def enqueue_replayable_recordings() -> tuple[str, ...]:
"""Reconcile finalized catalog sessions into the durable RRD work queue."""
def finalized_replayable_recording_ids() -> tuple[str, ...]:
"""List finalized replayable catalog identities without scheduling work."""
enqueued: list[str] = []
finalized: list[str] = []
cursor: str | None = None
while True:
page = session_store.list_recent(limit=100, cursor=cursor)
for summary in page.items:
if not summary.replayable or summary.status not in {
"ready",
"interrupted",
"failed",
}:
continue
try:
command = session_store.prepare_replay(summary.session_id)
session_recording_preparation_manager.enqueue(
command,
retry_interrupted=True,
)
except RecordingPreparationQueueFull:
return tuple(enqueued)
except Exception:
# One stale/corrupt catalog row must not starve every valid
# finalized session that follows it in the reconciliation
# page. The row remains visible as failed/interrupted evidence
# and can be repaired independently.
continue
enqueued.append(summary.session_id)
finalized.extend(
summary.session_id
for summary in page.items
if summary.replayable and summary.status in {"ready", "interrupted", "failed"}
)
cursor = page.next_cursor
if cursor is None:
return tuple(finalized)
def enqueue_replayable_recordings(session_ids: Iterable[str]) -> tuple[str, ...]:
"""Schedule only explicitly selected newly finalized sessions."""
enqueued: list[str] = []
for session_id in dict.fromkeys(session_ids):
try:
command = session_store.prepare_replay(session_id)
session_recording_preparation_manager.enqueue(
command,
retry_interrupted=True,
)
except RecordingPreparationQueueFull:
return tuple(enqueued)
except Exception:
# One stale/corrupt row must not starve a later newly completed
# session. Historical cold caches remain operator-triggered.
continue
enqueued.append(session_id)
return tuple(enqueued)
def newly_finalized_recording_ids(
known_finalized: set[str] | None,
current_finalized: Iterable[str],
) -> tuple[str, ...]:
"""Return only post-startup completions; the first scan is a baseline."""
if known_finalized is None:
return ()
return tuple(sorted(set(current_finalized) - known_finalized))
async def _recording_preparation_reconciler() -> None:
"""Discover newly completed/recovered captures without blocking requests."""
"""Prepare sessions finalized during this process, never historical rows."""
known_finalized: set[str] | None = None
while True:
try:
await asyncio.to_thread(refresh_observation_catalog)
await asyncio.to_thread(enqueue_replayable_recordings)
finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids))
newly_finalized = newly_finalized_recording_ids(
known_finalized,
finalized,
)
if newly_finalized:
await asyncio.to_thread(
enqueue_replayable_recordings,
newly_finalized,
)
known_finalized = finalized
except Exception:
# A transient filesystem/catalog failure must not permanently
# disable preparation of sessions completed later in the run.