fix(archive): preserve prepared sessions across restarts
This commit is contained in:
+70
-28
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Iterable
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -65,9 +65,25 @@ def _prepare_recorded_media_for_launch(
|
||||
)
|
||||
|
||||
|
||||
def _restore_recorded_media_for_launch(
|
||||
command: ReplayCommand,
|
||||
_: MaterializedRecording,
|
||||
) -> tuple[RecordedMediaManifest, ...] | None:
|
||||
"""Restore only previously published media descriptors."""
|
||||
|
||||
restored: list[RecordedMediaManifest] = []
|
||||
for artifact in session_store.list_recorded_media(command.session_id):
|
||||
manifest = session_recorded_media_inspector.restore_prepared(artifact, command)
|
||||
if manifest is None:
|
||||
return None
|
||||
restored.append(manifest)
|
||||
return tuple(restored)
|
||||
|
||||
|
||||
session_recording_preparation_manager = SessionRecordingPreparationManager(
|
||||
session_recording_materializer,
|
||||
ready_preparer=_prepare_recorded_media_for_launch,
|
||||
ready_restorer=_restore_recorded_media_for_launch,
|
||||
)
|
||||
|
||||
|
||||
@@ -82,47 +98,73 @@ def refresh_observation_catalog() -> tuple[str, ...]:
|
||||
return tuple(dict.fromkeys(imported))
|
||||
|
||||
|
||||
def enqueue_replayable_recordings() -> tuple[str, ...]:
|
||||
"""Reconcile finalized catalog sessions into the durable RRD work queue."""
|
||||
def finalized_replayable_recording_ids() -> tuple[str, ...]:
|
||||
"""List finalized replayable catalog identities without scheduling work."""
|
||||
|
||||
enqueued: list[str] = []
|
||||
finalized: list[str] = []
|
||||
cursor: str | None = None
|
||||
while True:
|
||||
page = session_store.list_recent(limit=100, cursor=cursor)
|
||||
for summary in page.items:
|
||||
if not summary.replayable or summary.status not in {
|
||||
"ready",
|
||||
"interrupted",
|
||||
"failed",
|
||||
}:
|
||||
continue
|
||||
try:
|
||||
command = session_store.prepare_replay(summary.session_id)
|
||||
session_recording_preparation_manager.enqueue(
|
||||
command,
|
||||
retry_interrupted=True,
|
||||
)
|
||||
except RecordingPreparationQueueFull:
|
||||
return tuple(enqueued)
|
||||
except Exception:
|
||||
# One stale/corrupt catalog row must not starve every valid
|
||||
# finalized session that follows it in the reconciliation
|
||||
# page. The row remains visible as failed/interrupted evidence
|
||||
# and can be repaired independently.
|
||||
continue
|
||||
enqueued.append(summary.session_id)
|
||||
finalized.extend(
|
||||
summary.session_id
|
||||
for summary in page.items
|
||||
if summary.replayable and summary.status in {"ready", "interrupted", "failed"}
|
||||
)
|
||||
cursor = page.next_cursor
|
||||
if cursor is None:
|
||||
return tuple(finalized)
|
||||
|
||||
|
||||
def enqueue_replayable_recordings(session_ids: Iterable[str]) -> tuple[str, ...]:
|
||||
"""Schedule only explicitly selected newly finalized sessions."""
|
||||
|
||||
enqueued: list[str] = []
|
||||
for session_id in dict.fromkeys(session_ids):
|
||||
try:
|
||||
command = session_store.prepare_replay(session_id)
|
||||
session_recording_preparation_manager.enqueue(
|
||||
command,
|
||||
retry_interrupted=True,
|
||||
)
|
||||
except RecordingPreparationQueueFull:
|
||||
return tuple(enqueued)
|
||||
except Exception:
|
||||
# One stale/corrupt row must not starve a later newly completed
|
||||
# session. Historical cold caches remain operator-triggered.
|
||||
continue
|
||||
enqueued.append(session_id)
|
||||
return tuple(enqueued)
|
||||
|
||||
|
||||
def newly_finalized_recording_ids(
|
||||
known_finalized: set[str] | None,
|
||||
current_finalized: Iterable[str],
|
||||
) -> tuple[str, ...]:
|
||||
"""Return only post-startup completions; the first scan is a baseline."""
|
||||
|
||||
if known_finalized is None:
|
||||
return ()
|
||||
return tuple(sorted(set(current_finalized) - known_finalized))
|
||||
|
||||
|
||||
async def _recording_preparation_reconciler() -> None:
|
||||
"""Discover newly completed/recovered captures without blocking requests."""
|
||||
"""Prepare sessions finalized during this process, never historical rows."""
|
||||
|
||||
known_finalized: set[str] | None = None
|
||||
while True:
|
||||
try:
|
||||
await asyncio.to_thread(refresh_observation_catalog)
|
||||
await asyncio.to_thread(enqueue_replayable_recordings)
|
||||
finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids))
|
||||
newly_finalized = newly_finalized_recording_ids(
|
||||
known_finalized,
|
||||
finalized,
|
||||
)
|
||||
if newly_finalized:
|
||||
await asyncio.to_thread(
|
||||
enqueue_replayable_recordings,
|
||||
newly_finalized,
|
||||
)
|
||||
known_finalized = finalized
|
||||
except Exception:
|
||||
# A transient filesystem/catalog failure must not permanently
|
||||
# disable preparation of sessions completed later in the run.
|
||||
|
||||
Reference in New Issue
Block a user