fix(replay): recover failed archives and release deleted sessions

This commit is contained in:
DCCONSTRUCTIONS
2026-07-29 01:55:23 +03:00
parent 2a97cf28c0
commit 1eba7c82ba
8 changed files with 217 additions and 7 deletions
+15
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import logging
import os
import queue
import threading
@@ -20,6 +21,8 @@ from .recording import (
SessionRecordingMaterializer,
)
LOGGER = logging.getLogger(__name__)
PreparationState = Literal[
"queued",
"validating",
@@ -582,6 +585,12 @@ class SessionRecordingPreparationManager:
except Exception:
# The public status intentionally does not expose paths,
# broker payloads or exporter internals.
LOGGER.exception(
"recording preparation materialization failed: "
"session_id=%s preparation_id=%s",
job.command.session_id,
job.preparation_id,
)
with self._guard:
if job.cancel_event.is_set():
self._transition_locked(job, "cancelled", job.progress)
@@ -616,6 +625,12 @@ class SessionRecordingPreparationManager:
recording_end_seconds=(recording.timeline_end_ns / 1_000_000_000),
)
except Exception:
LOGGER.exception(
"recording preparation finalization failed: "
"session_id=%s preparation_id=%s",
job.command.session_id,
job.preparation_id,
)
with self._guard:
if job.cancel_event.is_set():
self._transition_locked(job, "cancelled", job.progress)
+14 -5
View File
@@ -372,8 +372,10 @@ def build_session_router(
try:
store.get_session(session_id)
recorded_artifacts = store.list_recorded_media(session_id)
except SessionNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except SessionNotFoundError:
# DELETE is idempotent. Multiple mounted catalogs or another
# operator tab may have already removed the exact session.
return Response(status_code=204)
except ValueError as exc:
raise HTTPException(
status_code=422,
@@ -427,8 +429,10 @@ def build_session_router(
(artifact.artifact_id for artifact in recorded_artifacts),
)
await run_in_threadpool(store.delete_session, session_id)
except SessionNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except SessionNotFoundError:
# A concurrent idempotent delete won the race after both callers
# resolved the catalog row.
return Response(status_code=204)
except (OSError, SessionIntegrityError) as exc:
raise HTTPException(
status_code=409,
@@ -469,7 +473,12 @@ def build_session_router(
recording_preparation_manager.restore_published,
command,
)
if snapshot is None:
if snapshot is None or snapshot.state in {"failed", "cancelled"}:
# A terminal preparation describes one attempt, not the
# durable session. A fresh POST /replay is explicit retry
# intent and must replace that attempt. This also recovers
# the camera-finalization race where the RRD is already
# published but the media archive seals moments later.
snapshot = recording_preparation_manager.enqueue(
command,
retry_failed=True,