fix(replay): recover failed archives and release deleted sessions
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
@@ -20,6 +21,8 @@ from .recording import (
|
||||
SessionRecordingMaterializer,
|
||||
)
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
PreparationState = Literal[
|
||||
"queued",
|
||||
"validating",
|
||||
@@ -582,6 +585,12 @@ class SessionRecordingPreparationManager:
|
||||
except Exception:
|
||||
# The public status intentionally does not expose paths,
|
||||
# broker payloads or exporter internals.
|
||||
LOGGER.exception(
|
||||
"recording preparation materialization failed: "
|
||||
"session_id=%s preparation_id=%s",
|
||||
job.command.session_id,
|
||||
job.preparation_id,
|
||||
)
|
||||
with self._guard:
|
||||
if job.cancel_event.is_set():
|
||||
self._transition_locked(job, "cancelled", job.progress)
|
||||
@@ -616,6 +625,12 @@ class SessionRecordingPreparationManager:
|
||||
recording_end_seconds=(recording.timeline_end_ns / 1_000_000_000),
|
||||
)
|
||||
except Exception:
|
||||
LOGGER.exception(
|
||||
"recording preparation finalization failed: "
|
||||
"session_id=%s preparation_id=%s",
|
||||
job.command.session_id,
|
||||
job.preparation_id,
|
||||
)
|
||||
with self._guard:
|
||||
if job.cancel_event.is_set():
|
||||
self._transition_locked(job, "cancelled", job.progress)
|
||||
|
||||
@@ -372,8 +372,10 @@ def build_session_router(
|
||||
try:
|
||||
store.get_session(session_id)
|
||||
recorded_artifacts = store.list_recorded_media(session_id)
|
||||
except SessionNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except SessionNotFoundError:
|
||||
# DELETE is idempotent. Multiple mounted catalogs or another
|
||||
# operator tab may have already removed the exact session.
|
||||
return Response(status_code=204)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
@@ -427,8 +429,10 @@ def build_session_router(
|
||||
(artifact.artifact_id for artifact in recorded_artifacts),
|
||||
)
|
||||
await run_in_threadpool(store.delete_session, session_id)
|
||||
except SessionNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except SessionNotFoundError:
|
||||
# A concurrent idempotent delete won the race after both callers
|
||||
# resolved the catalog row.
|
||||
return Response(status_code=204)
|
||||
except (OSError, SessionIntegrityError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
@@ -469,7 +473,12 @@ def build_session_router(
|
||||
recording_preparation_manager.restore_published,
|
||||
command,
|
||||
)
|
||||
if snapshot is None:
|
||||
if snapshot is None or snapshot.state in {"failed", "cancelled"}:
|
||||
# A terminal preparation describes one attempt, not the
|
||||
# durable session. A fresh POST /replay is explicit retry
|
||||
# intent and must replace that attempt. This also recovers
|
||||
# the camera-finalization race where the RRD is already
|
||||
# published but the media archive seals moments later.
|
||||
snapshot = recording_preparation_manager.enqueue(
|
||||
command,
|
||||
retry_failed=True,
|
||||
|
||||
Reference in New Issue
Block a user