feat(archive): complete recorded session lifecycle
This commit is contained in:
@@ -8,7 +8,7 @@ import re
|
||||
import secrets
|
||||
import stat
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Iterable, Iterator
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
@@ -175,6 +175,36 @@ class RecordedMediaInspector:
|
||||
self._cache[key] = _CachedManifest(identity=identity, manifest=manifest)
|
||||
return manifest
|
||||
|
||||
def delete_prepared(self, session_id: str, artifact_ids: Iterable[str]) -> None:
|
||||
"""Forget and remove path-free preparation sidecars for one session."""
|
||||
|
||||
unique_artifact_ids = tuple(dict.fromkeys(artifact_ids))
|
||||
with self._lock:
|
||||
for artifact_id in unique_artifact_ids:
|
||||
self._cache.pop((session_id, artifact_id), None)
|
||||
if self._cache_root is None:
|
||||
return
|
||||
for artifact_id in unique_artifact_ids:
|
||||
sidecar = self._cache_root / _sidecar_name(session_id, artifact_id)
|
||||
try:
|
||||
metadata = sidecar.lstat()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
except OSError as exc:
|
||||
raise SessionIntegrityError(
|
||||
"recorded media preparation could not be inspected"
|
||||
) from exc
|
||||
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
|
||||
raise SessionIntegrityError(
|
||||
"recorded media preparation is not a regular file"
|
||||
)
|
||||
try:
|
||||
sidecar.unlink()
|
||||
except OSError as exc:
|
||||
raise SessionIntegrityError(
|
||||
"recorded media preparation could not be deleted"
|
||||
) from exc
|
||||
|
||||
def restore_prepared(
|
||||
self,
|
||||
artifact: RecordedMediaArtifact,
|
||||
|
||||
@@ -87,6 +87,13 @@ class _WorkerGeneration:
|
||||
worker: threading.Thread | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _LaunchReservation:
|
||||
preparation_id: str
|
||||
release: Callable[[], None]
|
||||
timer: threading.Timer
|
||||
|
||||
|
||||
class SessionRecordingPreparationManager:
|
||||
"""One bounded, process-owned conversion worker for durable recordings.
|
||||
|
||||
@@ -131,6 +138,7 @@ class SessionRecordingPreparationManager:
|
||||
self._ready_restorer = ready_restorer
|
||||
self._guard = threading.RLock()
|
||||
self._current_by_session: dict[str, _PreparationJob] = {}
|
||||
self._launch_reservations: dict[str, _LaunchReservation] = {}
|
||||
self._closed = True
|
||||
self._generation_counter = 0
|
||||
self._active_generation: _WorkerGeneration | None = None
|
||||
@@ -379,10 +387,7 @@ class SessionRecordingPreparationManager:
|
||||
if pinned is None:
|
||||
return None
|
||||
snapshot, release = pinned
|
||||
timer = threading.Timer(lease_seconds, release)
|
||||
timer.name = f"missioncore-recording-launch-lease-{snapshot.preparation_id}"
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
self._reserve_launch_lease(snapshot, release, lease_seconds)
|
||||
return snapshot
|
||||
|
||||
def pin_ready(
|
||||
@@ -393,6 +398,7 @@ class SessionRecordingPreparationManager:
|
||||
) -> tuple[RecordingPreparationSnapshot, Callable[[], None]] | None:
|
||||
"""Cheaply lease the exact already-validated ready generation."""
|
||||
|
||||
launch_release: Callable[[], None] | None = None
|
||||
with self._guard:
|
||||
job = self._current_by_session.get(session_id)
|
||||
if (
|
||||
@@ -409,7 +415,17 @@ class SessionRecordingPreparationManager:
|
||||
self._current_by_session.pop(session_id, None)
|
||||
return None
|
||||
snapshot = self._snapshot_locked(job)
|
||||
return snapshot, pinned
|
||||
reservation = self._launch_reservations.get(session_id)
|
||||
if (
|
||||
reservation is not None
|
||||
and reservation.preparation_id == snapshot.preparation_id
|
||||
):
|
||||
self._launch_reservations.pop(session_id, None)
|
||||
reservation.timer.cancel()
|
||||
launch_release = reservation.release
|
||||
if launch_release is not None:
|
||||
launch_release()
|
||||
return snapshot, pinned
|
||||
|
||||
def reserve_ready(
|
||||
self,
|
||||
@@ -426,12 +442,26 @@ class SessionRecordingPreparationManager:
|
||||
if pinned is None:
|
||||
return None
|
||||
snapshot, release = pinned
|
||||
timer = threading.Timer(lease_seconds, release)
|
||||
timer.name = f"missioncore-recording-launch-lease-{snapshot.preparation_id}"
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
self._reserve_launch_lease(snapshot, release, lease_seconds)
|
||||
return snapshot
|
||||
|
||||
def release_launch_reservation(self, session_id: str) -> bool:
|
||||
"""Release an unused launch-to-GET lease for one session.
|
||||
|
||||
The first matching recording GET consumes this reservation after it
|
||||
acquires its own response-lifetime pin. Deletion may release a launch
|
||||
reservation that was never consumed; an active response pin remains
|
||||
independently protected by the materializer.
|
||||
"""
|
||||
|
||||
with self._guard:
|
||||
reservation = self._launch_reservations.pop(session_id, None)
|
||||
if reservation is None:
|
||||
return False
|
||||
reservation.timer.cancel()
|
||||
reservation.release()
|
||||
return True
|
||||
|
||||
def cancel(self, session_id: str, *, preparation_id: str | None = None) -> bool:
|
||||
with self._guard:
|
||||
job = self._current_by_session.get(session_id)
|
||||
@@ -447,11 +477,30 @@ class SessionRecordingPreparationManager:
|
||||
self._transition_locked(job, "cancelled", job.progress)
|
||||
return True
|
||||
|
||||
def discard(self, session_id: str) -> bool:
|
||||
"""Forget one non-active job before its source session is deleted."""
|
||||
|
||||
with self._guard:
|
||||
job = self._current_by_session.get(session_id)
|
||||
if job is not None and job.state in ACTIVE_PREPARATION_STATES:
|
||||
return False
|
||||
self._current_by_session.pop(session_id, None)
|
||||
reservation = self._launch_reservations.pop(session_id, None)
|
||||
if reservation is not None:
|
||||
reservation.timer.cancel()
|
||||
if reservation is not None:
|
||||
reservation.release()
|
||||
return True
|
||||
|
||||
def close(self, *, timeout: float = 5.0) -> None:
|
||||
with self._guard:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
reservations = tuple(self._launch_reservations.values())
|
||||
self._launch_reservations.clear()
|
||||
for reservation in reservations:
|
||||
reservation.timer.cancel()
|
||||
for job in self._current_by_session.values():
|
||||
if job.state in ACTIVE_PREPARATION_STATES:
|
||||
job.interrupted_by_restart = not job.cancelled_by_operator
|
||||
@@ -466,9 +515,46 @@ class SessionRecordingPreparationManager:
|
||||
pending.stop_event.set()
|
||||
self._pending_generation = None
|
||||
worker = None if active is None else active.worker
|
||||
for reservation in reservations:
|
||||
reservation.release()
|
||||
if worker is not None:
|
||||
worker.join(timeout=max(0.0, timeout))
|
||||
|
||||
def _reserve_launch_lease(
|
||||
self,
|
||||
snapshot: RecordingPreparationSnapshot,
|
||||
release: Callable[[], None],
|
||||
lease_seconds: float,
|
||||
) -> None:
|
||||
timer = threading.Timer(
|
||||
lease_seconds,
|
||||
self._expire_launch_reservation,
|
||||
args=(snapshot.session_id, snapshot.preparation_id),
|
||||
)
|
||||
timer.name = f"missioncore-recording-launch-lease-{snapshot.preparation_id}"
|
||||
timer.daemon = True
|
||||
reservation = _LaunchReservation(
|
||||
preparation_id=snapshot.preparation_id,
|
||||
release=release,
|
||||
timer=timer,
|
||||
)
|
||||
with self._guard:
|
||||
previous = self._launch_reservations.pop(snapshot.session_id, None)
|
||||
if previous is not None:
|
||||
previous.timer.cancel()
|
||||
self._launch_reservations[snapshot.session_id] = reservation
|
||||
if previous is not None:
|
||||
previous.release()
|
||||
timer.start()
|
||||
|
||||
def _expire_launch_reservation(self, session_id: str, preparation_id: str) -> None:
|
||||
with self._guard:
|
||||
reservation = self._launch_reservations.get(session_id)
|
||||
if reservation is None or reservation.preparation_id != preparation_id:
|
||||
return
|
||||
self._launch_reservations.pop(session_id, None)
|
||||
reservation.release()
|
||||
|
||||
def _run_generation(self, generation: _WorkerGeneration) -> None:
|
||||
work_queue = generation.work_queue
|
||||
try:
|
||||
|
||||
@@ -218,6 +218,30 @@ class SessionRecordingMaterializer:
|
||||
self._increment_pin_locked(recording.session_id)
|
||||
return self._release_callback(recording.session_id)
|
||||
|
||||
def delete_cached(self, session_id: str) -> bool:
|
||||
"""Delete one exact derived RRD cache unless a response still leases it."""
|
||||
|
||||
if SESSION_ID_PATTERN.fullmatch(session_id) is None:
|
||||
raise ValueError("recording cache session id is invalid")
|
||||
with self._lock_for(session_id), self._cache_guard:
|
||||
if self._pinned_sessions.get(session_id, 0) > 0:
|
||||
return False
|
||||
session_root = self.recordings_root / session_id
|
||||
if session_root.exists() or session_root.is_symlink():
|
||||
metadata = session_root.lstat()
|
||||
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode):
|
||||
raise RecordingMaterializationError(
|
||||
"session recording cache is not a real directory"
|
||||
)
|
||||
if session_root.parent.resolve(strict=True) != self.recordings_root:
|
||||
raise RecordingMaterializationError(
|
||||
"session recording cache escapes the private data root"
|
||||
)
|
||||
shutil.rmtree(session_root)
|
||||
with self._memory_guard:
|
||||
self._validated_memory.pop(session_id, None)
|
||||
return True
|
||||
|
||||
def __call__(self, command: ReplayCommand) -> MaterializedRecording:
|
||||
"""Alias for :meth:`materialize`, suitable for the web API protocol."""
|
||||
|
||||
|
||||
@@ -3,12 +3,15 @@ from __future__ import annotations
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import stat
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
from uuid import uuid4
|
||||
|
||||
from k1link.artifacts import utc_now_iso
|
||||
|
||||
@@ -142,11 +145,18 @@ class SessionStore:
|
||||
candidates = source.discover(allowed_root)
|
||||
imported: list[str] = []
|
||||
for candidate in candidates:
|
||||
self._upsert_candidate(source, candidate)
|
||||
try:
|
||||
self._upsert_candidate(source, candidate)
|
||||
except SessionIntegrityError:
|
||||
# One incomplete/corrupt evidence directory must not starve a
|
||||
# later valid session in the same plugin archive. Keep any
|
||||
# previously indexed row because the candidate is still
|
||||
# physically present, but never admit its current artifacts.
|
||||
continue
|
||||
imported.append(candidate.session_id)
|
||||
with self._lock, self._connect() as connection:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
discovered = set(imported)
|
||||
discovered = {candidate.session_id for candidate in candidates}
|
||||
indexed = connection.execute(
|
||||
"SELECT session_id FROM observation_sessions "
|
||||
"WHERE plugin_id = ? AND archive_id = ? AND allowed_root = ?",
|
||||
@@ -232,6 +242,60 @@ class SessionStore:
|
||||
)
|
||||
return SessionDetail(summary=_summary_from_row(row), sources=sources, artifacts=artifacts)
|
||||
|
||||
def delete_session(self, session_id: str) -> None:
|
||||
"""Permanently delete one exact catalogued evidence directory and row."""
|
||||
|
||||
_validate_identifier(session_id, "session id")
|
||||
with self._lock:
|
||||
with self._connect() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT allowed_root, session_root FROM observation_sessions "
|
||||
"WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise SessionNotFoundError("observation session was not found")
|
||||
|
||||
allowed_root = Path(row["allowed_root"]).expanduser().resolve()
|
||||
session_root = Path(row["session_root"]).expanduser()
|
||||
try:
|
||||
unresolved_parent = session_root.parent.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise SessionIntegrityError("session deletion root is unavailable") from exc
|
||||
if (
|
||||
unresolved_parent != allowed_root
|
||||
or session_root.name != session_id
|
||||
or session_root == allowed_root
|
||||
):
|
||||
raise SessionIntegrityError("session deletion target escapes its allowed root")
|
||||
|
||||
if session_root.exists() or session_root.is_symlink():
|
||||
try:
|
||||
metadata = session_root.lstat()
|
||||
except OSError as exc:
|
||||
raise SessionIntegrityError("session deletion target is unavailable") from exc
|
||||
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode):
|
||||
raise SessionIntegrityError("session deletion target is not a real directory")
|
||||
tombstone = allowed_root / f".{session_id}.{uuid4().hex}.deleting"
|
||||
try:
|
||||
os.replace(session_root, tombstone)
|
||||
shutil.rmtree(tombstone)
|
||||
except OSError as exc:
|
||||
if tombstone.exists() and not session_root.exists():
|
||||
with _ignore_os_error():
|
||||
os.replace(tombstone, session_root)
|
||||
raise SessionIntegrityError("session evidence could not be deleted") from exc
|
||||
|
||||
with self._connect() as connection:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
deleted = connection.execute(
|
||||
"DELETE FROM observation_sessions WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).rowcount
|
||||
connection.commit()
|
||||
if deleted != 1:
|
||||
raise SessionNotFoundError("observation session was not found")
|
||||
|
||||
def prepare_replay(
|
||||
self,
|
||||
session_id: str,
|
||||
|
||||
@@ -303,6 +303,79 @@ def build_session_router(
|
||||
detail="Некорректный идентификатор сессии.",
|
||||
) from exc
|
||||
|
||||
@router.delete(
|
||||
"/api/v1/observation-sessions/{session_id}",
|
||||
status_code=204,
|
||||
)
|
||||
async def delete_observation_session(session_id: str) -> Response:
|
||||
try:
|
||||
store.get_session(session_id)
|
||||
recorded_artifacts = store.list_recorded_media(session_id)
|
||||
except SessionNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail="Некорректный идентификатор сессии.",
|
||||
) from exc
|
||||
|
||||
if recording_preparation_manager is not None:
|
||||
snapshot = recording_preparation_manager.status(session_id)
|
||||
if snapshot is not None and snapshot.state in {
|
||||
"queued",
|
||||
"validating",
|
||||
"exporting",
|
||||
"finalizing",
|
||||
}:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Дождитесь завершения подготовки записи перед удалением.",
|
||||
)
|
||||
await run_in_threadpool(
|
||||
recording_preparation_manager.release_launch_reservation,
|
||||
session_id,
|
||||
)
|
||||
|
||||
cache_deleter = getattr(recording_materializer, "delete_cached", None)
|
||||
if callable(cache_deleter):
|
||||
try:
|
||||
cache_deleted = await run_in_threadpool(cache_deleter, session_id)
|
||||
except (OSError, RecordingMaterializationError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Подготовленную запись пока нельзя удалить.",
|
||||
) from exc
|
||||
if cache_deleted is not True:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Запись сейчас открыта в интерфейсе. Закройте её и повторите удаление.",
|
||||
)
|
||||
|
||||
if (
|
||||
recording_preparation_manager is not None
|
||||
and not recording_preparation_manager.discard(session_id)
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Подготовка записи ещё выполняется.",
|
||||
)
|
||||
|
||||
try:
|
||||
await run_in_threadpool(
|
||||
recorded_media_inspector.delete_prepared,
|
||||
session_id,
|
||||
(artifact.artifact_id for artifact in recorded_artifacts),
|
||||
)
|
||||
await run_in_threadpool(store.delete_session, session_id)
|
||||
except SessionNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except (OSError, SessionIntegrityError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Сервер не смог безопасно удалить файлы этой сессии.",
|
||||
) from exc
|
||||
return Response(status_code=204)
|
||||
|
||||
@router.post("/api/v1/observation-sessions/{session_id}/replay")
|
||||
async def replay_observation_session(
|
||||
session_id: str,
|
||||
|
||||
Reference in New Issue
Block a user