from __future__ import annotations import os import queue import threading from collections.abc import Callable from dataclasses import dataclass, field, replace from datetime import UTC, datetime from functools import partial from pathlib import Path from time import monotonic from typing import Literal, cast from uuid import uuid4 from .media import RecordedMediaManifest, validate_recorded_media_timeline from .models import ReplayCommand from .recording import ( MaterializedRecording, RecordingMaterializationCancelled, SessionRecordingMaterializer, ) PreparationState = Literal[ "queued", "validating", "exporting", "finalizing", "ready", "failed", "cancelled", ] ACTIVE_PREPARATION_STATES = frozenset({"queued", "validating", "exporting", "finalizing"}) _PREPARATION_PHASE: dict[PreparationState, int] = { "queued": 0, "validating": 1, "exporting": 2, "finalizing": 3, "ready": 4, "failed": 4, "cancelled": 4, } class RecordingPreparationQueueFull(RuntimeError): """The bounded conversion queue cannot accept another recording.""" @dataclass(frozen=True, slots=True) class RecordingPreparationSnapshot: preparation_id: str session_id: str state: PreparationState progress: float updated_at_utc: str cancellable: bool retryable: bool error: str | None command: ReplayCommand recording: MaterializedRecording | None recorded_media: tuple[RecordedMediaManifest, ...] | None @dataclass(slots=True) class _PreparationJob: preparation_id: str source_identity: tuple[object, ...] # This is an immutable source-preparation command. Per-browser playback # policy (speed/loop) never belongs to a shared conversion job. command: ReplayCommand state: PreparationState = "queued" progress: float = 0.0 updated_at_utc: str = field(default_factory=lambda: _utc_now_iso()) error: str | None = None recording: MaterializedRecording | None = None recorded_media: tuple[RecordedMediaManifest, ...] | None = None cancel_event: threading.Event = field(default_factory=threading.Event) last_activity_monotonic: float = field(default_factory=monotonic) interrupted_by_restart: bool = False cancelled_by_operator: bool = False @dataclass(slots=True) class _WorkerGeneration: generation_id: int work_queue: queue.Queue[_PreparationJob] stop_event: threading.Event = field(default_factory=threading.Event) worker: threading.Thread | None = None @dataclass(slots=True) class _LaunchReservation: preparation_id: str release: Callable[[], None] timer: threading.Timer class SessionRecordingPreparationManager: """One bounded, process-owned conversion worker for durable recordings. Jobs are keyed by the session plus an inexpensive source identity. They intentionally outlive HTTP requests and browser tabs. Conversion stays single-worker to bound Rerun's CPU, memory and temporary-disk pressure. Worker generations make lifespan restart safe even if a third-party exporter ignores cancellation longer than ``close(timeout=...)``. New jobs can queue immediately, but a successor worker starts only after the previous generation has actually exited, so two writers never overlap. """ def __init__( self, materializer: SessionRecordingMaterializer, *, queue_capacity: int = 128, heartbeat_interval_seconds: float = 5.0, ready_preparer: Callable[ [ReplayCommand, MaterializedRecording], tuple[RecordedMediaManifest, ...], ] | None = None, ready_restorer: Callable[ [ReplayCommand, MaterializedRecording], tuple[RecordedMediaManifest, ...] | None, ] | None = None, ) -> None: if queue_capacity < 1: raise ValueError("recording preparation queue capacity must be positive") if heartbeat_interval_seconds <= 0: raise ValueError("recording preparation heartbeat interval must be positive") self.materializer = materializer self._queue_capacity = queue_capacity # Activity callbacks from the real exporter are rate-limited to this # interval. There is deliberately no independent fake heartbeat: a # hung exporter must become observable to the browser stall detector. self._heartbeat_interval_seconds = heartbeat_interval_seconds self._ready_preparer = ready_preparer self._ready_restorer = ready_restorer self._guard = threading.RLock() self._current_by_session: dict[str, _PreparationJob] = {} self._launch_reservations: dict[str, _LaunchReservation] = {} self._closed = True self._generation_counter = 0 self._active_generation: _WorkerGeneration | None = None self._pending_generation: _WorkerGeneration | None = None self.start() def start(self) -> None: """Start or restart the single worker for an application lifespan.""" with self._guard: if not self._closed: return self._current_by_session = { session_id: job for session_id, job in self._current_by_session.items() if job.state != "cancelled" } self._closed = False active = self._active_generation if active is not None and active.worker is not None and active.worker.is_alive(): self._pending_generation = self._new_generation_locked() return self._active_generation = None generation = self._pending_generation or self._new_generation_locked() self._pending_generation = None self._start_generation_locked(generation) def enqueue( self, command: ReplayCommand, *, retry_failed: bool = False, retry_interrupted: bool = False, ) -> RecordingPreparationSnapshot: source_command = _source_command(command) identity = _source_identity(source_command) with self._guard: if self._closed: raise RuntimeError("recording preparation manager is closed") current = self._current_by_session.get(command.session_id) if current is not None and current.source_identity == identity: retry_terminal = retry_failed and current.state in {"failed", "cancelled"} retry_restart = ( retry_interrupted and current.state == "cancelled" and current.interrupted_by_restart ) if not retry_terminal and not retry_restart: return self._snapshot_locked(current) job = _PreparationJob( preparation_id=uuid4().hex, source_identity=identity, command=source_command, ) self._current_by_session[command.session_id] = job try: self._intake_queue_locked().put_nowait(job) except queue.Full as exc: if self._current_by_session.get(command.session_id) is job: if current is None: self._current_by_session.pop(command.session_id, None) else: self._current_by_session[command.session_id] = current raise RecordingPreparationQueueFull("recording preparation queue is full") from exc return self._snapshot_locked(job) def resolve_cached( self, command: ReplayCommand, ) -> RecordingPreparationSnapshot | None: """Validate a published cache and register it as a ready job.""" # When launch preparation includes camera manifests, a disk-only RRD # is not a complete ready result. The worker must validate both parts # in one background transaction before publishing ``ready``. if self._ready_preparer is not None: return None source_command = _source_command(command) identity = _source_identity(source_command) recording = self.materializer.get_cached(source_command) if recording is None: with self._guard: current = self._current_by_session.get(command.session_id) if current is not None and current.state == "ready": self._current_by_session.pop(command.session_id, None) return None with self._guard: current = self._current_by_session.get(command.session_id) if ( current is not None and current.source_identity == identity and current.state in ACTIVE_PREPARATION_STATES | {"ready"} ): # A worker may have started between the disk lookup and this # transaction. Let that single job publish its own result. if current.state == "ready": current.recording = recording return self._snapshot_locked(current) ready = _PreparationJob( preparation_id=uuid4().hex, source_identity=identity, command=source_command, state="ready", progress=1.0, recording=recording, recorded_media=(), ) self._current_by_session[command.session_id] = ready return self._snapshot_locked(ready) def restore_published( self, command: ReplayCommand, ) -> RecordingPreparationSnapshot | None: """Restore a complete durable package without enqueueing preparation. The RRD and every recorded-media descriptor must already have been published by an earlier successful job. Missing/stale pieces return a cold miss; this method never invokes an exporter or media parser. """ source_command = _source_command(command) identity = _source_identity(source_command) with self._guard: current = self._current_by_session.get(command.session_id) if ( current is not None and current.source_identity == identity and current.state in ACTIVE_PREPARATION_STATES | {"ready"} ): return self._snapshot_locked(current) recording = self.materializer.restore_published(source_command) if recording is None: with self._guard: current = self._current_by_session.get(command.session_id) if current is not None and current.state == "ready": self._current_by_session.pop(command.session_id, None) return None if self._ready_preparer is None: recorded_media: tuple[RecordedMediaManifest, ...] = () else: if self._ready_restorer is None: return None restored_media = self._ready_restorer(source_command, recording) if restored_media is None: return None recorded_media = restored_media validate_recorded_media_timeline( recorded_media, recording_start_seconds=recording.timeline_start_ns / 1_000_000_000, recording_end_seconds=recording.timeline_end_ns / 1_000_000_000, ) with self._guard: current = self._current_by_session.get(command.session_id) if ( current is not None and current.source_identity == identity and current.state in ACTIVE_PREPARATION_STATES | {"ready"} ): if current.state == "ready": current.recording = recording current.recorded_media = recorded_media return self._snapshot_locked(current) ready = _PreparationJob( preparation_id=uuid4().hex, source_identity=identity, command=source_command, state="ready", progress=1.0, recording=recording, recorded_media=recorded_media, ) self._current_by_session[command.session_id] = ready return self._snapshot_locked(ready) def resolve_cached_pinned( self, command: ReplayCommand, ) -> tuple[RecordingPreparationSnapshot, Callable[[], None]] | None: """Validate and lease a published cache without starting conversion.""" if self._ready_preparer is not None: return None source_command = _source_command(command) result = self.materializer.get_cached_pinned(source_command) if result is None: with self._guard: current = self._current_by_session.get(command.session_id) if current is not None and current.state == "ready": self._current_by_session.pop(command.session_id, None) return None recording, release = result identity = _source_identity(source_command) with self._guard: current = self._current_by_session.get(command.session_id) if ( current is None or current.source_identity != identity or current.state not in ACTIVE_PREPARATION_STATES | {"ready"} ): current = _PreparationJob( preparation_id=uuid4().hex, source_identity=identity, command=source_command, state="ready", progress=1.0, recording=recording, recorded_media=(), ) self._current_by_session[command.session_id] = current snapshot = self._snapshot_locked(current) return snapshot, release def status(self, session_id: str) -> RecordingPreparationSnapshot | None: with self._guard: job = self._current_by_session.get(session_id) if ( job is not None and job.state == "ready" and ( job.recording is None or not self.materializer.is_recording_available(job.recording) ) ): self._current_by_session.pop(session_id, None) return None return None if job is None else self._snapshot_locked(job) def reserve_cached( self, command: ReplayCommand, *, lease_seconds: float = 120.0, ) -> RecordingPreparationSnapshot | None: """Pin a ready artifact across the launch-document to file-GET gap.""" if lease_seconds <= 0: raise ValueError("recording launch lease must be positive") pinned = self.resolve_cached_pinned(command) if pinned is None: return None snapshot, release = pinned self._reserve_launch_lease(snapshot, release, lease_seconds) return snapshot def pin_ready( self, session_id: str, *, preparation_id: str | None = None, ) -> tuple[RecordingPreparationSnapshot, Callable[[], None]] | None: """Cheaply lease the exact already-validated ready generation.""" launch_release: Callable[[], None] | None = None with self._guard: job = self._current_by_session.get(session_id) if ( job is None or job.state != "ready" or job.recording is None or (preparation_id is not None and job.preparation_id != preparation_id) ): return None recording = job.recording pinned = self.materializer.pin_recording(recording) if pinned is None: if self._current_by_session.get(session_id) is job: self._current_by_session.pop(session_id, None) return None snapshot = self._snapshot_locked(job) reservation = self._launch_reservations.get(session_id) if ( reservation is not None and reservation.preparation_id == snapshot.preparation_id ): self._launch_reservations.pop(session_id, None) reservation.timer.cancel() launch_release = reservation.release if launch_release is not None: launch_release() return snapshot, pinned def reserve_ready( self, session_id: str, *, preparation_id: str | None = None, lease_seconds: float = 120.0, ) -> RecordingPreparationSnapshot | None: """Hold a cheap launch lease without reopening or hashing the cache.""" if lease_seconds <= 0: raise ValueError("recording launch lease must be positive") pinned = self.pin_ready(session_id, preparation_id=preparation_id) if pinned is None: return None snapshot, release = pinned self._reserve_launch_lease(snapshot, release, lease_seconds) return snapshot def release_launch_reservation(self, session_id: str) -> bool: """Release an unused launch-to-GET lease for one session. The first matching recording GET consumes this reservation after it acquires its own response-lifetime pin. Deletion may release a launch reservation that was never consumed; an active response pin remains independently protected by the materializer. """ with self._guard: reservation = self._launch_reservations.pop(session_id, None) if reservation is None: return False reservation.timer.cancel() reservation.release() return True def cancel(self, session_id: str, *, preparation_id: str | None = None) -> bool: with self._guard: job = self._current_by_session.get(session_id) if ( job is None or job.state not in ACTIVE_PREPARATION_STATES or (preparation_id is not None and job.preparation_id != preparation_id) ): return False job.cancelled_by_operator = True job.cancel_event.set() if job.state == "queued": self._transition_locked(job, "cancelled", job.progress) return True def discard(self, session_id: str) -> bool: """Forget one non-active job before its source session is deleted.""" with self._guard: job = self._current_by_session.get(session_id) if job is not None and job.state in ACTIVE_PREPARATION_STATES: return False self._current_by_session.pop(session_id, None) reservation = self._launch_reservations.pop(session_id, None) if reservation is not None: reservation.timer.cancel() if reservation is not None: reservation.release() return True def close(self, *, timeout: float = 5.0) -> None: with self._guard: if self._closed: return self._closed = True reservations = tuple(self._launch_reservations.values()) self._launch_reservations.clear() for reservation in reservations: reservation.timer.cancel() for job in self._current_by_session.values(): if job.state in ACTIVE_PREPARATION_STATES: job.interrupted_by_restart = not job.cancelled_by_operator job.cancel_event.set() if job.state == "queued": self._transition_locked(job, "cancelled", job.progress) active = self._active_generation if active is not None: active.stop_event.set() pending = self._pending_generation if pending is not None: pending.stop_event.set() self._pending_generation = None worker = None if active is None else active.worker for reservation in reservations: reservation.release() if worker is not None: worker.join(timeout=max(0.0, timeout)) def _reserve_launch_lease( self, snapshot: RecordingPreparationSnapshot, release: Callable[[], None], lease_seconds: float, ) -> None: timer = threading.Timer( lease_seconds, self._expire_launch_reservation, args=(snapshot.session_id, snapshot.preparation_id), ) timer.name = f"missioncore-recording-launch-lease-{snapshot.preparation_id}" timer.daemon = True reservation = _LaunchReservation( preparation_id=snapshot.preparation_id, release=release, timer=timer, ) with self._guard: previous = self._launch_reservations.pop(snapshot.session_id, None) if previous is not None: previous.timer.cancel() self._launch_reservations[snapshot.session_id] = reservation if previous is not None: previous.release() timer.start() def _expire_launch_reservation(self, session_id: str, preparation_id: str) -> None: with self._guard: reservation = self._launch_reservations.get(session_id) if reservation is None or reservation.preparation_id != preparation_id: return self._launch_reservations.pop(session_id, None) reservation.release() def _run_generation(self, generation: _WorkerGeneration) -> None: work_queue = generation.work_queue try: while not generation.stop_event.is_set(): try: job = work_queue.get(timeout=0.1) except queue.Empty: continue try: if job.state == "cancelled" or job.cancel_event.is_set(): with self._guard: self._transition_locked(job, "cancelled", job.progress) continue with self._guard: self._transition_locked(job, "validating", 0.05) try: recording = self.materializer.materialize( job.command, progress_callback=partial(self._progress, job), cancel_event=job.cancel_event, ) except RecordingMaterializationCancelled: with self._guard: self._transition_locked(job, "cancelled", job.progress) except Exception: # The public status intentionally does not expose paths, # broker payloads or exporter internals. with self._guard: if job.cancel_event.is_set(): self._transition_locked(job, "cancelled", job.progress) else: self._transition_locked(job, "failed", job.progress) job.error = "Не удалось подготовить запись сессии." job.updated_at_utc = _utc_now_iso() else: with self._guard: # Cancellation may race the final materializer # return. Never publish ``ready`` after accepting a # cancel request for this exact job. if job.cancel_event.is_set(): self._transition_locked(job, "cancelled", job.progress) else: self._transition_locked(job, "finalizing", 0.95) if job.cancel_event.is_set(): with self._guard: self._transition_locked(job, "cancelled", job.progress) continue try: recorded_media = ( () if self._ready_preparer is None else self._ready_preparer(job.command, recording) ) validate_recorded_media_timeline( recorded_media, recording_start_seconds=( recording.timeline_start_ns / 1_000_000_000 ), recording_end_seconds=(recording.timeline_end_ns / 1_000_000_000), ) except Exception: with self._guard: if job.cancel_event.is_set(): self._transition_locked(job, "cancelled", job.progress) else: self._transition_locked(job, "failed", job.progress) job.error = "Не удалось подготовить запись сессии." job.updated_at_utc = _utc_now_iso() continue with self._guard: if job.cancel_event.is_set(): self._transition_locked(job, "cancelled", job.progress) else: job.recording = recording job.recorded_media = recorded_media self._transition_locked(job, "ready", 1.0) finally: work_queue.task_done() finally: self._generation_exited(generation) def _new_generation_locked(self) -> _WorkerGeneration: self._generation_counter += 1 return _WorkerGeneration( generation_id=self._generation_counter, work_queue=queue.Queue(maxsize=self._queue_capacity), ) def _start_generation_locked(self, generation: _WorkerGeneration) -> None: self._active_generation = generation worker = threading.Thread( target=self._run_generation, args=(generation,), name=f"missioncore-recording-preparation-{generation.generation_id}", daemon=True, ) generation.worker = worker worker.start() def _generation_exited(self, generation: _WorkerGeneration) -> None: with self._guard: if self._active_generation is not generation: return self._active_generation = None if self._closed: return successor = self._pending_generation or self._new_generation_locked() self._pending_generation = None self._start_generation_locked(successor) def _intake_queue_locked(self) -> queue.Queue[_PreparationJob]: pending = self._pending_generation if pending is not None: return pending.work_queue active = self._active_generation if active is None or active.stop_event.is_set(): pending = self._new_generation_locked() self._pending_generation = pending return pending.work_queue return active.work_queue def _progress(self, job: _PreparationJob, state: str, progress: float) -> None: # Only the worker publishes ``ready`` after it owns the verified # recording handle; accepting the final callback would expose a short # ready-without-launch race to status polling. if state not in {"validating", "exporting", "finalizing"}: return next_state = cast(PreparationState, state) with self._guard: if job.state in {"cancelled", "failed", "ready"}: return if _PREPARATION_PHASE[next_state] < _PREPARATION_PHASE[job.state]: return now = monotonic() if ( job.state == next_state and progress <= job.progress and now - job.last_activity_monotonic < self._heartbeat_interval_seconds ): return self._transition_locked(job, next_state, progress) def _transition_locked( self, job: _PreparationJob, state: PreparationState, progress: float, ) -> None: if job.state not in ACTIVE_PREPARATION_STATES and state != job.state: return if _PREPARATION_PHASE[state] < _PREPARATION_PHASE[job.state]: return job.state = state job.progress = max(job.progress, min(1.0, max(0.0, progress))) job.updated_at_utc = _utc_now_iso() job.last_activity_monotonic = monotonic() def _snapshot_locked(self, job: _PreparationJob) -> RecordingPreparationSnapshot: return RecordingPreparationSnapshot( preparation_id=job.preparation_id, session_id=job.command.session_id, state=job.state, progress=job.progress, updated_at_utc=job.updated_at_utc, cancellable=( job.state == "queued" or ( job.state in {"validating", "exporting", "finalizing"} and self.materializer.supports_cooperative_cancellation ) ), retryable=job.state in {"failed", "cancelled"}, error=job.error, command=job.command, recording=job.recording, recorded_media=job.recorded_media, ) def _source_identity(command: ReplayCommand) -> tuple[object, ...]: """Build a non-blocking identity; full validation belongs to the worker.""" identities: list[object] = [ command.session_id, command.plugin_id, command.primary_artifact_id, ] for artifact in command.artifacts: path = artifact.path identities.extend( ( artifact.artifact_id, artifact.media_type, artifact.file_byte_length, artifact.replay_byte_length, artifact.expected_sha256, ) ) try: value = os.lstat(path) except OSError: identities.extend((str(Path(path)), None, None, None, None)) else: identities.extend( ( str(Path(path)), value.st_size, value.st_mtime_ns, value.st_ctime_ns, value.st_ino, ) ) return tuple(identities) def _source_command(command: ReplayCommand) -> ReplayCommand: """Remove per-viewer launch policy from a shared preparation job.""" return replace(command, speed=1.0, loop=False) def _utc_now_iso() -> str: return datetime.now(UTC).isoformat().replace("+00:00", "Z")