781 lines
31 KiB
Python
781 lines
31 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import queue
|
|
import threading
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass, field, replace
|
|
from datetime import UTC, datetime
|
|
from functools import partial
|
|
from pathlib import Path
|
|
from time import monotonic
|
|
from typing import Literal, cast
|
|
from uuid import uuid4
|
|
|
|
from .media import RecordedMediaManifest, validate_recorded_media_timeline
|
|
from .models import ReplayCommand
|
|
from .recording import (
|
|
MaterializedRecording,
|
|
RecordingMaterializationCancelled,
|
|
SessionRecordingMaterializer,
|
|
)
|
|
|
|
PreparationState = Literal[
|
|
"queued",
|
|
"validating",
|
|
"exporting",
|
|
"finalizing",
|
|
"ready",
|
|
"failed",
|
|
"cancelled",
|
|
]
|
|
ACTIVE_PREPARATION_STATES = frozenset({"queued", "validating", "exporting", "finalizing"})
|
|
_PREPARATION_PHASE: dict[PreparationState, int] = {
|
|
"queued": 0,
|
|
"validating": 1,
|
|
"exporting": 2,
|
|
"finalizing": 3,
|
|
"ready": 4,
|
|
"failed": 4,
|
|
"cancelled": 4,
|
|
}
|
|
|
|
|
|
class RecordingPreparationQueueFull(RuntimeError):
|
|
"""The bounded conversion queue cannot accept another recording."""
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class RecordingPreparationSnapshot:
|
|
preparation_id: str
|
|
session_id: str
|
|
state: PreparationState
|
|
progress: float
|
|
updated_at_utc: str
|
|
cancellable: bool
|
|
retryable: bool
|
|
error: str | None
|
|
command: ReplayCommand
|
|
recording: MaterializedRecording | None
|
|
recorded_media: tuple[RecordedMediaManifest, ...] | None
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _PreparationJob:
|
|
preparation_id: str
|
|
source_identity: tuple[object, ...]
|
|
# This is an immutable source-preparation command. Per-browser playback
|
|
# policy (speed/loop) never belongs to a shared conversion job.
|
|
command: ReplayCommand
|
|
state: PreparationState = "queued"
|
|
progress: float = 0.0
|
|
updated_at_utc: str = field(default_factory=lambda: _utc_now_iso())
|
|
error: str | None = None
|
|
recording: MaterializedRecording | None = None
|
|
recorded_media: tuple[RecordedMediaManifest, ...] | None = None
|
|
cancel_event: threading.Event = field(default_factory=threading.Event)
|
|
last_activity_monotonic: float = field(default_factory=monotonic)
|
|
interrupted_by_restart: bool = False
|
|
cancelled_by_operator: bool = False
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _WorkerGeneration:
|
|
generation_id: int
|
|
work_queue: queue.Queue[_PreparationJob]
|
|
stop_event: threading.Event = field(default_factory=threading.Event)
|
|
worker: threading.Thread | None = None
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _LaunchReservation:
|
|
preparation_id: str
|
|
release: Callable[[], None]
|
|
timer: threading.Timer
|
|
|
|
|
|
class SessionRecordingPreparationManager:
|
|
"""One bounded, process-owned conversion worker for durable recordings.
|
|
|
|
Jobs are keyed by the session plus an inexpensive source identity. They
|
|
intentionally outlive HTTP requests and browser tabs. Conversion stays
|
|
single-worker to bound Rerun's CPU, memory and temporary-disk pressure.
|
|
|
|
Worker generations make lifespan restart safe even if a third-party
|
|
exporter ignores cancellation longer than ``close(timeout=...)``. New
|
|
jobs can queue immediately, but a successor worker starts only after the
|
|
previous generation has actually exited, so two writers never overlap.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
materializer: SessionRecordingMaterializer,
|
|
*,
|
|
queue_capacity: int = 128,
|
|
heartbeat_interval_seconds: float = 5.0,
|
|
ready_preparer: Callable[
|
|
[ReplayCommand, MaterializedRecording],
|
|
tuple[RecordedMediaManifest, ...],
|
|
]
|
|
| None = None,
|
|
ready_restorer: Callable[
|
|
[ReplayCommand, MaterializedRecording],
|
|
tuple[RecordedMediaManifest, ...] | None,
|
|
]
|
|
| None = None,
|
|
) -> None:
|
|
if queue_capacity < 1:
|
|
raise ValueError("recording preparation queue capacity must be positive")
|
|
if heartbeat_interval_seconds <= 0:
|
|
raise ValueError("recording preparation heartbeat interval must be positive")
|
|
self.materializer = materializer
|
|
self._queue_capacity = queue_capacity
|
|
# Activity callbacks from the real exporter are rate-limited to this
|
|
# interval. There is deliberately no independent fake heartbeat: a
|
|
# hung exporter must become observable to the browser stall detector.
|
|
self._heartbeat_interval_seconds = heartbeat_interval_seconds
|
|
self._ready_preparer = ready_preparer
|
|
self._ready_restorer = ready_restorer
|
|
self._guard = threading.RLock()
|
|
self._current_by_session: dict[str, _PreparationJob] = {}
|
|
self._launch_reservations: dict[str, _LaunchReservation] = {}
|
|
self._closed = True
|
|
self._generation_counter = 0
|
|
self._active_generation: _WorkerGeneration | None = None
|
|
self._pending_generation: _WorkerGeneration | None = None
|
|
self.start()
|
|
|
|
def start(self) -> None:
|
|
"""Start or restart the single worker for an application lifespan."""
|
|
|
|
with self._guard:
|
|
if not self._closed:
|
|
return
|
|
self._current_by_session = {
|
|
session_id: job
|
|
for session_id, job in self._current_by_session.items()
|
|
if job.state != "cancelled"
|
|
}
|
|
self._closed = False
|
|
active = self._active_generation
|
|
if active is not None and active.worker is not None and active.worker.is_alive():
|
|
self._pending_generation = self._new_generation_locked()
|
|
return
|
|
self._active_generation = None
|
|
generation = self._pending_generation or self._new_generation_locked()
|
|
self._pending_generation = None
|
|
self._start_generation_locked(generation)
|
|
|
|
def enqueue(
|
|
self,
|
|
command: ReplayCommand,
|
|
*,
|
|
retry_failed: bool = False,
|
|
retry_interrupted: bool = False,
|
|
) -> RecordingPreparationSnapshot:
|
|
source_command = _source_command(command)
|
|
identity = _source_identity(source_command)
|
|
with self._guard:
|
|
if self._closed:
|
|
raise RuntimeError("recording preparation manager is closed")
|
|
current = self._current_by_session.get(command.session_id)
|
|
if current is not None and current.source_identity == identity:
|
|
retry_terminal = retry_failed and current.state in {"failed", "cancelled"}
|
|
retry_restart = (
|
|
retry_interrupted
|
|
and current.state == "cancelled"
|
|
and current.interrupted_by_restart
|
|
)
|
|
if not retry_terminal and not retry_restart:
|
|
return self._snapshot_locked(current)
|
|
|
|
job = _PreparationJob(
|
|
preparation_id=uuid4().hex,
|
|
source_identity=identity,
|
|
command=source_command,
|
|
)
|
|
self._current_by_session[command.session_id] = job
|
|
try:
|
|
self._intake_queue_locked().put_nowait(job)
|
|
except queue.Full as exc:
|
|
if self._current_by_session.get(command.session_id) is job:
|
|
if current is None:
|
|
self._current_by_session.pop(command.session_id, None)
|
|
else:
|
|
self._current_by_session[command.session_id] = current
|
|
raise RecordingPreparationQueueFull("recording preparation queue is full") from exc
|
|
return self._snapshot_locked(job)
|
|
|
|
def resolve_cached(
|
|
self,
|
|
command: ReplayCommand,
|
|
) -> RecordingPreparationSnapshot | None:
|
|
"""Validate a published cache and register it as a ready job."""
|
|
|
|
# When launch preparation includes camera manifests, a disk-only RRD
|
|
# is not a complete ready result. The worker must validate both parts
|
|
# in one background transaction before publishing ``ready``.
|
|
if self._ready_preparer is not None:
|
|
return None
|
|
|
|
source_command = _source_command(command)
|
|
identity = _source_identity(source_command)
|
|
recording = self.materializer.get_cached(source_command)
|
|
if recording is None:
|
|
with self._guard:
|
|
current = self._current_by_session.get(command.session_id)
|
|
if current is not None and current.state == "ready":
|
|
self._current_by_session.pop(command.session_id, None)
|
|
return None
|
|
with self._guard:
|
|
current = self._current_by_session.get(command.session_id)
|
|
if (
|
|
current is not None
|
|
and current.source_identity == identity
|
|
and current.state in ACTIVE_PREPARATION_STATES | {"ready"}
|
|
):
|
|
# A worker may have started between the disk lookup and this
|
|
# transaction. Let that single job publish its own result.
|
|
if current.state == "ready":
|
|
current.recording = recording
|
|
return self._snapshot_locked(current)
|
|
ready = _PreparationJob(
|
|
preparation_id=uuid4().hex,
|
|
source_identity=identity,
|
|
command=source_command,
|
|
state="ready",
|
|
progress=1.0,
|
|
recording=recording,
|
|
recorded_media=(),
|
|
)
|
|
self._current_by_session[command.session_id] = ready
|
|
return self._snapshot_locked(ready)
|
|
|
|
def restore_published(
|
|
self,
|
|
command: ReplayCommand,
|
|
) -> RecordingPreparationSnapshot | None:
|
|
"""Restore a complete durable package without enqueueing preparation.
|
|
|
|
The RRD and every recorded-media descriptor must already have been
|
|
published by an earlier successful job. Missing/stale pieces return a
|
|
cold miss; this method never invokes an exporter or media parser.
|
|
"""
|
|
|
|
source_command = _source_command(command)
|
|
identity = _source_identity(source_command)
|
|
with self._guard:
|
|
current = self._current_by_session.get(command.session_id)
|
|
if (
|
|
current is not None
|
|
and current.source_identity == identity
|
|
and current.state in ACTIVE_PREPARATION_STATES | {"ready"}
|
|
):
|
|
return self._snapshot_locked(current)
|
|
|
|
recording = self.materializer.restore_published(source_command)
|
|
if recording is None:
|
|
with self._guard:
|
|
current = self._current_by_session.get(command.session_id)
|
|
if current is not None and current.state == "ready":
|
|
self._current_by_session.pop(command.session_id, None)
|
|
return None
|
|
|
|
if self._ready_preparer is None:
|
|
recorded_media: tuple[RecordedMediaManifest, ...] = ()
|
|
else:
|
|
if self._ready_restorer is None:
|
|
return None
|
|
restored_media = self._ready_restorer(source_command, recording)
|
|
if restored_media is None:
|
|
return None
|
|
recorded_media = restored_media
|
|
validate_recorded_media_timeline(
|
|
recorded_media,
|
|
recording_start_seconds=recording.timeline_start_ns / 1_000_000_000,
|
|
recording_end_seconds=recording.timeline_end_ns / 1_000_000_000,
|
|
)
|
|
|
|
with self._guard:
|
|
current = self._current_by_session.get(command.session_id)
|
|
if (
|
|
current is not None
|
|
and current.source_identity == identity
|
|
and current.state in ACTIVE_PREPARATION_STATES | {"ready"}
|
|
):
|
|
if current.state == "ready":
|
|
current.recording = recording
|
|
current.recorded_media = recorded_media
|
|
return self._snapshot_locked(current)
|
|
ready = _PreparationJob(
|
|
preparation_id=uuid4().hex,
|
|
source_identity=identity,
|
|
command=source_command,
|
|
state="ready",
|
|
progress=1.0,
|
|
recording=recording,
|
|
recorded_media=recorded_media,
|
|
)
|
|
self._current_by_session[command.session_id] = ready
|
|
return self._snapshot_locked(ready)
|
|
|
|
def resolve_cached_pinned(
|
|
self,
|
|
command: ReplayCommand,
|
|
) -> tuple[RecordingPreparationSnapshot, Callable[[], None]] | None:
|
|
"""Validate and lease a published cache without starting conversion."""
|
|
|
|
if self._ready_preparer is not None:
|
|
return None
|
|
|
|
source_command = _source_command(command)
|
|
result = self.materializer.get_cached_pinned(source_command)
|
|
if result is None:
|
|
with self._guard:
|
|
current = self._current_by_session.get(command.session_id)
|
|
if current is not None and current.state == "ready":
|
|
self._current_by_session.pop(command.session_id, None)
|
|
return None
|
|
recording, release = result
|
|
identity = _source_identity(source_command)
|
|
with self._guard:
|
|
current = self._current_by_session.get(command.session_id)
|
|
if (
|
|
current is None
|
|
or current.source_identity != identity
|
|
or current.state not in ACTIVE_PREPARATION_STATES | {"ready"}
|
|
):
|
|
current = _PreparationJob(
|
|
preparation_id=uuid4().hex,
|
|
source_identity=identity,
|
|
command=source_command,
|
|
state="ready",
|
|
progress=1.0,
|
|
recording=recording,
|
|
recorded_media=(),
|
|
)
|
|
self._current_by_session[command.session_id] = current
|
|
snapshot = self._snapshot_locked(current)
|
|
return snapshot, release
|
|
|
|
def status(self, session_id: str) -> RecordingPreparationSnapshot | None:
|
|
with self._guard:
|
|
job = self._current_by_session.get(session_id)
|
|
if (
|
|
job is not None
|
|
and job.state == "ready"
|
|
and (
|
|
job.recording is None
|
|
or not self.materializer.is_recording_available(job.recording)
|
|
)
|
|
):
|
|
self._current_by_session.pop(session_id, None)
|
|
return None
|
|
return None if job is None else self._snapshot_locked(job)
|
|
|
|
def reserve_cached(
|
|
self,
|
|
command: ReplayCommand,
|
|
*,
|
|
lease_seconds: float = 120.0,
|
|
) -> RecordingPreparationSnapshot | None:
|
|
"""Pin a ready artifact across the launch-document to file-GET gap."""
|
|
|
|
if lease_seconds <= 0:
|
|
raise ValueError("recording launch lease must be positive")
|
|
pinned = self.resolve_cached_pinned(command)
|
|
if pinned is None:
|
|
return None
|
|
snapshot, release = pinned
|
|
self._reserve_launch_lease(snapshot, release, lease_seconds)
|
|
return snapshot
|
|
|
|
def pin_ready(
|
|
self,
|
|
session_id: str,
|
|
*,
|
|
preparation_id: str | None = None,
|
|
) -> tuple[RecordingPreparationSnapshot, Callable[[], None]] | None:
|
|
"""Cheaply lease the exact already-validated ready generation."""
|
|
|
|
launch_release: Callable[[], None] | None = None
|
|
with self._guard:
|
|
job = self._current_by_session.get(session_id)
|
|
if (
|
|
job is None
|
|
or job.state != "ready"
|
|
or job.recording is None
|
|
or (preparation_id is not None and job.preparation_id != preparation_id)
|
|
):
|
|
return None
|
|
recording = job.recording
|
|
pinned = self.materializer.pin_recording(recording)
|
|
if pinned is None:
|
|
if self._current_by_session.get(session_id) is job:
|
|
self._current_by_session.pop(session_id, None)
|
|
return None
|
|
snapshot = self._snapshot_locked(job)
|
|
reservation = self._launch_reservations.get(session_id)
|
|
if (
|
|
reservation is not None
|
|
and reservation.preparation_id == snapshot.preparation_id
|
|
):
|
|
self._launch_reservations.pop(session_id, None)
|
|
reservation.timer.cancel()
|
|
launch_release = reservation.release
|
|
if launch_release is not None:
|
|
launch_release()
|
|
return snapshot, pinned
|
|
|
|
def reserve_ready(
|
|
self,
|
|
session_id: str,
|
|
*,
|
|
preparation_id: str | None = None,
|
|
lease_seconds: float = 120.0,
|
|
) -> RecordingPreparationSnapshot | None:
|
|
"""Hold a cheap launch lease without reopening or hashing the cache."""
|
|
|
|
if lease_seconds <= 0:
|
|
raise ValueError("recording launch lease must be positive")
|
|
pinned = self.pin_ready(session_id, preparation_id=preparation_id)
|
|
if pinned is None:
|
|
return None
|
|
snapshot, release = pinned
|
|
self._reserve_launch_lease(snapshot, release, lease_seconds)
|
|
return snapshot
|
|
|
|
def release_launch_reservation(self, session_id: str) -> bool:
|
|
"""Release an unused launch-to-GET lease for one session.
|
|
|
|
The first matching recording GET consumes this reservation after it
|
|
acquires its own response-lifetime pin. Deletion may release a launch
|
|
reservation that was never consumed; an active response pin remains
|
|
independently protected by the materializer.
|
|
"""
|
|
|
|
with self._guard:
|
|
reservation = self._launch_reservations.pop(session_id, None)
|
|
if reservation is None:
|
|
return False
|
|
reservation.timer.cancel()
|
|
reservation.release()
|
|
return True
|
|
|
|
def cancel(self, session_id: str, *, preparation_id: str | None = None) -> bool:
|
|
with self._guard:
|
|
job = self._current_by_session.get(session_id)
|
|
if (
|
|
job is None
|
|
or job.state not in ACTIVE_PREPARATION_STATES
|
|
or (preparation_id is not None and job.preparation_id != preparation_id)
|
|
):
|
|
return False
|
|
job.cancelled_by_operator = True
|
|
job.cancel_event.set()
|
|
if job.state == "queued":
|
|
self._transition_locked(job, "cancelled", job.progress)
|
|
return True
|
|
|
|
def discard(self, session_id: str) -> bool:
|
|
"""Forget one non-active job before its source session is deleted."""
|
|
|
|
with self._guard:
|
|
job = self._current_by_session.get(session_id)
|
|
if job is not None and job.state in ACTIVE_PREPARATION_STATES:
|
|
return False
|
|
self._current_by_session.pop(session_id, None)
|
|
reservation = self._launch_reservations.pop(session_id, None)
|
|
if reservation is not None:
|
|
reservation.timer.cancel()
|
|
if reservation is not None:
|
|
reservation.release()
|
|
return True
|
|
|
|
def close(self, *, timeout: float = 5.0) -> None:
|
|
with self._guard:
|
|
if self._closed:
|
|
return
|
|
self._closed = True
|
|
reservations = tuple(self._launch_reservations.values())
|
|
self._launch_reservations.clear()
|
|
for reservation in reservations:
|
|
reservation.timer.cancel()
|
|
for job in self._current_by_session.values():
|
|
if job.state in ACTIVE_PREPARATION_STATES:
|
|
job.interrupted_by_restart = not job.cancelled_by_operator
|
|
job.cancel_event.set()
|
|
if job.state == "queued":
|
|
self._transition_locked(job, "cancelled", job.progress)
|
|
active = self._active_generation
|
|
if active is not None:
|
|
active.stop_event.set()
|
|
pending = self._pending_generation
|
|
if pending is not None:
|
|
pending.stop_event.set()
|
|
self._pending_generation = None
|
|
worker = None if active is None else active.worker
|
|
for reservation in reservations:
|
|
reservation.release()
|
|
if worker is not None:
|
|
worker.join(timeout=max(0.0, timeout))
|
|
|
|
def _reserve_launch_lease(
|
|
self,
|
|
snapshot: RecordingPreparationSnapshot,
|
|
release: Callable[[], None],
|
|
lease_seconds: float,
|
|
) -> None:
|
|
timer = threading.Timer(
|
|
lease_seconds,
|
|
self._expire_launch_reservation,
|
|
args=(snapshot.session_id, snapshot.preparation_id),
|
|
)
|
|
timer.name = f"missioncore-recording-launch-lease-{snapshot.preparation_id}"
|
|
timer.daemon = True
|
|
reservation = _LaunchReservation(
|
|
preparation_id=snapshot.preparation_id,
|
|
release=release,
|
|
timer=timer,
|
|
)
|
|
with self._guard:
|
|
previous = self._launch_reservations.pop(snapshot.session_id, None)
|
|
if previous is not None:
|
|
previous.timer.cancel()
|
|
self._launch_reservations[snapshot.session_id] = reservation
|
|
if previous is not None:
|
|
previous.release()
|
|
timer.start()
|
|
|
|
def _expire_launch_reservation(self, session_id: str, preparation_id: str) -> None:
|
|
with self._guard:
|
|
reservation = self._launch_reservations.get(session_id)
|
|
if reservation is None or reservation.preparation_id != preparation_id:
|
|
return
|
|
self._launch_reservations.pop(session_id, None)
|
|
reservation.release()
|
|
|
|
def _run_generation(self, generation: _WorkerGeneration) -> None:
|
|
work_queue = generation.work_queue
|
|
try:
|
|
while not generation.stop_event.is_set():
|
|
try:
|
|
job = work_queue.get(timeout=0.1)
|
|
except queue.Empty:
|
|
continue
|
|
try:
|
|
if job.state == "cancelled" or job.cancel_event.is_set():
|
|
with self._guard:
|
|
self._transition_locked(job, "cancelled", job.progress)
|
|
continue
|
|
with self._guard:
|
|
self._transition_locked(job, "validating", 0.05)
|
|
try:
|
|
recording = self.materializer.materialize(
|
|
job.command,
|
|
progress_callback=partial(self._progress, job),
|
|
cancel_event=job.cancel_event,
|
|
)
|
|
except RecordingMaterializationCancelled:
|
|
with self._guard:
|
|
self._transition_locked(job, "cancelled", job.progress)
|
|
except Exception:
|
|
# The public status intentionally does not expose paths,
|
|
# broker payloads or exporter internals.
|
|
with self._guard:
|
|
if job.cancel_event.is_set():
|
|
self._transition_locked(job, "cancelled", job.progress)
|
|
else:
|
|
self._transition_locked(job, "failed", job.progress)
|
|
job.error = "Не удалось подготовить запись сессии."
|
|
job.updated_at_utc = _utc_now_iso()
|
|
else:
|
|
with self._guard:
|
|
# Cancellation may race the final materializer
|
|
# return. Never publish ``ready`` after accepting a
|
|
# cancel request for this exact job.
|
|
if job.cancel_event.is_set():
|
|
self._transition_locked(job, "cancelled", job.progress)
|
|
else:
|
|
self._transition_locked(job, "finalizing", 0.95)
|
|
if job.cancel_event.is_set():
|
|
with self._guard:
|
|
self._transition_locked(job, "cancelled", job.progress)
|
|
continue
|
|
try:
|
|
recorded_media = (
|
|
()
|
|
if self._ready_preparer is None
|
|
else self._ready_preparer(job.command, recording)
|
|
)
|
|
validate_recorded_media_timeline(
|
|
recorded_media,
|
|
recording_start_seconds=(
|
|
recording.timeline_start_ns / 1_000_000_000
|
|
),
|
|
recording_end_seconds=(recording.timeline_end_ns / 1_000_000_000),
|
|
)
|
|
except Exception:
|
|
with self._guard:
|
|
if job.cancel_event.is_set():
|
|
self._transition_locked(job, "cancelled", job.progress)
|
|
else:
|
|
self._transition_locked(job, "failed", job.progress)
|
|
job.error = "Не удалось подготовить запись сессии."
|
|
job.updated_at_utc = _utc_now_iso()
|
|
continue
|
|
with self._guard:
|
|
if job.cancel_event.is_set():
|
|
self._transition_locked(job, "cancelled", job.progress)
|
|
else:
|
|
job.recording = recording
|
|
job.recorded_media = recorded_media
|
|
self._transition_locked(job, "ready", 1.0)
|
|
finally:
|
|
work_queue.task_done()
|
|
finally:
|
|
self._generation_exited(generation)
|
|
|
|
def _new_generation_locked(self) -> _WorkerGeneration:
|
|
self._generation_counter += 1
|
|
return _WorkerGeneration(
|
|
generation_id=self._generation_counter,
|
|
work_queue=queue.Queue(maxsize=self._queue_capacity),
|
|
)
|
|
|
|
def _start_generation_locked(self, generation: _WorkerGeneration) -> None:
|
|
self._active_generation = generation
|
|
worker = threading.Thread(
|
|
target=self._run_generation,
|
|
args=(generation,),
|
|
name=f"missioncore-recording-preparation-{generation.generation_id}",
|
|
daemon=True,
|
|
)
|
|
generation.worker = worker
|
|
worker.start()
|
|
|
|
def _generation_exited(self, generation: _WorkerGeneration) -> None:
|
|
with self._guard:
|
|
if self._active_generation is not generation:
|
|
return
|
|
self._active_generation = None
|
|
if self._closed:
|
|
return
|
|
successor = self._pending_generation or self._new_generation_locked()
|
|
self._pending_generation = None
|
|
self._start_generation_locked(successor)
|
|
|
|
def _intake_queue_locked(self) -> queue.Queue[_PreparationJob]:
|
|
pending = self._pending_generation
|
|
if pending is not None:
|
|
return pending.work_queue
|
|
active = self._active_generation
|
|
if active is None or active.stop_event.is_set():
|
|
pending = self._new_generation_locked()
|
|
self._pending_generation = pending
|
|
return pending.work_queue
|
|
return active.work_queue
|
|
|
|
def _progress(self, job: _PreparationJob, state: str, progress: float) -> None:
|
|
# Only the worker publishes ``ready`` after it owns the verified
|
|
# recording handle; accepting the final callback would expose a short
|
|
# ready-without-launch race to status polling.
|
|
if state not in {"validating", "exporting", "finalizing"}:
|
|
return
|
|
next_state = cast(PreparationState, state)
|
|
with self._guard:
|
|
if job.state in {"cancelled", "failed", "ready"}:
|
|
return
|
|
if _PREPARATION_PHASE[next_state] < _PREPARATION_PHASE[job.state]:
|
|
return
|
|
now = monotonic()
|
|
if (
|
|
job.state == next_state
|
|
and progress <= job.progress
|
|
and now - job.last_activity_monotonic < self._heartbeat_interval_seconds
|
|
):
|
|
return
|
|
self._transition_locked(job, next_state, progress)
|
|
|
|
def _transition_locked(
|
|
self,
|
|
job: _PreparationJob,
|
|
state: PreparationState,
|
|
progress: float,
|
|
) -> None:
|
|
if job.state not in ACTIVE_PREPARATION_STATES and state != job.state:
|
|
return
|
|
if _PREPARATION_PHASE[state] < _PREPARATION_PHASE[job.state]:
|
|
return
|
|
job.state = state
|
|
job.progress = max(job.progress, min(1.0, max(0.0, progress)))
|
|
job.updated_at_utc = _utc_now_iso()
|
|
job.last_activity_monotonic = monotonic()
|
|
|
|
def _snapshot_locked(self, job: _PreparationJob) -> RecordingPreparationSnapshot:
|
|
return RecordingPreparationSnapshot(
|
|
preparation_id=job.preparation_id,
|
|
session_id=job.command.session_id,
|
|
state=job.state,
|
|
progress=job.progress,
|
|
updated_at_utc=job.updated_at_utc,
|
|
cancellable=(
|
|
job.state == "queued"
|
|
or (
|
|
job.state in {"validating", "exporting", "finalizing"}
|
|
and self.materializer.supports_cooperative_cancellation
|
|
)
|
|
),
|
|
retryable=job.state in {"failed", "cancelled"},
|
|
error=job.error,
|
|
command=job.command,
|
|
recording=job.recording,
|
|
recorded_media=job.recorded_media,
|
|
)
|
|
|
|
|
|
def _source_identity(command: ReplayCommand) -> tuple[object, ...]:
|
|
"""Build a non-blocking identity; full validation belongs to the worker."""
|
|
|
|
identities: list[object] = [
|
|
command.session_id,
|
|
command.plugin_id,
|
|
command.primary_artifact_id,
|
|
]
|
|
for artifact in command.artifacts:
|
|
path = artifact.path
|
|
identities.extend(
|
|
(
|
|
artifact.artifact_id,
|
|
artifact.media_type,
|
|
artifact.file_byte_length,
|
|
artifact.replay_byte_length,
|
|
artifact.expected_sha256,
|
|
)
|
|
)
|
|
try:
|
|
value = os.lstat(path)
|
|
except OSError:
|
|
identities.extend((str(Path(path)), None, None, None, None))
|
|
else:
|
|
identities.extend(
|
|
(
|
|
str(Path(path)),
|
|
value.st_size,
|
|
value.st_mtime_ns,
|
|
value.st_ctime_ns,
|
|
value.st_ino,
|
|
)
|
|
)
|
|
return tuple(identities)
|
|
|
|
|
|
def _source_command(command: ReplayCommand) -> ReplayCommand:
|
|
"""Remove per-viewer launch policy from a shared preparation job."""
|
|
|
|
return replace(command, speed=1.0, loop=False)
|
|
|
|
|
|
def _utc_now_iso() -> str:
|
|
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|