feat(sessions): add durable observation archive and replay API
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,617 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field, replace
|
||||
from datetime import UTC, datetime
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from time import monotonic
|
||||
from typing import Literal, cast
|
||||
from uuid import uuid4
|
||||
|
||||
from .media import RecordedMediaManifest, validate_recorded_media_timeline
|
||||
from .models import ReplayCommand
|
||||
from .recording import (
|
||||
MaterializedRecording,
|
||||
RecordingMaterializationCancelled,
|
||||
SessionRecordingMaterializer,
|
||||
)
|
||||
|
||||
PreparationState = Literal[
|
||||
"queued",
|
||||
"validating",
|
||||
"exporting",
|
||||
"finalizing",
|
||||
"ready",
|
||||
"failed",
|
||||
"cancelled",
|
||||
]
|
||||
ACTIVE_PREPARATION_STATES = frozenset({"queued", "validating", "exporting", "finalizing"})
|
||||
_PREPARATION_PHASE: dict[PreparationState, int] = {
|
||||
"queued": 0,
|
||||
"validating": 1,
|
||||
"exporting": 2,
|
||||
"finalizing": 3,
|
||||
"ready": 4,
|
||||
"failed": 4,
|
||||
"cancelled": 4,
|
||||
}
|
||||
|
||||
|
||||
class RecordingPreparationQueueFull(RuntimeError):
|
||||
"""The bounded conversion queue cannot accept another recording."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RecordingPreparationSnapshot:
|
||||
preparation_id: str
|
||||
session_id: str
|
||||
state: PreparationState
|
||||
progress: float
|
||||
updated_at_utc: str
|
||||
cancellable: bool
|
||||
retryable: bool
|
||||
error: str | None
|
||||
command: ReplayCommand
|
||||
recording: MaterializedRecording | None
|
||||
recorded_media: tuple[RecordedMediaManifest, ...] | None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _PreparationJob:
|
||||
preparation_id: str
|
||||
source_identity: tuple[object, ...]
|
||||
# This is an immutable source-preparation command. Per-browser playback
|
||||
# policy (speed/loop) never belongs to a shared conversion job.
|
||||
command: ReplayCommand
|
||||
state: PreparationState = "queued"
|
||||
progress: float = 0.0
|
||||
updated_at_utc: str = field(default_factory=lambda: _utc_now_iso())
|
||||
error: str | None = None
|
||||
recording: MaterializedRecording | None = None
|
||||
recorded_media: tuple[RecordedMediaManifest, ...] | None = None
|
||||
cancel_event: threading.Event = field(default_factory=threading.Event)
|
||||
last_activity_monotonic: float = field(default_factory=monotonic)
|
||||
interrupted_by_restart: bool = False
|
||||
cancelled_by_operator: bool = False
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _WorkerGeneration:
|
||||
generation_id: int
|
||||
work_queue: queue.Queue[_PreparationJob]
|
||||
stop_event: threading.Event = field(default_factory=threading.Event)
|
||||
worker: threading.Thread | None = None
|
||||
|
||||
|
||||
class SessionRecordingPreparationManager:
|
||||
"""One bounded, process-owned conversion worker for durable recordings.
|
||||
|
||||
Jobs are keyed by the session plus an inexpensive source identity. They
|
||||
intentionally outlive HTTP requests and browser tabs. Conversion stays
|
||||
single-worker to bound Rerun's CPU, memory and temporary-disk pressure.
|
||||
|
||||
Worker generations make lifespan restart safe even if a third-party
|
||||
exporter ignores cancellation longer than ``close(timeout=...)``. New
|
||||
jobs can queue immediately, but a successor worker starts only after the
|
||||
previous generation has actually exited, so two writers never overlap.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
materializer: SessionRecordingMaterializer,
|
||||
*,
|
||||
queue_capacity: int = 128,
|
||||
heartbeat_interval_seconds: float = 5.0,
|
||||
ready_preparer: Callable[
|
||||
[ReplayCommand, MaterializedRecording],
|
||||
tuple[RecordedMediaManifest, ...],
|
||||
]
|
||||
| None = None,
|
||||
) -> None:
|
||||
if queue_capacity < 1:
|
||||
raise ValueError("recording preparation queue capacity must be positive")
|
||||
if heartbeat_interval_seconds <= 0:
|
||||
raise ValueError("recording preparation heartbeat interval must be positive")
|
||||
self.materializer = materializer
|
||||
self._queue_capacity = queue_capacity
|
||||
# Activity callbacks from the real exporter are rate-limited to this
|
||||
# interval. There is deliberately no independent fake heartbeat: a
|
||||
# hung exporter must become observable to the browser stall detector.
|
||||
self._heartbeat_interval_seconds = heartbeat_interval_seconds
|
||||
self._ready_preparer = ready_preparer
|
||||
self._guard = threading.RLock()
|
||||
self._current_by_session: dict[str, _PreparationJob] = {}
|
||||
self._closed = True
|
||||
self._generation_counter = 0
|
||||
self._active_generation: _WorkerGeneration | None = None
|
||||
self._pending_generation: _WorkerGeneration | None = None
|
||||
self.start()
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start or restart the single worker for an application lifespan."""
|
||||
|
||||
with self._guard:
|
||||
if not self._closed:
|
||||
return
|
||||
self._current_by_session = {
|
||||
session_id: job
|
||||
for session_id, job in self._current_by_session.items()
|
||||
if job.state != "cancelled"
|
||||
}
|
||||
self._closed = False
|
||||
active = self._active_generation
|
||||
if active is not None and active.worker is not None and active.worker.is_alive():
|
||||
self._pending_generation = self._new_generation_locked()
|
||||
return
|
||||
self._active_generation = None
|
||||
generation = self._pending_generation or self._new_generation_locked()
|
||||
self._pending_generation = None
|
||||
self._start_generation_locked(generation)
|
||||
|
||||
def enqueue(
|
||||
self,
|
||||
command: ReplayCommand,
|
||||
*,
|
||||
retry_failed: bool = False,
|
||||
retry_interrupted: bool = False,
|
||||
) -> RecordingPreparationSnapshot:
|
||||
source_command = _source_command(command)
|
||||
identity = _source_identity(source_command)
|
||||
with self._guard:
|
||||
if self._closed:
|
||||
raise RuntimeError("recording preparation manager is closed")
|
||||
current = self._current_by_session.get(command.session_id)
|
||||
if current is not None and current.source_identity == identity:
|
||||
retry_terminal = retry_failed and current.state in {"failed", "cancelled"}
|
||||
retry_restart = (
|
||||
retry_interrupted
|
||||
and current.state == "cancelled"
|
||||
and current.interrupted_by_restart
|
||||
)
|
||||
if not retry_terminal and not retry_restart:
|
||||
return self._snapshot_locked(current)
|
||||
|
||||
job = _PreparationJob(
|
||||
preparation_id=uuid4().hex,
|
||||
source_identity=identity,
|
||||
command=source_command,
|
||||
)
|
||||
self._current_by_session[command.session_id] = job
|
||||
try:
|
||||
self._intake_queue_locked().put_nowait(job)
|
||||
except queue.Full as exc:
|
||||
if self._current_by_session.get(command.session_id) is job:
|
||||
if current is None:
|
||||
self._current_by_session.pop(command.session_id, None)
|
||||
else:
|
||||
self._current_by_session[command.session_id] = current
|
||||
raise RecordingPreparationQueueFull("recording preparation queue is full") from exc
|
||||
return self._snapshot_locked(job)
|
||||
|
||||
def resolve_cached(
|
||||
self,
|
||||
command: ReplayCommand,
|
||||
) -> RecordingPreparationSnapshot | None:
|
||||
"""Validate a published cache and register it as a ready job."""
|
||||
|
||||
# When launch preparation includes camera manifests, a disk-only RRD
|
||||
# is not a complete ready result. The worker must validate both parts
|
||||
# in one background transaction before publishing ``ready``.
|
||||
if self._ready_preparer is not None:
|
||||
return None
|
||||
|
||||
source_command = _source_command(command)
|
||||
identity = _source_identity(source_command)
|
||||
recording = self.materializer.get_cached(source_command)
|
||||
if recording is None:
|
||||
with self._guard:
|
||||
current = self._current_by_session.get(command.session_id)
|
||||
if current is not None and current.state == "ready":
|
||||
self._current_by_session.pop(command.session_id, None)
|
||||
return None
|
||||
with self._guard:
|
||||
current = self._current_by_session.get(command.session_id)
|
||||
if (
|
||||
current is not None
|
||||
and current.source_identity == identity
|
||||
and current.state in ACTIVE_PREPARATION_STATES | {"ready"}
|
||||
):
|
||||
# A worker may have started between the disk lookup and this
|
||||
# transaction. Let that single job publish its own result.
|
||||
if current.state == "ready":
|
||||
current.recording = recording
|
||||
return self._snapshot_locked(current)
|
||||
ready = _PreparationJob(
|
||||
preparation_id=uuid4().hex,
|
||||
source_identity=identity,
|
||||
command=source_command,
|
||||
state="ready",
|
||||
progress=1.0,
|
||||
recording=recording,
|
||||
recorded_media=(),
|
||||
)
|
||||
self._current_by_session[command.session_id] = ready
|
||||
return self._snapshot_locked(ready)
|
||||
|
||||
def resolve_cached_pinned(
|
||||
self,
|
||||
command: ReplayCommand,
|
||||
) -> tuple[RecordingPreparationSnapshot, Callable[[], None]] | None:
|
||||
"""Validate and lease a published cache without starting conversion."""
|
||||
|
||||
if self._ready_preparer is not None:
|
||||
return None
|
||||
|
||||
source_command = _source_command(command)
|
||||
result = self.materializer.get_cached_pinned(source_command)
|
||||
if result is None:
|
||||
with self._guard:
|
||||
current = self._current_by_session.get(command.session_id)
|
||||
if current is not None and current.state == "ready":
|
||||
self._current_by_session.pop(command.session_id, None)
|
||||
return None
|
||||
recording, release = result
|
||||
identity = _source_identity(source_command)
|
||||
with self._guard:
|
||||
current = self._current_by_session.get(command.session_id)
|
||||
if (
|
||||
current is None
|
||||
or current.source_identity != identity
|
||||
or current.state not in ACTIVE_PREPARATION_STATES | {"ready"}
|
||||
):
|
||||
current = _PreparationJob(
|
||||
preparation_id=uuid4().hex,
|
||||
source_identity=identity,
|
||||
command=source_command,
|
||||
state="ready",
|
||||
progress=1.0,
|
||||
recording=recording,
|
||||
recorded_media=(),
|
||||
)
|
||||
self._current_by_session[command.session_id] = current
|
||||
snapshot = self._snapshot_locked(current)
|
||||
return snapshot, release
|
||||
|
||||
def status(self, session_id: str) -> RecordingPreparationSnapshot | None:
|
||||
with self._guard:
|
||||
job = self._current_by_session.get(session_id)
|
||||
if (
|
||||
job is not None
|
||||
and job.state == "ready"
|
||||
and (
|
||||
job.recording is None
|
||||
or not self.materializer.is_recording_available(job.recording)
|
||||
)
|
||||
):
|
||||
self._current_by_session.pop(session_id, None)
|
||||
return None
|
||||
return None if job is None else self._snapshot_locked(job)
|
||||
|
||||
def reserve_cached(
|
||||
self,
|
||||
command: ReplayCommand,
|
||||
*,
|
||||
lease_seconds: float = 120.0,
|
||||
) -> RecordingPreparationSnapshot | None:
|
||||
"""Pin a ready artifact across the launch-document to file-GET gap."""
|
||||
|
||||
if lease_seconds <= 0:
|
||||
raise ValueError("recording launch lease must be positive")
|
||||
pinned = self.resolve_cached_pinned(command)
|
||||
if pinned is None:
|
||||
return None
|
||||
snapshot, release = pinned
|
||||
timer = threading.Timer(lease_seconds, release)
|
||||
timer.name = f"missioncore-recording-launch-lease-{snapshot.preparation_id}"
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
return snapshot
|
||||
|
||||
def pin_ready(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
preparation_id: str | None = None,
|
||||
) -> tuple[RecordingPreparationSnapshot, Callable[[], None]] | None:
|
||||
"""Cheaply lease the exact already-validated ready generation."""
|
||||
|
||||
with self._guard:
|
||||
job = self._current_by_session.get(session_id)
|
||||
if (
|
||||
job is None
|
||||
or job.state != "ready"
|
||||
or job.recording is None
|
||||
or (
|
||||
preparation_id is not None
|
||||
and job.preparation_id != preparation_id
|
||||
)
|
||||
):
|
||||
return None
|
||||
recording = job.recording
|
||||
pinned = self.materializer.pin_recording(recording)
|
||||
if pinned is None:
|
||||
if self._current_by_session.get(session_id) is job:
|
||||
self._current_by_session.pop(session_id, None)
|
||||
return None
|
||||
snapshot = self._snapshot_locked(job)
|
||||
return snapshot, pinned
|
||||
|
||||
def reserve_ready(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
preparation_id: str | None = None,
|
||||
lease_seconds: float = 120.0,
|
||||
) -> RecordingPreparationSnapshot | None:
|
||||
"""Hold a cheap launch lease without reopening or hashing the cache."""
|
||||
|
||||
if lease_seconds <= 0:
|
||||
raise ValueError("recording launch lease must be positive")
|
||||
pinned = self.pin_ready(session_id, preparation_id=preparation_id)
|
||||
if pinned is None:
|
||||
return None
|
||||
snapshot, release = pinned
|
||||
timer = threading.Timer(lease_seconds, release)
|
||||
timer.name = f"missioncore-recording-launch-lease-{snapshot.preparation_id}"
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
return snapshot
|
||||
|
||||
def cancel(self, session_id: str, *, preparation_id: str | None = None) -> bool:
|
||||
with self._guard:
|
||||
job = self._current_by_session.get(session_id)
|
||||
if (
|
||||
job is None
|
||||
or job.state not in ACTIVE_PREPARATION_STATES
|
||||
or (preparation_id is not None and job.preparation_id != preparation_id)
|
||||
):
|
||||
return False
|
||||
job.cancelled_by_operator = True
|
||||
job.cancel_event.set()
|
||||
if job.state == "queued":
|
||||
self._transition_locked(job, "cancelled", job.progress)
|
||||
return True
|
||||
|
||||
def close(self, *, timeout: float = 5.0) -> None:
|
||||
with self._guard:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
for job in self._current_by_session.values():
|
||||
if job.state in ACTIVE_PREPARATION_STATES:
|
||||
job.interrupted_by_restart = not job.cancelled_by_operator
|
||||
job.cancel_event.set()
|
||||
if job.state == "queued":
|
||||
self._transition_locked(job, "cancelled", job.progress)
|
||||
active = self._active_generation
|
||||
if active is not None:
|
||||
active.stop_event.set()
|
||||
pending = self._pending_generation
|
||||
if pending is not None:
|
||||
pending.stop_event.set()
|
||||
self._pending_generation = None
|
||||
worker = None if active is None else active.worker
|
||||
if worker is not None:
|
||||
worker.join(timeout=max(0.0, timeout))
|
||||
|
||||
def _run_generation(self, generation: _WorkerGeneration) -> None:
|
||||
work_queue = generation.work_queue
|
||||
try:
|
||||
while not generation.stop_event.is_set():
|
||||
try:
|
||||
job = work_queue.get(timeout=0.1)
|
||||
except queue.Empty:
|
||||
continue
|
||||
try:
|
||||
if job.state == "cancelled" or job.cancel_event.is_set():
|
||||
with self._guard:
|
||||
self._transition_locked(job, "cancelled", job.progress)
|
||||
continue
|
||||
with self._guard:
|
||||
self._transition_locked(job, "validating", 0.05)
|
||||
try:
|
||||
recording = self.materializer.materialize(
|
||||
job.command,
|
||||
progress_callback=partial(self._progress, job),
|
||||
cancel_event=job.cancel_event,
|
||||
)
|
||||
except RecordingMaterializationCancelled:
|
||||
with self._guard:
|
||||
self._transition_locked(job, "cancelled", job.progress)
|
||||
except Exception:
|
||||
# The public status intentionally does not expose paths,
|
||||
# broker payloads or exporter internals.
|
||||
with self._guard:
|
||||
if job.cancel_event.is_set():
|
||||
self._transition_locked(job, "cancelled", job.progress)
|
||||
else:
|
||||
self._transition_locked(job, "failed", job.progress)
|
||||
job.error = "Не удалось подготовить запись сессии."
|
||||
job.updated_at_utc = _utc_now_iso()
|
||||
else:
|
||||
with self._guard:
|
||||
# Cancellation may race the final materializer
|
||||
# return. Never publish ``ready`` after accepting a
|
||||
# cancel request for this exact job.
|
||||
if job.cancel_event.is_set():
|
||||
self._transition_locked(job, "cancelled", job.progress)
|
||||
else:
|
||||
self._transition_locked(job, "finalizing", 0.95)
|
||||
if job.cancel_event.is_set():
|
||||
with self._guard:
|
||||
self._transition_locked(job, "cancelled", job.progress)
|
||||
continue
|
||||
try:
|
||||
recorded_media = (
|
||||
()
|
||||
if self._ready_preparer is None
|
||||
else self._ready_preparer(job.command, recording)
|
||||
)
|
||||
validate_recorded_media_timeline(
|
||||
recorded_media,
|
||||
recording_start_seconds=(
|
||||
recording.timeline_start_ns / 1_000_000_000
|
||||
),
|
||||
recording_end_seconds=(
|
||||
recording.timeline_end_ns / 1_000_000_000
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
with self._guard:
|
||||
if job.cancel_event.is_set():
|
||||
self._transition_locked(job, "cancelled", job.progress)
|
||||
else:
|
||||
self._transition_locked(job, "failed", job.progress)
|
||||
job.error = "Не удалось подготовить запись сессии."
|
||||
job.updated_at_utc = _utc_now_iso()
|
||||
continue
|
||||
with self._guard:
|
||||
if job.cancel_event.is_set():
|
||||
self._transition_locked(job, "cancelled", job.progress)
|
||||
else:
|
||||
job.recording = recording
|
||||
job.recorded_media = recorded_media
|
||||
self._transition_locked(job, "ready", 1.0)
|
||||
finally:
|
||||
work_queue.task_done()
|
||||
finally:
|
||||
self._generation_exited(generation)
|
||||
|
||||
def _new_generation_locked(self) -> _WorkerGeneration:
|
||||
self._generation_counter += 1
|
||||
return _WorkerGeneration(
|
||||
generation_id=self._generation_counter,
|
||||
work_queue=queue.Queue(maxsize=self._queue_capacity),
|
||||
)
|
||||
|
||||
def _start_generation_locked(self, generation: _WorkerGeneration) -> None:
|
||||
self._active_generation = generation
|
||||
worker = threading.Thread(
|
||||
target=self._run_generation,
|
||||
args=(generation,),
|
||||
name=f"missioncore-recording-preparation-{generation.generation_id}",
|
||||
daemon=True,
|
||||
)
|
||||
generation.worker = worker
|
||||
worker.start()
|
||||
|
||||
def _generation_exited(self, generation: _WorkerGeneration) -> None:
|
||||
with self._guard:
|
||||
if self._active_generation is not generation:
|
||||
return
|
||||
self._active_generation = None
|
||||
if self._closed:
|
||||
return
|
||||
successor = self._pending_generation or self._new_generation_locked()
|
||||
self._pending_generation = None
|
||||
self._start_generation_locked(successor)
|
||||
|
||||
def _intake_queue_locked(self) -> queue.Queue[_PreparationJob]:
|
||||
pending = self._pending_generation
|
||||
if pending is not None:
|
||||
return pending.work_queue
|
||||
active = self._active_generation
|
||||
if active is None or active.stop_event.is_set():
|
||||
pending = self._new_generation_locked()
|
||||
self._pending_generation = pending
|
||||
return pending.work_queue
|
||||
return active.work_queue
|
||||
|
||||
def _progress(self, job: _PreparationJob, state: str, progress: float) -> None:
|
||||
# Only the worker publishes ``ready`` after it owns the verified
|
||||
# recording handle; accepting the final callback would expose a short
|
||||
# ready-without-launch race to status polling.
|
||||
if state not in {"validating", "exporting", "finalizing"}:
|
||||
return
|
||||
next_state = cast(PreparationState, state)
|
||||
with self._guard:
|
||||
if job.state in {"cancelled", "failed", "ready"}:
|
||||
return
|
||||
if _PREPARATION_PHASE[next_state] < _PREPARATION_PHASE[job.state]:
|
||||
return
|
||||
now = monotonic()
|
||||
if (
|
||||
job.state == next_state
|
||||
and progress <= job.progress
|
||||
and now - job.last_activity_monotonic < self._heartbeat_interval_seconds
|
||||
):
|
||||
return
|
||||
self._transition_locked(job, next_state, progress)
|
||||
|
||||
def _transition_locked(
|
||||
self,
|
||||
job: _PreparationJob,
|
||||
state: PreparationState,
|
||||
progress: float,
|
||||
) -> None:
|
||||
if job.state not in ACTIVE_PREPARATION_STATES and state != job.state:
|
||||
return
|
||||
if _PREPARATION_PHASE[state] < _PREPARATION_PHASE[job.state]:
|
||||
return
|
||||
job.state = state
|
||||
job.progress = max(job.progress, min(1.0, max(0.0, progress)))
|
||||
job.updated_at_utc = _utc_now_iso()
|
||||
job.last_activity_monotonic = monotonic()
|
||||
|
||||
def _snapshot_locked(self, job: _PreparationJob) -> RecordingPreparationSnapshot:
|
||||
return RecordingPreparationSnapshot(
|
||||
preparation_id=job.preparation_id,
|
||||
session_id=job.command.session_id,
|
||||
state=job.state,
|
||||
progress=job.progress,
|
||||
updated_at_utc=job.updated_at_utc,
|
||||
cancellable=(
|
||||
job.state == "queued"
|
||||
or (
|
||||
job.state in {"validating", "exporting", "finalizing"}
|
||||
and self.materializer.supports_cooperative_cancellation
|
||||
)
|
||||
),
|
||||
retryable=job.state in {"failed", "cancelled"},
|
||||
error=job.error,
|
||||
command=job.command,
|
||||
recording=job.recording,
|
||||
recorded_media=job.recorded_media,
|
||||
)
|
||||
|
||||
|
||||
def _source_identity(command: ReplayCommand) -> tuple[object, ...]:
|
||||
"""Build a non-blocking identity; full validation belongs to the worker."""
|
||||
|
||||
identities: list[object] = [
|
||||
command.session_id,
|
||||
str(command.source_path),
|
||||
command.replay_byte_length,
|
||||
command.metadata_byte_length,
|
||||
command.expected_source_sha256,
|
||||
]
|
||||
for path in (command.source_path, command.source_path.with_name("mqtt.metadata.jsonl")):
|
||||
try:
|
||||
value = os.lstat(path)
|
||||
except OSError:
|
||||
identities.extend((str(Path(path)), None, None, None, None))
|
||||
else:
|
||||
identities.extend(
|
||||
(
|
||||
str(Path(path)),
|
||||
value.st_size,
|
||||
value.st_mtime_ns,
|
||||
value.st_ctime_ns,
|
||||
value.st_ino,
|
||||
)
|
||||
)
|
||||
return tuple(identities)
|
||||
|
||||
|
||||
def _source_command(command: ReplayCommand) -> ReplayCommand:
|
||||
"""Remove per-viewer launch policy from a shared preparation job."""
|
||||
|
||||
return replace(command, speed=1.0, loop=False)
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||
Reference in New Issue
Block a user