436 lines
14 KiB
Python
436 lines
14 KiB
Python
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import threading
|
||
import time
|
||
from collections.abc import Callable
|
||
from dataclasses import replace
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from k1link.sessions.models import ReplayArtifact, ReplayCommand
|
||
from k1link.sessions.preparation import (
|
||
RecordingPreparationSnapshot,
|
||
SessionRecordingPreparationManager,
|
||
)
|
||
from k1link.sessions.recording import RecordingMaterializationError, SessionRecordingMaterializer
|
||
|
||
|
||
def _command(root: Path, session_id: str = "20260717T100000Z_viewer_live") -> ReplayCommand:
|
||
root.mkdir(parents=True)
|
||
source = root / "mqtt.raw.k1mqtt"
|
||
metadata = root / "mqtt.metadata.jsonl"
|
||
source.write_bytes(b"native-session-source")
|
||
metadata.write_text('{"record_type":"message","sequence":1}\n', encoding="utf-8")
|
||
return ReplayCommand(
|
||
session_id=session_id,
|
||
plugin_id="test.recording-plugin",
|
||
allowed_root=root,
|
||
session_root=root,
|
||
primary_artifact_id="primary",
|
||
artifacts=(
|
||
ReplayArtifact(
|
||
artifact_id="primary",
|
||
path=source,
|
||
media_type="application/x-test-recording",
|
||
file_byte_length=source.stat().st_size,
|
||
replay_byte_length=source.stat().st_size,
|
||
expected_sha256=None,
|
||
),
|
||
ReplayArtifact(
|
||
artifact_id="index",
|
||
path=metadata,
|
||
media_type="application/x-ndjson",
|
||
file_byte_length=metadata.stat().st_size,
|
||
replay_byte_length=metadata.stat().st_size,
|
||
expected_sha256=None,
|
||
),
|
||
),
|
||
timeline_origin_epoch_ns=0,
|
||
timeline_origin_monotonic_ns=0,
|
||
speed=1.0,
|
||
loop=False,
|
||
)
|
||
|
||
|
||
def _summary(source: Path, destination: Path, payload: bytes) -> dict[str, object]:
|
||
destination.write_bytes(payload)
|
||
return {
|
||
"source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(),
|
||
"rrd_sha256": hashlib.sha256(payload).hexdigest(),
|
||
"rrd_bytes": len(payload),
|
||
"timeline": "session_time",
|
||
"timeline_start_ns": 0,
|
||
"timeline_end_ns": 1_000_000_000,
|
||
}
|
||
|
||
|
||
def _wait_for_state(
|
||
manager: SessionRecordingPreparationManager,
|
||
session_id: str,
|
||
states: set[str],
|
||
timeout: float = 2.0,
|
||
) -> RecordingPreparationSnapshot:
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
snapshot = manager.status(session_id)
|
||
if snapshot is not None and snapshot.state in states:
|
||
return snapshot
|
||
time.sleep(0.005)
|
||
raise AssertionError(f"preparation did not reach {states}")
|
||
|
||
|
||
def test_manager_returns_quick_job_deduplicates_and_reports_monotonic_progress(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
command = _command(tmp_path / "session")
|
||
started = threading.Event()
|
||
release = threading.Event()
|
||
calls = 0
|
||
|
||
def exporter(
|
||
source: Path,
|
||
destination: Path,
|
||
*,
|
||
cancel_event: threading.Event | None = None,
|
||
activity_callback: Callable[[], None] | None = None,
|
||
) -> dict[str, object]:
|
||
nonlocal calls
|
||
calls += 1
|
||
started.set()
|
||
deadline = time.monotonic() + 2
|
||
while not release.wait(timeout=0.005):
|
||
assert cancel_event is None or not cancel_event.is_set()
|
||
if activity_callback is not None:
|
||
activity_callback()
|
||
assert time.monotonic() < deadline
|
||
return _summary(source, destination, b"prepared-recording")
|
||
|
||
manager = SessionRecordingPreparationManager(
|
||
SessionRecordingMaterializer(tmp_path / "private", exporter=exporter),
|
||
heartbeat_interval_seconds=0.01,
|
||
)
|
||
try:
|
||
before = time.monotonic()
|
||
first = manager.enqueue(command)
|
||
elapsed = time.monotonic() - before
|
||
duplicate = manager.enqueue(command)
|
||
|
||
assert elapsed < 0.1
|
||
assert duplicate.preparation_id == first.preparation_id
|
||
assert started.wait(timeout=1)
|
||
exporting = _wait_for_state(manager, command.session_id, {"exporting"})
|
||
time.sleep(0.03)
|
||
heartbeat = manager.status(command.session_id)
|
||
assert heartbeat is not None
|
||
assert heartbeat.updated_at_utc > exporting.updated_at_utc
|
||
assert heartbeat.progress >= exporting.progress
|
||
assert heartbeat.cancellable is True
|
||
release.set()
|
||
ready = _wait_for_state(manager, command.session_id, {"ready"})
|
||
|
||
assert calls == 1
|
||
assert exporting.progress <= ready.progress == 1.0
|
||
assert ready.recording is not None
|
||
assert ready.updated_at_utc.endswith("Z")
|
||
finally:
|
||
release.set()
|
||
manager.close()
|
||
|
||
|
||
def test_manager_failure_is_retryable_and_retry_publishes_new_job(tmp_path: Path) -> None:
|
||
command = _command(tmp_path / "session")
|
||
calls = 0
|
||
|
||
def exporter(source: Path, destination: Path) -> dict[str, object]:
|
||
nonlocal calls
|
||
calls += 1
|
||
if calls == 1:
|
||
raise OSError("simulated converter failure")
|
||
return _summary(source, destination, b"retry-recording")
|
||
|
||
manager = SessionRecordingPreparationManager(
|
||
SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
|
||
)
|
||
try:
|
||
first = manager.enqueue(command)
|
||
failed = _wait_for_state(manager, command.session_id, {"failed"})
|
||
retry = manager.enqueue(command, retry_failed=True)
|
||
ready = _wait_for_state(manager, command.session_id, {"ready"})
|
||
|
||
assert failed.retryable is True
|
||
assert failed.error == "Не удалось подготовить запись сессии."
|
||
assert retry.preparation_id != first.preparation_id
|
||
assert ready.preparation_id == retry.preparation_id
|
||
assert calls == 2
|
||
finally:
|
||
manager.close()
|
||
|
||
|
||
def test_manager_cancels_queued_job_without_exporting_it(tmp_path: Path) -> None:
|
||
first = _command(tmp_path / "first")
|
||
second = replace(
|
||
_command(tmp_path / "second"),
|
||
session_id="20260717T100001Z_viewer_live",
|
||
)
|
||
started = threading.Event()
|
||
release = threading.Event()
|
||
calls = 0
|
||
|
||
def exporter(source: Path, destination: Path) -> dict[str, object]:
|
||
nonlocal calls
|
||
calls += 1
|
||
started.set()
|
||
assert release.wait(timeout=2)
|
||
return _summary(source, destination, b"recording")
|
||
|
||
manager = SessionRecordingPreparationManager(
|
||
SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
|
||
)
|
||
try:
|
||
manager.enqueue(first)
|
||
assert started.wait(timeout=1)
|
||
manager.enqueue(second)
|
||
assert manager.cancel(second.session_id) is True
|
||
cancelled = _wait_for_state(manager, second.session_id, {"cancelled"})
|
||
release.set()
|
||
_wait_for_state(manager, first.session_id, {"ready"})
|
||
time.sleep(0.02)
|
||
|
||
assert cancelled.cancellable is False
|
||
assert calls == 1
|
||
finally:
|
||
release.set()
|
||
manager.close()
|
||
|
||
|
||
def test_manager_can_restart_across_repeated_application_lifespans(tmp_path: Path) -> None:
|
||
command = _command(tmp_path / "session")
|
||
calls = 0
|
||
|
||
def exporter(source: Path, destination: Path) -> dict[str, object]:
|
||
nonlocal calls
|
||
calls += 1
|
||
return _summary(source, destination, b"restartable-recording")
|
||
|
||
manager = SessionRecordingPreparationManager(
|
||
SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
|
||
)
|
||
try:
|
||
manager.enqueue(command)
|
||
_wait_for_state(manager, command.session_id, {"ready"})
|
||
manager.close()
|
||
manager.start()
|
||
|
||
resumed = manager.enqueue(command)
|
||
|
||
assert resumed.state == "ready"
|
||
assert calls == 1
|
||
finally:
|
||
manager.close()
|
||
|
||
|
||
def test_published_package_restores_after_process_restart_without_export(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
command = _command(tmp_path / "session")
|
||
private_root = tmp_path / "private"
|
||
calls = 0
|
||
|
||
def exporter(source: Path, destination: Path) -> dict[str, object]:
|
||
nonlocal calls
|
||
calls += 1
|
||
return _summary(source, destination, b"durable-recording")
|
||
|
||
first = SessionRecordingPreparationManager(
|
||
SessionRecordingMaterializer(private_root, exporter=exporter)
|
||
)
|
||
try:
|
||
first.enqueue(command)
|
||
ready = _wait_for_state(first, command.session_id, {"ready"})
|
||
assert ready.recording is not None
|
||
finally:
|
||
first.close()
|
||
|
||
def forbidden_exporter(_source: Path, _destination: Path) -> dict[str, object]:
|
||
raise AssertionError("a published package must not be exported again")
|
||
|
||
restarted = SessionRecordingPreparationManager(
|
||
SessionRecordingMaterializer(private_root, exporter=forbidden_exporter)
|
||
)
|
||
try:
|
||
restored = restarted.restore_published(command)
|
||
|
||
assert restored is not None
|
||
assert restored.state == "ready"
|
||
assert restored.recording is not None
|
||
assert restored.recording.path.read_bytes() == b"durable-recording"
|
||
assert calls == 1
|
||
finally:
|
||
restarted.close()
|
||
|
||
|
||
def test_noncooperative_exporter_has_no_fake_heartbeat_and_restart_waits_for_old_writer(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
first = _command(tmp_path / "first")
|
||
second = replace(
|
||
_command(tmp_path / "second"),
|
||
session_id="20260717T100002Z_viewer_live",
|
||
)
|
||
first_started = threading.Event()
|
||
release_first = threading.Event()
|
||
active = 0
|
||
max_active = 0
|
||
calls = 0
|
||
guard = threading.Lock()
|
||
|
||
def exporter(source: Path, destination: Path) -> dict[str, object]:
|
||
nonlocal active, calls, max_active
|
||
with guard:
|
||
calls += 1
|
||
active += 1
|
||
max_active = max(max_active, active)
|
||
call = calls
|
||
try:
|
||
if call == 1:
|
||
first_started.set()
|
||
assert release_first.wait(timeout=2)
|
||
return _summary(source, destination, f"recording-{call}".encode())
|
||
finally:
|
||
with guard:
|
||
active -= 1
|
||
|
||
manager = SessionRecordingPreparationManager(
|
||
SessionRecordingMaterializer(tmp_path / "private", exporter=exporter),
|
||
heartbeat_interval_seconds=0.01,
|
||
)
|
||
try:
|
||
manager.enqueue(first)
|
||
assert first_started.wait(timeout=1)
|
||
blocked = _wait_for_state(manager, first.session_id, {"exporting"})
|
||
time.sleep(0.03)
|
||
unchanged = manager.status(first.session_id)
|
||
assert unchanged is not None
|
||
assert unchanged.updated_at_utc == blocked.updated_at_utc
|
||
assert unchanged.cancellable is False
|
||
|
||
manager.close(timeout=0.001)
|
||
manager.start()
|
||
queued = manager.enqueue(second)
|
||
assert queued.state == "queued"
|
||
time.sleep(0.03)
|
||
assert calls == 1
|
||
|
||
release_first.set()
|
||
ready = _wait_for_state(manager, second.session_id, {"ready"})
|
||
assert ready.recording is not None
|
||
assert calls == 2
|
||
assert max_active == 1
|
||
finally:
|
||
release_first.set()
|
||
manager.close()
|
||
|
||
|
||
def test_reconciler_retry_does_not_replace_operator_cancel_across_restart(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
command = _command(tmp_path / "session")
|
||
started = threading.Event()
|
||
release = threading.Event()
|
||
calls = 0
|
||
|
||
def exporter(source: Path, destination: Path) -> dict[str, object]:
|
||
nonlocal calls
|
||
calls += 1
|
||
started.set()
|
||
assert release.wait(timeout=2)
|
||
return _summary(source, destination, b"operator-cancelled")
|
||
|
||
manager = SessionRecordingPreparationManager(
|
||
SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
|
||
)
|
||
try:
|
||
original = manager.enqueue(command)
|
||
assert started.wait(timeout=1)
|
||
assert manager.cancel(
|
||
command.session_id,
|
||
preparation_id=original.preparation_id,
|
||
)
|
||
manager.close(timeout=0.001)
|
||
manager.start()
|
||
release.set()
|
||
cancelled = _wait_for_state(manager, command.session_id, {"cancelled"})
|
||
|
||
reconciled = manager.enqueue(command, retry_interrupted=True)
|
||
|
||
assert reconciled.preparation_id == cancelled.preparation_id
|
||
assert reconciled.state == "cancelled"
|
||
assert calls == 1
|
||
finally:
|
||
release.set()
|
||
manager.close()
|
||
|
||
|
||
def test_shared_preparation_command_ignores_per_request_playback_policy(tmp_path: Path) -> None:
|
||
command = _command(tmp_path / "session")
|
||
started = threading.Event()
|
||
release = threading.Event()
|
||
|
||
def exporter(source: Path, destination: Path) -> dict[str, object]:
|
||
started.set()
|
||
assert release.wait(timeout=2)
|
||
return _summary(source, destination, b"recording")
|
||
|
||
manager = SessionRecordingPreparationManager(
|
||
SessionRecordingMaterializer(tmp_path / "private", exporter=exporter)
|
||
)
|
||
try:
|
||
first = manager.enqueue(replace(command, speed=2.0, loop=True))
|
||
assert started.wait(timeout=1)
|
||
duplicate = manager.enqueue(replace(command, speed=7.0, loop=False))
|
||
|
||
assert duplicate.preparation_id == first.preparation_id
|
||
assert duplicate.command.speed == 1.0
|
||
assert duplicate.command.loop is False
|
||
finally:
|
||
release.set()
|
||
manager.close()
|
||
|
||
|
||
def test_launch_reservation_blocks_eviction_until_lease_expires(tmp_path: Path) -> None:
|
||
first = _command(tmp_path / "first")
|
||
second = replace(
|
||
_command(tmp_path / "second"),
|
||
session_id="20260717T100003Z_viewer_live",
|
||
)
|
||
|
||
def exporter(source: Path, destination: Path) -> dict[str, object]:
|
||
return _summary(source, destination, b"R" * (40 * 1024))
|
||
|
||
materializer = SessionRecordingMaterializer(
|
||
tmp_path / "private",
|
||
exporter=exporter,
|
||
cache_max_bytes=50 * 1024,
|
||
free_space_reserve_bytes=0,
|
||
)
|
||
manager = SessionRecordingPreparationManager(materializer)
|
||
try:
|
||
first_recording = materializer.materialize(first)
|
||
resolved = manager.resolve_cached(first)
|
||
assert resolved is not None and resolved.state == "ready"
|
||
reserved = manager.reserve_cached(first, lease_seconds=0.05)
|
||
assert reserved is not None and reserved.recording is not None
|
||
|
||
with pytest.raises(RecordingMaterializationError, match="cache quota"):
|
||
materializer.materialize(second)
|
||
assert first_recording.path.exists()
|
||
|
||
time.sleep(0.08)
|
||
second_recording = materializer.materialize(second)
|
||
assert second_recording.path.exists()
|
||
assert not first_recording.path.exists()
|
||
finally:
|
||
manager.close()
|