fix(archive): preserve prepared sessions across restarts

This commit is contained in:
DCCONSTRUCTIONS
2026-07-18 11:08:27 +03:00
parent aa3680948f
commit 574a494759
13 changed files with 398 additions and 132 deletions
+33
View File
@@ -574,6 +574,39 @@ def test_cold_replay_returns_quick_202_then_status_returns_ready_launch(
manager.close()
def test_catalog_read_never_prepares_a_cold_historical_session(tmp_path: Path) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
store = SessionStore(repository, data_dir=tmp_path / "data")
store.reconcile_archive(xgrids_k1_archive_source(sessions))
calls = 0
def exporter(_source: Path, _destination: Path) -> dict[str, object]:
nonlocal calls
calls += 1
raise AssertionError("catalog reads must not invoke the exporter")
materializer = SessionRecordingMaterializer(store.data_dir, exporter=exporter)
manager = SessionRecordingPreparationManager(materializer)
router = build_session_router(
store,
recording_materializer=materializer,
recording_preparation_manager=manager,
)
list_route = endpoint(router, "/api/v1/observation-sessions", "GET")
try:
item = list_route(limit=20, cursor=None)["items"][0]
time.sleep(0.02)
assert item["id"] == session.name
assert item["preparation"] is None
assert manager.status(session.name) is None
assert calls == 0
finally:
manager.close()
def test_recording_response_releases_pin_once_when_asgi_send_fails(
tmp_path: Path,
) -> None:
+15 -22
View File
@@ -160,7 +160,7 @@ def test_catalog_refresh_failure_is_a_stable_service_error(tmp_path: Path) -> No
assert "private local path" not in str(error.value.detail)
def test_startup_warmup_enqueues_every_finalized_replayable_session(
def test_startup_scan_baselines_historical_sessions_without_enqueuing(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -188,17 +188,14 @@ def test_startup_warmup_enqueues_every_finalized_replayable_session(
monkeypatch.setattr(app_module, "session_store", store)
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
try:
enqueued = app_module.enqueue_replayable_recordings()
deadline = time.monotonic() + 2
snapshot = manager.status(session.name)
while snapshot is not None and snapshot.state != "ready" and time.monotonic() < deadline:
time.sleep(0.005)
snapshot = manager.status(session.name)
historical = set(app_module.finalized_replayable_recording_ids())
newly_finalized = app_module.newly_finalized_recording_ids(None, historical)
enqueued = app_module.enqueue_replayable_recordings(newly_finalized)
assert enqueued == (session.name,)
assert snapshot is not None
assert snapshot.state == "ready"
assert snapshot.recording is not None
assert historical == {session.name}
assert newly_finalized == ()
assert enqueued == ()
assert manager.status(session.name) is None
finally:
manager.close()
@@ -233,7 +230,7 @@ def test_reconciliation_skips_one_stale_session_and_prepares_later_valid_session
monkeypatch.setattr(app_module, "session_store", store)
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
try:
enqueued = app_module.enqueue_replayable_recordings()
enqueued = app_module.enqueue_replayable_recordings((stale.name, good.name))
deadline = time.monotonic() + 2
snapshot = manager.status(good.name)
while snapshot is not None and snapshot.state != "ready" and time.monotonic() < deadline:
@@ -283,7 +280,7 @@ def test_reconciler_requeues_only_a_restart_interrupted_job(
monkeypatch.setattr(app_module, "session_store", store)
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
try:
app_module.enqueue_replayable_recordings()
app_module.enqueue_replayable_recordings((session.name,))
assert started.wait(timeout=1)
manager.close(timeout=0.001)
manager.start()
@@ -291,15 +288,13 @@ def test_reconciler_requeues_only_a_restart_interrupted_job(
deadline = time.monotonic() + 2
snapshot = manager.status(session.name)
while (
snapshot is None or snapshot.state != "cancelled"
) and time.monotonic() < deadline:
while (snapshot is None or snapshot.state != "cancelled") and time.monotonic() < deadline:
time.sleep(0.005)
snapshot = manager.status(session.name)
assert snapshot is not None and snapshot.state == "cancelled"
interrupted_id = snapshot.preparation_id
app_module.enqueue_replayable_recordings()
app_module.enqueue_replayable_recordings((session.name,))
deadline = time.monotonic() + 2
while time.monotonic() < deadline:
snapshot = manager.status(session.name)
@@ -336,18 +331,16 @@ def test_reconciler_does_not_loop_retry_a_genuine_failed_job(
monkeypatch.setattr(app_module, "session_store", store)
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
try:
app_module.enqueue_replayable_recordings()
app_module.enqueue_replayable_recordings((session.name,))
deadline = time.monotonic() + 2
snapshot = manager.status(session.name)
while (
snapshot is None or snapshot.state != "failed"
) and time.monotonic() < deadline:
while (snapshot is None or snapshot.state != "failed") and time.monotonic() < deadline:
time.sleep(0.005)
snapshot = manager.status(session.name)
assert snapshot is not None and snapshot.state == "failed"
failed_id = snapshot.preparation_id
app_module.enqueue_replayable_recordings()
app_module.enqueue_replayable_recordings((session.name,))
time.sleep(0.05)
unchanged = manager.status(session.name)
assert unchanged is not None
+40
View File
@@ -231,6 +231,46 @@ def test_manager_can_restart_across_repeated_application_lifespans(tmp_path: Pat
manager.close()
def test_published_package_restores_after_process_restart_without_export(
tmp_path: Path,
) -> None:
command = _command(tmp_path / "session")
private_root = tmp_path / "private"
calls = 0
def exporter(source: Path, destination: Path) -> dict[str, object]:
nonlocal calls
calls += 1
return _summary(source, destination, b"durable-recording")
first = SessionRecordingPreparationManager(
SessionRecordingMaterializer(private_root, exporter=exporter)
)
try:
first.enqueue(command)
ready = _wait_for_state(first, command.session_id, {"ready"})
assert ready.recording is not None
finally:
first.close()
def forbidden_exporter(_source: Path, _destination: Path) -> dict[str, object]:
raise AssertionError("a published package must not be exported again")
restarted = SessionRecordingPreparationManager(
SessionRecordingMaterializer(private_root, exporter=forbidden_exporter)
)
try:
restored = restarted.restore_published(command)
assert restored is not None
assert restored.state == "ready"
assert restored.recording is not None
assert restored.recording.path.read_bytes() == b"durable-recording"
assert calls == 1
finally:
restarted.close()
def test_noncooperative_exporter_has_no_fake_heartbeat_and_restart_waits_for_old_writer(
tmp_path: Path,
) -> None: