fix(archive): preserve prepared sessions across restarts
This commit is contained in:
@@ -574,6 +574,39 @@ def test_cold_replay_returns_quick_202_then_status_returns_ready_launch(
|
||||
manager.close()
|
||||
|
||||
|
||||
def test_catalog_read_never_prepares_a_cold_historical_session(tmp_path: Path) -> None:
|
||||
repository = tmp_path / "repo"
|
||||
sessions = repository / "sessions"
|
||||
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
|
||||
store = SessionStore(repository, data_dir=tmp_path / "data")
|
||||
store.reconcile_archive(xgrids_k1_archive_source(sessions))
|
||||
calls = 0
|
||||
|
||||
def exporter(_source: Path, _destination: Path) -> dict[str, object]:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise AssertionError("catalog reads must not invoke the exporter")
|
||||
|
||||
materializer = SessionRecordingMaterializer(store.data_dir, exporter=exporter)
|
||||
manager = SessionRecordingPreparationManager(materializer)
|
||||
router = build_session_router(
|
||||
store,
|
||||
recording_materializer=materializer,
|
||||
recording_preparation_manager=manager,
|
||||
)
|
||||
list_route = endpoint(router, "/api/v1/observation-sessions", "GET")
|
||||
try:
|
||||
item = list_route(limit=20, cursor=None)["items"][0]
|
||||
time.sleep(0.02)
|
||||
|
||||
assert item["id"] == session.name
|
||||
assert item["preparation"] is None
|
||||
assert manager.status(session.name) is None
|
||||
assert calls == 0
|
||||
finally:
|
||||
manager.close()
|
||||
|
||||
|
||||
def test_recording_response_releases_pin_once_when_asgi_send_fails(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
|
||||
@@ -160,7 +160,7 @@ def test_catalog_refresh_failure_is_a_stable_service_error(tmp_path: Path) -> No
|
||||
assert "private local path" not in str(error.value.detail)
|
||||
|
||||
|
||||
def test_startup_warmup_enqueues_every_finalized_replayable_session(
|
||||
def test_startup_scan_baselines_historical_sessions_without_enqueuing(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
@@ -188,17 +188,14 @@ def test_startup_warmup_enqueues_every_finalized_replayable_session(
|
||||
monkeypatch.setattr(app_module, "session_store", store)
|
||||
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
||||
try:
|
||||
enqueued = app_module.enqueue_replayable_recordings()
|
||||
deadline = time.monotonic() + 2
|
||||
snapshot = manager.status(session.name)
|
||||
while snapshot is not None and snapshot.state != "ready" and time.monotonic() < deadline:
|
||||
time.sleep(0.005)
|
||||
snapshot = manager.status(session.name)
|
||||
historical = set(app_module.finalized_replayable_recording_ids())
|
||||
newly_finalized = app_module.newly_finalized_recording_ids(None, historical)
|
||||
enqueued = app_module.enqueue_replayable_recordings(newly_finalized)
|
||||
|
||||
assert enqueued == (session.name,)
|
||||
assert snapshot is not None
|
||||
assert snapshot.state == "ready"
|
||||
assert snapshot.recording is not None
|
||||
assert historical == {session.name}
|
||||
assert newly_finalized == ()
|
||||
assert enqueued == ()
|
||||
assert manager.status(session.name) is None
|
||||
finally:
|
||||
manager.close()
|
||||
|
||||
@@ -233,7 +230,7 @@ def test_reconciliation_skips_one_stale_session_and_prepares_later_valid_session
|
||||
monkeypatch.setattr(app_module, "session_store", store)
|
||||
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
||||
try:
|
||||
enqueued = app_module.enqueue_replayable_recordings()
|
||||
enqueued = app_module.enqueue_replayable_recordings((stale.name, good.name))
|
||||
deadline = time.monotonic() + 2
|
||||
snapshot = manager.status(good.name)
|
||||
while snapshot is not None and snapshot.state != "ready" and time.monotonic() < deadline:
|
||||
@@ -283,7 +280,7 @@ def test_reconciler_requeues_only_a_restart_interrupted_job(
|
||||
monkeypatch.setattr(app_module, "session_store", store)
|
||||
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
||||
try:
|
||||
app_module.enqueue_replayable_recordings()
|
||||
app_module.enqueue_replayable_recordings((session.name,))
|
||||
assert started.wait(timeout=1)
|
||||
manager.close(timeout=0.001)
|
||||
manager.start()
|
||||
@@ -291,15 +288,13 @@ def test_reconciler_requeues_only_a_restart_interrupted_job(
|
||||
|
||||
deadline = time.monotonic() + 2
|
||||
snapshot = manager.status(session.name)
|
||||
while (
|
||||
snapshot is None or snapshot.state != "cancelled"
|
||||
) and time.monotonic() < deadline:
|
||||
while (snapshot is None or snapshot.state != "cancelled") and time.monotonic() < deadline:
|
||||
time.sleep(0.005)
|
||||
snapshot = manager.status(session.name)
|
||||
assert snapshot is not None and snapshot.state == "cancelled"
|
||||
interrupted_id = snapshot.preparation_id
|
||||
|
||||
app_module.enqueue_replayable_recordings()
|
||||
app_module.enqueue_replayable_recordings((session.name,))
|
||||
deadline = time.monotonic() + 2
|
||||
while time.monotonic() < deadline:
|
||||
snapshot = manager.status(session.name)
|
||||
@@ -336,18 +331,16 @@ def test_reconciler_does_not_loop_retry_a_genuine_failed_job(
|
||||
monkeypatch.setattr(app_module, "session_store", store)
|
||||
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
||||
try:
|
||||
app_module.enqueue_replayable_recordings()
|
||||
app_module.enqueue_replayable_recordings((session.name,))
|
||||
deadline = time.monotonic() + 2
|
||||
snapshot = manager.status(session.name)
|
||||
while (
|
||||
snapshot is None or snapshot.state != "failed"
|
||||
) and time.monotonic() < deadline:
|
||||
while (snapshot is None or snapshot.state != "failed") and time.monotonic() < deadline:
|
||||
time.sleep(0.005)
|
||||
snapshot = manager.status(session.name)
|
||||
assert snapshot is not None and snapshot.state == "failed"
|
||||
failed_id = snapshot.preparation_id
|
||||
|
||||
app_module.enqueue_replayable_recordings()
|
||||
app_module.enqueue_replayable_recordings((session.name,))
|
||||
time.sleep(0.05)
|
||||
unchanged = manager.status(session.name)
|
||||
assert unchanged is not None
|
||||
|
||||
@@ -231,6 +231,46 @@ def test_manager_can_restart_across_repeated_application_lifespans(tmp_path: Pat
|
||||
manager.close()
|
||||
|
||||
|
||||
def test_published_package_restores_after_process_restart_without_export(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
command = _command(tmp_path / "session")
|
||||
private_root = tmp_path / "private"
|
||||
calls = 0
|
||||
|
||||
def exporter(source: Path, destination: Path) -> dict[str, object]:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return _summary(source, destination, b"durable-recording")
|
||||
|
||||
first = SessionRecordingPreparationManager(
|
||||
SessionRecordingMaterializer(private_root, exporter=exporter)
|
||||
)
|
||||
try:
|
||||
first.enqueue(command)
|
||||
ready = _wait_for_state(first, command.session_id, {"ready"})
|
||||
assert ready.recording is not None
|
||||
finally:
|
||||
first.close()
|
||||
|
||||
def forbidden_exporter(_source: Path, _destination: Path) -> dict[str, object]:
|
||||
raise AssertionError("a published package must not be exported again")
|
||||
|
||||
restarted = SessionRecordingPreparationManager(
|
||||
SessionRecordingMaterializer(private_root, exporter=forbidden_exporter)
|
||||
)
|
||||
try:
|
||||
restored = restarted.restore_published(command)
|
||||
|
||||
assert restored is not None
|
||||
assert restored.state == "ready"
|
||||
assert restored.recording is not None
|
||||
assert restored.recording.path.read_bytes() == b"durable-recording"
|
||||
assert calls == 1
|
||||
finally:
|
||||
restarted.close()
|
||||
|
||||
|
||||
def test_noncooperative_exporter_has_no_fake_heartbeat_and_restart_waits_for_old_writer(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user