SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(observatory): seal blocked run preparations

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-30 23:14:05 +03:00
parent 8d5aeb0533
commit 2a5763d3fb
12 changed files with 1935 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_m49_tgs_full_shadow.py
+108 -2
View File
@@ -3,10 +3,13 @@ from __future__ import annotations
import hashlib
import importlib.util
import json
import subprocess
import sys
import tarfile
from pathlib import Path
import numpy as np
import pytest
from k1link.laboratory.m49_tgs_full_shadow import (
M49TgsFullShadowResult,
@@ -34,6 +37,49 @@ ARTIFACT = importlib.util.module_from_spec(ARTIFACT_SPEC)
ARTIFACT_SPEC.loader.exec_module(ARTIFACT)
def _create_artifact_git_repository(root: Path) -> tuple[str, dict[Path, bytes]]:
subprocess.run(["git", "init", "-q", str(root)], check=True, capture_output=True)
subprocess.run(
["git", "config", "user.email", "mission-core-test@example.invalid"],
cwd=root,
check=True,
capture_output=True,
)
subprocess.run(
["git", "config", "user.name", "Mission Core Test"],
cwd=root,
check=True,
capture_output=True,
)
committed: dict[Path, bytes] = {}
for index, relative in enumerate(ARTIFACT.SOURCES):
payload = f"committed source {index}: {relative.as_posix()}\n".encode()
path = root / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(payload)
committed[relative] = payload
subprocess.run(
["git", "add", "--", *ARTIFACT.SOURCES],
cwd=root,
check=True,
capture_output=True,
)
subprocess.run(
["git", "commit", "-q", "-m", "fixture"],
cwd=root,
check=True,
capture_output=True,
)
revision = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=root,
check=True,
capture_output=True,
text=True,
).stdout.strip()
return revision, committed
def test_full_shadow_exact_multiset_and_costmap_priority() -> None:
native = np.asarray(
[[2.0, 0.0, 0.0, 0.0], [2.0, 0.0, 0.0, 0.0], [3.0, 0.0, 1.0, 0.0], [4.0, 0.0, 2.0, 0.0]],
@@ -47,11 +93,10 @@ def test_full_shadow_exact_multiset_and_costmap_priority() -> None:
def test_full_shadow_worker_artifact_is_deterministic_and_cpu_only(tmp_path: Path) -> None:
revision = "a" * 40
revision = ARTIFACT.git_revision()
first = ARTIFACT.build("m49-tgs-full-test", tmp_path / "one", revision=revision)
second = ARTIFACT.build("m49-tgs-full-test", tmp_path / "two", revision=revision)
assert first["sha256"] == second["sha256"]
import tarfile
with tarfile.open(first["artifact"], "r:gz") as archive:
release = archive.extractfile("payload/release.json")
@@ -64,6 +109,67 @@ def test_full_shadow_worker_artifact_is_deterministic_and_cpu_only(tmp_path: Pat
assert "gpu_requested = $false" in runner_text
def test_full_shadow_explicit_revision_ignores_current_file_drift(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
repository = tmp_path / "repository"
revision, committed = _create_artifact_git_repository(repository)
drifted_source = ARTIFACT.SOURCES[0]
(repository / drifted_source).write_bytes(b"dirty current-worktree replacement\n")
monkeypatch.setattr(ARTIFACT, "REPOSITORY_ROOT", repository)
explicit = ARTIFACT.build(
"m49-tgs-full-clean-revision",
tmp_path / "explicit",
revision=revision,
)
default_head = ARTIFACT.build("m49-tgs-full-clean-revision", tmp_path / "default")
assert explicit["code_revision"] == revision
assert default_head["code_revision"] == revision
assert explicit["sha256"] == default_head["sha256"]
with tarfile.open(explicit["artifact"], "r:gz") as archive:
source = archive.extractfile(f"payload/{drifted_source.name}")
release_stream = archive.extractfile("payload/release.json")
assert source is not None and release_stream is not None
assert source.read() == committed[drifted_source]
release = json.loads(release_stream.read())
assert release["code_revision"] == revision
assert release["files"][drifted_source.name]["sha256"] == hashlib.sha256(
committed[drifted_source]
).hexdigest()
@pytest.mark.parametrize("revision", ["HEAD", "../HEAD", "--all", "A" * 40])
def test_full_shadow_artifact_rejects_unsafe_or_symbolic_revision(
revision: str,
tmp_path: Path,
) -> None:
with pytest.raises(ARTIFACT.ArtifactBuildError, match="artifact revision is invalid"):
ARTIFACT.build("m49-tgs-full-invalid-revision", tmp_path, revision=revision)
def test_full_shadow_artifact_rejects_missing_revision(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
repository = tmp_path / "repository"
revision, _ = _create_artifact_git_repository(repository)
missing_revision = "f" * 40 if revision != "f" * 40 else "e" * 40
monkeypatch.setattr(ARTIFACT, "REPOSITORY_ROOT", repository)
with pytest.raises(
ARTIFACT.ArtifactBuildError,
match="artifact revision does not identify an existing commit",
):
ARTIFACT.build(
"m49-tgs-full-missing-revision",
tmp_path / "artifact",
revision=missing_revision,
)
def test_full_shadow_seal_binds_visual_and_semantic_timelines(tmp_path: Path) -> None:
source = tmp_path / "worker"
source.mkdir()
tests/test_observatory_run_preparations.py
+615
View File
@@ -0,0 +1,615 @@
from __future__ import annotations
import sqlite3
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from pathlib import Path
from threading import Barrier
from time import monotonic
from types import SimpleNamespace
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from k1link.observatory import (
MAX_RUN_PREPARATION_RECORDS,
RUN_PREPARATION_DATABASE_NAME,
LaboratorySetupRegistry,
ObservatoryRunPreparationCapacityError,
ObservatoryRunPreparationConflictError,
ObservatoryRunPreparationIntegrityError,
ObservatoryRunPreparationIntent,
ObservatoryRunPreparationLedger,
load_observatory_run_preparation_ledger,
)
from k1link.observatory import run_preparations as run_preparations_module
from k1link.sessions import SessionNotFoundError
from k1link.sessions.models import SessionSummary
from k1link.web.observatory_api import build_observatory_router
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-laboratory-setups.json"
SOURCE_SESSION_ID = "20260720T065719Z_viewer_live"
SOURCE_LABEL = "RAVNOVES00"
CURRENT_SOURCE_SESSION_ID = "20260828T130511Z_viewer_live"
SNAPSHOT_A = "a" * 64
SNAPSHOT_B = "b" * 64
def _registry() -> LaboratorySetupRegistry:
return LaboratorySetupRegistry.from_file(
REGISTRY_PATH,
repository_root=REPOSITORY_ROOT,
)
def _source(
session_id: str = SOURCE_SESSION_ID,
label: str = SOURCE_LABEL,
) -> SessionSummary:
return SessionSummary(
session_id=session_id,
display_name=label,
status="ready",
started_at_utc="2026-07-20T06:57:19Z",
completed_at_utc="2026-07-20T07:10:47Z",
duration_seconds=808.0,
modalities=("point-cloud", "trajectory", "video"),
source_count=3,
total_bytes=1,
replayable=True,
origin="recorded",
)
def _definition() -> dict[str, object]:
setup = _registry().catalog(_source())["setups"][0]
assert isinstance(setup, dict)
definition = setup["run_definition"]
assert isinstance(definition, dict)
return definition
def _intent(
*,
idempotency_key: str = "operator-request-001",
source_catalog_sha256: str = SNAPSHOT_A,
) -> ObservatoryRunPreparationIntent:
definition = _definition()
return ObservatoryRunPreparationIntent(
idempotency_key=idempotency_key,
source_session_id=SOURCE_SESSION_ID,
source_catalog_sha256=source_catalog_sha256,
setup_id="m49-tgs-full-shadow-v1",
definition_id=str(definition["definition_id"]),
definition_version=int(definition["version"]),
definition_sha256=str(definition["definition_sha256"]),
blocker_code="laboratory-runner-adapter-not-installed",
blocker_message=(
"Повторный запуск этого сетапа ещё не подключён к общему контуру расчёта."
),
)
class _ReadOnlySourceStore:
def __init__(self, source: SessionSummary | None = None) -> None:
resolved = source or _source()
self.sessions = {resolved.session_id: resolved}
self.snapshot_sha256 = SNAPSHOT_A
self.snapshot_reads = 0
def get_session(self, session_id: str):
try:
summary = self.sessions[session_id]
except KeyError as exc:
raise SessionNotFoundError(session_id) from exc
return SimpleNamespace(summary=summary)
def get_session_with_catalog_snapshot(self, session_id: str):
self.snapshot_reads += 1
return self.get_session(session_id), self.snapshot_sha256
def _client(
tmp_path: Path,
*,
store: _ReadOnlySourceStore | None = None,
) -> tuple[TestClient, ObservatoryRunPreparationLedger, _ReadOnlySourceStore]:
source_store = store or _ReadOnlySourceStore()
ledger = ObservatoryRunPreparationLedger(
tmp_path / "mission-core",
clock=lambda: "2026-08-30T20:00:00.000Z",
)
app = FastAPI()
app.include_router(
build_observatory_router(
source_store, # type: ignore[arg-type]
setup_registry=_registry(),
run_preparation_ledger=ledger,
)
)
return TestClient(app), ledger, source_store
def _request(
*,
idempotency_key: str = "operator-request-001",
definition_sha256: str | None = None,
) -> dict[str, object]:
definition = _definition()
return {
"schema_version": "missioncore.observatory-run-preparation-request/v1",
"idempotency_key": idempotency_key,
"source_session_id": SOURCE_SESSION_ID,
"setup_id": "m49-tgs-full-shadow-v1",
"definition_sha256": definition_sha256 or definition["definition_sha256"],
}
def _create_custom_ledger_schema(
data_dir: Path,
*,
idempotency_declaration: str,
extra_sql: str = "",
) -> None:
data_dir.mkdir(exist_ok=True)
with sqlite3.connect(data_dir / RUN_PREPARATION_DATABASE_NAME) as connection:
connection.executescript(
f"""
CREATE TABLE observatory_run_preparations (
preparation_id TEXT PRIMARY KEY,
idempotency_key {idempotency_declaration},
request_sha256 TEXT NOT NULL,
receipt_sha256 TEXT NOT NULL,
source_session_id TEXT NOT NULL,
source_catalog_sha256 TEXT NOT NULL,
setup_id TEXT NOT NULL,
definition_id TEXT NOT NULL,
definition_version INTEGER NOT NULL CHECK (definition_version > 0),
definition_sha256 TEXT NOT NULL,
state TEXT NOT NULL CHECK (state = 'blocked'),
blocker_code TEXT NOT NULL,
blocker_message TEXT NOT NULL,
created_at_utc TEXT NOT NULL
);
{extra_sql}
"""
)
def test_preparation_ledger_is_durable_and_exactly_idempotent(tmp_path: Path) -> None:
data_dir = tmp_path / "mission-core"
ledger = ObservatoryRunPreparationLedger(
data_dir,
clock=lambda: "2026-08-30T20:00:00.000Z",
)
first, first_created = ledger.prepare(_intent())
second, second_created = ledger.prepare(_intent())
restored = ObservatoryRunPreparationLedger(data_dir).get(first.preparation_id)
restored_by_key = ObservatoryRunPreparationLedger(data_dir).get_by_idempotency_key(
first.idempotency_key
)
assert first_created is True
assert second_created is False
assert second == first
assert restored == first
assert restored_by_key == first
assert first.state == "blocked"
assert first.as_dict()["run_id"] is None
assert first.as_dict()["dispatch_receipt"] is None
assert first.as_dict()["submission_allowed"] is False
assert first.as_dict()["receipt_sha256"] == first.receipt_sha256
with sqlite3.connect(data_dir / RUN_PREPARATION_DATABASE_NAME) as connection:
count = connection.execute(
"SELECT COUNT(*) FROM observatory_run_preparations"
).fetchone()[0]
assert count == 1
assert (data_dir / RUN_PREPARATION_DATABASE_NAME).stat().st_mode & 0o777 == 0o600
def test_preparation_idempotency_key_replays_original_resolved_source_identity(
tmp_path: Path,
) -> None:
ledger = ObservatoryRunPreparationLedger(tmp_path)
first, _created = ledger.prepare(_intent())
replayed, replay_created = ledger.prepare(
_intent(source_catalog_sha256=SNAPSHOT_B)
)
assert replay_created is False
assert replayed == first
assert replayed.source_catalog_sha256 == SNAPSHOT_A
with pytest.raises(ObservatoryRunPreparationConflictError, match="idempotency"):
ledger.prepare(replace(_intent(), definition_sha256="c" * 64))
def test_preparation_ledger_detects_stored_identity_drift(tmp_path: Path) -> None:
ledger = ObservatoryRunPreparationLedger(tmp_path)
record, _created = ledger.prepare(_intent())
with sqlite3.connect(ledger.database_path) as connection:
connection.execute(
"UPDATE observatory_run_preparations SET source_catalog_sha256 = ?",
(SNAPSHOT_B,),
)
connection.commit()
with pytest.raises(
ObservatoryRunPreparationIntegrityError,
match="receipt fingerprint",
):
ledger.get(record.preparation_id)
def test_preparation_ledger_rejects_an_incompatible_existing_schema(
tmp_path: Path,
) -> None:
tmp_path.mkdir(exist_ok=True)
with sqlite3.connect(tmp_path / RUN_PREPARATION_DATABASE_NAME) as connection:
connection.execute(
"CREATE TABLE observatory_run_preparations (preparation_id TEXT)"
)
with pytest.raises(
ObservatoryRunPreparationIntegrityError,
match="initialization|schema",
):
ObservatoryRunPreparationLedger(tmp_path)
def test_preparation_ledger_rejects_partial_idempotency_uniqueness(
tmp_path: Path,
) -> None:
_create_custom_ledger_schema(
tmp_path,
idempotency_declaration="TEXT NOT NULL",
extra_sql=(
"CREATE UNIQUE INDEX partial_idempotency "
"ON observatory_run_preparations(idempotency_key) "
"WHERE state = 'blocked';"
),
)
with pytest.raises(ObservatoryRunPreparationIntegrityError, match="uniqueness"):
ObservatoryRunPreparationLedger(tmp_path)
def test_preparation_ledger_rejects_non_binary_idempotency_collation(
tmp_path: Path,
) -> None:
_create_custom_ledger_schema(
tmp_path,
idempotency_declaration="TEXT COLLATE NOCASE NOT NULL UNIQUE",
)
with pytest.raises(ObservatoryRunPreparationIntegrityError, match="uniqueness"):
ObservatoryRunPreparationLedger(tmp_path)
def test_preparation_ledger_enforces_record_quota_but_keeps_retries_available(
tmp_path: Path,
) -> None:
ledger = ObservatoryRunPreparationLedger(tmp_path, max_records=1)
first, _created = ledger.prepare(_intent())
replayed, replay_created = ledger.prepare(_intent())
assert replay_created is False
assert replayed == first
with pytest.raises(ObservatoryRunPreparationCapacityError, match="quota"):
ledger.prepare(_intent(idempotency_key="operator-request-002"))
def test_preparation_ledger_concurrent_retry_commits_one_receipt(
tmp_path: Path,
) -> None:
first_ledger = ObservatoryRunPreparationLedger(tmp_path)
second_ledger = ObservatoryRunPreparationLedger(tmp_path)
barrier = Barrier(2)
def prepare(
ledger: ObservatoryRunPreparationLedger,
snapshot_sha256: str,
) -> tuple[str, str, bool]:
barrier.wait()
record, created = ledger.prepare(
_intent(source_catalog_sha256=snapshot_sha256)
)
return record.preparation_id, record.source_catalog_sha256, created
with ThreadPoolExecutor(max_workers=2) as executor:
first_future = executor.submit(prepare, first_ledger, SNAPSHOT_A)
second_future = executor.submit(prepare, second_ledger, SNAPSHOT_B)
first = first_future.result(timeout=10)
second = second_future.result(timeout=10)
assert first[0] == second[0]
assert first[1] == second[1]
assert sorted((first[2], second[2])) == [False, True]
with sqlite3.connect(first_ledger.database_path) as connection:
assert connection.execute(
"SELECT COUNT(*) FROM observatory_run_preparations"
).fetchone()[0] == 1
def test_preparation_ledger_startup_uses_only_bounded_admission_queries(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
statements: list[str] = []
connect = sqlite3.connect
def traced_connect(*args: object, **kwargs: object) -> sqlite3.Connection:
connection = connect(*args, **kwargs)
connection.set_trace_callback(statements.append)
return connection
monkeypatch.setattr(run_preparations_module.sqlite3, "connect", traced_connect)
ObservatoryRunPreparationLedger(tmp_path)
normalized = [statement.lower() for statement in statements]
assert not any("quick_check" in statement for statement in normalized)
quota_queries = [
statement
for statement in normalized
if "select 1 from observatory_run_preparations limit" in statement
]
assert len(quota_queries) == 1
assert str(MAX_RUN_PREPARATION_RECORDS + 1) in quota_queries[0]
def test_optional_ledger_loader_isolates_startup_from_ledger_corruption(
tmp_path: Path,
) -> None:
_create_custom_ledger_schema(
tmp_path,
idempotency_declaration="TEXT NOT NULL",
)
ledger, error = load_observatory_run_preparation_ledger(tmp_path)
assert ledger is None
assert error is not None
def test_optional_ledger_loader_has_a_short_database_lock_budget(
tmp_path: Path,
) -> None:
ledger = ObservatoryRunPreparationLedger(tmp_path)
locked = sqlite3.connect(ledger.database_path, timeout=0)
locked.execute("PRAGMA journal_mode = DELETE")
locked.execute("BEGIN EXCLUSIVE")
try:
started = monotonic()
loaded, error = load_observatory_run_preparation_ledger(tmp_path)
elapsed = monotonic() - started
finally:
locked.rollback()
locked.close()
assert loaded is None
assert error is not None
assert elapsed < 0.5
def test_preparation_endpoint_records_only_a_blocked_non_run_receipt(
tmp_path: Path,
) -> None:
client, ledger, source_store = _client(tmp_path)
response = client.post(
"/api/v1/observatory/run-preparations",
json=_request(),
)
assert response.status_code == 200
document = response.json()
assert document == ledger.get(document["preparation_id"]).as_dict()
assert document["schema_version"] == "missioncore.observatory-run-preparation/v1"
assert document["run_id"] is None
assert document["state"] == "blocked"
assert document["preconditions_passed"] is True
assert document["submission_allowed"] is False
assert document["dispatch_receipt"] is None
assert document["source"] == {
"session_id": SOURCE_SESSION_ID,
"catalog_sha256": SNAPSHOT_A,
}
assert document["setup"]["definition"]["definition_sha256"] == (
_definition()["definition_sha256"]
)
assert document["blocker"]["reason_code"] == (
"laboratory-runner-adapter-not-installed"
)
assert document["authority"] == {
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
"production_accepted": False,
}
assert source_store.snapshot_reads == 1
assert client.post("/api/v1/observatory/runs", json={}).status_code == 404
def test_preparation_endpoint_duplicate_returns_the_same_durable_receipt(
tmp_path: Path,
) -> None:
client, _ledger, _source_store = _client(tmp_path)
first = client.post("/api/v1/observatory/run-preparations", json=_request())
second = client.post("/api/v1/observatory/run-preparations", json=_request())
restored = client.get(
f"/api/v1/observatory/run-preparations/{first.json()['preparation_id']}"
)
assert first.status_code == second.status_code == restored.status_code == 200
assert second.json() == first.json()
assert restored.json() == first.json()
def test_preparation_endpoint_exact_retry_precedes_mutable_source_and_catalog_checks(
tmp_path: Path,
) -> None:
client, _ledger, source_store = _client(tmp_path)
first = client.post(
"/api/v1/observatory/run-preparations",
json=_request(),
)
assert first.status_code == 200
source_store.snapshot_sha256 = SNAPSHOT_B
source_store.sessions.clear()
replayed = client.post(
"/api/v1/observatory/run-preparations",
json=_request(),
)
assert replayed.status_code == 200
assert replayed.json() == first.json()
assert replayed.json()["source"]["catalog_sha256"] == SNAPSHOT_A
assert source_store.snapshot_reads == 1
def test_preparation_endpoint_new_key_captures_a_new_source_snapshot(
tmp_path: Path,
) -> None:
client, _ledger, source_store = _client(tmp_path)
first = client.post(
"/api/v1/observatory/run-preparations",
json=_request(),
)
assert first.status_code == 200
source_store.snapshot_sha256 = SNAPSHOT_B
second = client.post(
"/api/v1/observatory/run-preparations",
json=_request(idempotency_key="operator-request-002"),
)
assert second.status_code == 200
assert second.json()["preparation_id"] != first.json()["preparation_id"]
assert second.json()["source"]["catalog_sha256"] == SNAPSHOT_B
def test_preparation_endpoint_same_key_rejects_a_changed_client_payload_early(
tmp_path: Path,
) -> None:
client, _ledger, source_store = _client(tmp_path)
assert client.post(
"/api/v1/observatory/run-preparations",
json=_request(),
).status_code == 200
source_store.sessions.clear()
conflict = client.post(
"/api/v1/observatory/run-preparations",
json=_request(definition_sha256="f" * 64),
)
assert conflict.status_code == 409
assert source_store.snapshot_reads == 1
def test_preparation_endpoint_maps_early_ledger_capacity_failure_to_503(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, _ledger, source_store = _client(tmp_path)
monkeypatch.setattr(
run_preparations_module,
"MAX_RUN_PREPARATION_STORAGE_BYTES",
0,
)
response = client.post(
"/api/v1/observatory/run-preparations",
json=_request(),
)
assert response.status_code == 503
assert source_store.snapshot_reads == 0
def test_preparation_endpoint_rejects_stale_incompatible_and_non_executable_inputs(
tmp_path: Path,
) -> None:
client, _ledger, source_store = _client(tmp_path)
stale = client.post(
"/api/v1/observatory/run-preparations",
json=_request(definition_sha256="f" * 64),
)
assert stale.status_code == 409
source_store.sessions[SOURCE_SESSION_ID] = replace(
_source(),
replayable=False,
)
incompatible = client.post(
"/api/v1/observatory/run-preparations",
json=_request(idempotency_key="operator-request-002"),
)
assert incompatible.status_code == 409
current_store = _ReadOnlySourceStore(
_source(CURRENT_SOURCE_SESSION_ID, "RAVNOVES004TREE")
)
current_client, _current_ledger, _ = _client(tmp_path / "current", store=current_store)
no_definition = current_client.post(
"/api/v1/observatory/run-preparations",
json={
"schema_version": "missioncore.observatory-run-preparation-request/v1",
"idempotency_key": "operator-request-003",
"source_session_id": CURRENT_SOURCE_SESSION_ID,
"setup_id": "lab-v1-ravnoves004tree-final",
"definition_sha256": "f" * 64,
},
)
assert no_definition.status_code == 409
@pytest.mark.parametrize(
"request_patch",
[
{"idempotency_key": "../escape"},
{"source_session_id": "../source"},
{"setup_id": "../setup"},
{"definition_sha256": "not-a-digest"},
],
)
def test_preparation_endpoint_rejects_unsafe_request_fields(
tmp_path: Path,
request_patch: dict[str, object],
) -> None:
client, _ledger, _source_store = _client(tmp_path)
document = {**_request(), **request_patch}
assert client.post(
"/api/v1/observatory/run-preparations",
json=document,
).status_code == 422
def test_preparation_endpoint_fails_closed_when_ledger_is_unavailable() -> None:
app = FastAPI()
app.include_router(
build_observatory_router(
_ReadOnlySourceStore(), # type: ignore[arg-type]
setup_registry=_registry(),
run_preparation_ledger_error="database corrupt",
)
)
client = TestClient(app)
assert client.post(
"/api/v1/observatory/run-preparations",
json=_request(),
).status_code == 503
assert client.get(
"/api/v1/observatory/run-preparations/"
"observatory-prep-00000000000000000000000000000000"
).status_code == 503