feat(observatory): ship modular AI inference labs
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,45 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.observatory.composition_runs import CompositionRunStore
|
||||
from k1link.observatory.modular_composition import COMPOSITION_SCHEMA, ModuleRegistry
|
||||
|
||||
|
||||
def test_operator_projection_can_be_renamed_and_hidden_without_changing_run(tmp_path: Path) -> None:
|
||||
registry = ModuleRegistry.from_file(
|
||||
Path(__file__).parents[1] / "config" / "observatory-ai-modules.json"
|
||||
)
|
||||
module = next(item for item in registry.modules if item.module_id == "tgs")
|
||||
composition = registry.compose(
|
||||
{
|
||||
"schema_version": COMPOSITION_SCHEMA,
|
||||
"selections": [
|
||||
{
|
||||
"group": module.group,
|
||||
"module_id": module.module_id,
|
||||
"module_sha256": module.sha256,
|
||||
"parameters": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
store = CompositionRunStore(tmp_path / "runs")
|
||||
run = store.save(
|
||||
source_session_id="source-1",
|
||||
composition=composition,
|
||||
setup_ids=("setup-1",),
|
||||
job_ids=("job-1",),
|
||||
idempotency_key="run-1",
|
||||
created_at_utc="2026-09-04T12:00:00.000Z",
|
||||
)
|
||||
|
||||
assert store.display_name(run.run_id) is None
|
||||
assert store.list(source_session_id="source-1", include_hidden=False) == (run,)
|
||||
assert store.rename_projection(run.run_id, " Маршрут у школы ") == "Маршрут у школы"
|
||||
assert store.display_name(run.run_id) == "Маршрут у школы"
|
||||
|
||||
store.delete_projection(run.run_id)
|
||||
assert store.list(source_session_id="source-1", include_hidden=False) == ()
|
||||
assert store.list(source_session_id="source-1") == (run,)
|
||||
assert store.get(run.run_id) == run
|
||||
@@ -0,0 +1,174 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from test_lidar_replay import _capture
|
||||
|
||||
from k1link.compute import lidar_replay
|
||||
from k1link.compute.lidar_preparation import prepare_lidar_replay_pack_v2
|
||||
from k1link.compute.lidar_replay import LidarReplayError
|
||||
|
||||
|
||||
def test_warm_preparation_reuses_legacy_identity_without_source_decode(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
source = _capture(tmp_path)
|
||||
producer = Path(lidar_replay.__file__)
|
||||
producer_digest = hashlib.sha256(producer.read_bytes()).hexdigest()
|
||||
original = lidar_replay.build_lidar_replay_pack_v2(source, tmp_path / "packs")
|
||||
manifest_before = (original / "manifest.json").read_bytes()
|
||||
closed = []
|
||||
original_close = lidar_replay.LidarReplayPackV2.close
|
||||
|
||||
def no_decode(*args: object, **kwargs: object) -> None:
|
||||
pytest.fail("cache hit must not decode the original capture")
|
||||
|
||||
def close(pack: lidar_replay.LidarReplayPackV2) -> None:
|
||||
original_close(pack)
|
||||
closed.append(not pack.arrays._arrays)
|
||||
|
||||
monkeypatch.setattr(lidar_replay, "_capture_arrays", no_decode)
|
||||
monkeypatch.setattr(lidar_replay.LidarReplayPackV2, "close", close)
|
||||
assert prepare_lidar_replay_pack_v2(source, tmp_path / "packs") == original
|
||||
assert (original / "manifest.json").read_bytes() == manifest_before
|
||||
assert json.loads(manifest_before)["identity"]["producer_sha256"] == producer_digest
|
||||
assert hashlib.sha256(producer.read_bytes()).hexdigest() == producer_digest
|
||||
assert closed == [True]
|
||||
|
||||
|
||||
def test_cold_preparation_calls_existing_builder_once(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
source = _capture(tmp_path)
|
||||
calls = []
|
||||
original = lidar_replay._capture_arrays
|
||||
|
||||
def decode(path: Path):
|
||||
calls.append(path)
|
||||
return original(path)
|
||||
|
||||
monkeypatch.setattr(lidar_replay, "_capture_arrays", decode)
|
||||
result = prepare_lidar_replay_pack_v2(source, tmp_path / "packs")
|
||||
assert result.is_dir()
|
||||
assert calls == [source]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("changed", ["metadata", "origin", "remove-origin", "raw", "session"])
|
||||
def test_changed_source_never_reuses_previous_pack(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
changed: str,
|
||||
) -> None:
|
||||
source = _capture(tmp_path)
|
||||
output = tmp_path / "packs"
|
||||
old = prepare_lidar_replay_pack_v2(source, output)
|
||||
if changed == "metadata":
|
||||
path = source.with_name("mqtt.metadata.jsonl")
|
||||
path.write_bytes(path.read_bytes().replace(b"5010000000", b"5009000000"))
|
||||
elif changed == "origin":
|
||||
path = source.with_name("mqtt.timeline.origin.json")
|
||||
path.write_bytes(path.read_bytes().replace(b"5000000000", b"4999999999"))
|
||||
elif changed == "remove-origin":
|
||||
source.with_name("mqtt.timeline.origin.json").unlink()
|
||||
elif changed == "raw":
|
||||
# The cache boundary checks bytes before trying to decode invalid data.
|
||||
source.write_bytes(source.read_bytes()[:-1] + b"x")
|
||||
called = []
|
||||
|
||||
def build(*args: object, **kwargs: object) -> Path:
|
||||
called.append((args, kwargs))
|
||||
return output / "new-test-pack"
|
||||
|
||||
monkeypatch.setattr(lidar_replay, "build_lidar_replay_pack_v2", build)
|
||||
result = prepare_lidar_replay_pack_v2(
|
||||
source,
|
||||
output,
|
||||
session_id="other-session" if changed == "session" else None,
|
||||
)
|
||||
assert result != old
|
||||
assert len(called) == 1
|
||||
assert old.is_dir()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"artifact", ["lidar-replay.npz", "quality-report.json", "equivalence-report.json"]
|
||||
)
|
||||
def test_corrupt_matching_pack_fails_without_rebuild_or_overwrite(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
artifact: str,
|
||||
) -> None:
|
||||
source = _capture(tmp_path)
|
||||
root = prepare_lidar_replay_pack_v2(source, tmp_path / "packs")
|
||||
path = root / artifact
|
||||
damaged = b"damaged-evidence"
|
||||
path.write_bytes(damaged)
|
||||
|
||||
def forbidden(*args: object, **kwargs: object) -> None:
|
||||
pytest.fail("must preserve corrupt evidence instead of overwriting")
|
||||
|
||||
monkeypatch.setattr(lidar_replay, "build_lidar_replay_pack_v2", forbidden)
|
||||
with pytest.raises(LidarReplayError, match="artifact identity changed"):
|
||||
prepare_lidar_replay_pack_v2(source, tmp_path / "packs")
|
||||
assert path.read_bytes() == damaged
|
||||
|
||||
|
||||
def test_report_threshold_collision_is_explicit_and_releases_arrays(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
source = _capture(tmp_path)
|
||||
root = prepare_lidar_replay_pack_v2(source, tmp_path / "packs")
|
||||
closed = []
|
||||
original = lidar_replay.LidarReplayPackV2.close
|
||||
|
||||
def close(pack: lidar_replay.LidarReplayPackV2) -> None:
|
||||
original(pack)
|
||||
closed.append(not pack.arrays._arrays)
|
||||
|
||||
monkeypatch.setattr(lidar_replay.LidarReplayPackV2, "close", close)
|
||||
with pytest.raises(LidarReplayError, match="threshold differs"):
|
||||
prepare_lidar_replay_pack_v2(source, tmp_path / "packs", pose_coverage_threshold_ms=50)
|
||||
assert closed == [True]
|
||||
assert root.is_dir()
|
||||
|
||||
|
||||
def test_source_mutation_during_cache_validation_is_rejected(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
source = _capture(tmp_path)
|
||||
prepare_lidar_replay_pack_v2(source, tmp_path / "packs")
|
||||
original = lidar_replay.LidarReplayPackV2.__init__
|
||||
|
||||
def changed(pack: lidar_replay.LidarReplayPackV2, root: Path) -> None:
|
||||
original(pack, root)
|
||||
source.with_name("mqtt.metadata.jsonl").touch()
|
||||
|
||||
monkeypatch.setattr(lidar_replay.LidarReplayPackV2, "__init__", changed)
|
||||
with pytest.raises(LidarReplayError, match="changed during preparation"):
|
||||
prepare_lidar_replay_pack_v2(source, tmp_path / "packs")
|
||||
|
||||
|
||||
def test_missing_metadata_never_uses_cache(tmp_path: Path) -> None:
|
||||
source = _capture(tmp_path)
|
||||
prepare_lidar_replay_pack_v2(source, tmp_path / "packs")
|
||||
source.with_name("mqtt.metadata.jsonl").unlink()
|
||||
with pytest.raises(LidarReplayError, match="exact host timing"):
|
||||
prepare_lidar_replay_pack_v2(source, tmp_path / "packs")
|
||||
|
||||
|
||||
def test_symlink_manifest_is_not_followed(tmp_path: Path) -> None:
|
||||
source = _capture(tmp_path)
|
||||
root = prepare_lidar_replay_pack_v2(source, tmp_path / "packs")
|
||||
manifest = root / "manifest.json"
|
||||
saved = root / "saved-manifest.json"
|
||||
manifest.rename(saved)
|
||||
manifest.symlink_to(saved)
|
||||
with pytest.raises(LidarReplayError, match="manifest cannot be a symlink"):
|
||||
prepare_lidar_replay_pack_v2(source, tmp_path / "packs")
|
||||
@@ -707,7 +707,7 @@ def test_fixed_worker_stage_consumer_requires_manifested_metadata_member(
|
||||
built_from.append(capture)
|
||||
return worker_root
|
||||
|
||||
monkeypatch.setattr(source_module, "build_lidar_replay_pack_v2", fake_build)
|
||||
monkeypatch.setattr(source_module, "prepare_lidar_replay_pack_v2", fake_build)
|
||||
|
||||
class _DeliveredSource:
|
||||
def materialize(self, requested: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage:
|
||||
|
||||
@@ -0,0 +1,508 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from k1link.observatory.composition_runs import CompositionRunStore
|
||||
from k1link.observatory.lab_view_profiles import PROFILE_SCHEMA, LabViewProfileStore
|
||||
from k1link.observatory.modular_composition import COMPOSITION_SCHEMA, ModuleRegistry
|
||||
from k1link.observatory.modular_composition_store import ModularCompositionStore
|
||||
from k1link.observatory.source_admission import PortableSourceNotPreparedError
|
||||
from k1link.web.modular_observatory_api import build_modular_observatory_router
|
||||
|
||||
|
||||
def _client(tmp_path: Path) -> tuple[TestClient, dict]:
|
||||
registry = ModuleRegistry.from_file(
|
||||
Path(__file__).parents[1] / "config" / "observatory-ai-modules.json"
|
||||
)
|
||||
|
||||
class Store:
|
||||
def get_session(self, session_id: str) -> object:
|
||||
return SimpleNamespace(summary=SimpleNamespace(session_id=session_id, lab=None))
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_modular_observatory_router(
|
||||
store=Store(), # type: ignore[arg-type]
|
||||
compositions=ModularCompositionStore(tmp_path / "compositions", registry),
|
||||
view_profiles=LabViewProfileStore(tmp_path / "view-profiles"),
|
||||
)
|
||||
)
|
||||
catalog = TestClient(app).get("/api/v1/observatory/ai-module-catalog").json()
|
||||
return TestClient(app), catalog
|
||||
|
||||
|
||||
def test_composition_projection_can_be_renamed_and_removed_through_api(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry = ModuleRegistry.from_file(
|
||||
Path(__file__).parents[1] / "config" / "observatory-ai-modules.json"
|
||||
)
|
||||
tgs = next(module for module in registry.modules if module.module_id == "tgs")
|
||||
composition = registry.compose(
|
||||
{
|
||||
"schema_version": COMPOSITION_SCHEMA,
|
||||
"selections": [
|
||||
{
|
||||
"group": tgs.group,
|
||||
"module_id": tgs.module_id,
|
||||
"module_sha256": tgs.sha256,
|
||||
"parameters": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
runs = CompositionRunStore(tmp_path / "runs")
|
||||
run = runs.save(
|
||||
source_session_id="source-1",
|
||||
composition=composition,
|
||||
setup_ids=("setup-1",),
|
||||
job_ids=("job-1",),
|
||||
idempotency_key="run-1",
|
||||
created_at_utc="2026-09-04T12:00:00.000Z",
|
||||
)
|
||||
|
||||
class Store:
|
||||
def get_session(self, session_id: str) -> object:
|
||||
return SimpleNamespace(summary=SimpleNamespace(session_id=session_id, lab=None))
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_modular_observatory_router(
|
||||
store=Store(), # type: ignore[arg-type]
|
||||
compositions=ModularCompositionStore(tmp_path / "compositions", registry),
|
||||
composition_runs=runs,
|
||||
)
|
||||
)
|
||||
client = TestClient(app)
|
||||
renamed = client.patch(
|
||||
f"/api/v1/observatory/ai-composition-runs/{run.run_id}",
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-ai-composition-run-rename/v1",
|
||||
"display_name": " Контрольный прогон ",
|
||||
},
|
||||
)
|
||||
assert renamed.status_code == 200
|
||||
assert renamed.json() == {
|
||||
"schema_version": "missioncore.observatory-ai-composition-run-projection/v1",
|
||||
"run_id": run.run_id,
|
||||
"display_name": "Контрольный прогон",
|
||||
}
|
||||
assert runs.display_name(run.run_id) == "Контрольный прогон"
|
||||
|
||||
removed = client.delete(f"/api/v1/observatory/ai-composition-runs/{run.run_id}")
|
||||
assert removed.status_code == 204
|
||||
assert runs.list(source_session_id="source-1", include_hidden=False) == ()
|
||||
assert runs.get(run.run_id) == run
|
||||
|
||||
|
||||
def test_lab_view_profile_is_saved_against_the_exact_result(tmp_path: Path) -> None:
|
||||
client, _catalog = _client(tmp_path)
|
||||
result_id = "modular-composition-result-123"
|
||||
endpoint = f"/api/v1/observatory/lab-view-profiles/{result_id}"
|
||||
assert client.get(endpoint).status_code == 404
|
||||
|
||||
document = {
|
||||
"schema_version": PROFILE_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"scene_settings": {
|
||||
"point_size": 128.5,
|
||||
"accumulation_seconds": 50_000,
|
||||
"color_mode": "height",
|
||||
"palette": "viridis",
|
||||
"show_grid": False,
|
||||
"show_labels": True,
|
||||
"show_camera_frustums": False,
|
||||
},
|
||||
}
|
||||
saved = client.put(endpoint, json=document)
|
||||
assert saved.status_code == 200
|
||||
assert saved.json()["scene_settings"] == document["scene_settings"]
|
||||
assert saved.json()["updated_at_utc"].endswith("Z")
|
||||
assert client.get(endpoint).json() == saved.json()
|
||||
|
||||
foreign = {**document, "result_id": "another-result"}
|
||||
rejected = client.put(endpoint, json=foreign)
|
||||
assert rejected.status_code == 422
|
||||
assert rejected.json()["detail"] == "Профиль отображения относится к другой LAB."
|
||||
|
||||
|
||||
def test_catalog_and_idempotent_independent_ddrnet_composition(tmp_path: Path) -> None:
|
||||
client, catalog = _client(tmp_path)
|
||||
assert catalog["schema_version"] == "missioncore.observatory-ai-module-catalog/v1"
|
||||
segmenters = next(group for group in catalog["groups"] if group["group"] == "segmentation")
|
||||
assert [module["docker_name"] for module in segmenters["modules"]] == [
|
||||
"ndc-mission-core-ai-module-ddrnet",
|
||||
"ndc-mission-core-ai-module-eomt",
|
||||
]
|
||||
geometry = next(group for group in catalog["groups"] if group["group"] == "geometry")
|
||||
assert [module["docker_name"] for module in geometry["modules"]] == [
|
||||
"ndc-mission-core-ai-module-tgs"
|
||||
]
|
||||
detection = next(group for group in catalog["groups"] if group["group"] == "detection")
|
||||
assert [module["docker_name"] for module in detection["modules"]] == [
|
||||
"ndc-mission-core-ai-module-rf-detr"
|
||||
]
|
||||
ranges = next(group for group in catalog["groups"] if group["group"] == "range")
|
||||
assert [module["docker_name"] for module in ranges["modules"]] == [
|
||||
"ndc-mission-core-ai-module-object-distance"
|
||||
]
|
||||
ddrnet = segmenters["modules"][0]
|
||||
request = {
|
||||
"schema_version": COMPOSITION_SCHEMA,
|
||||
"source_session_id": "source-1",
|
||||
"idempotency_key": "ai-layer:source-1:ddrnet",
|
||||
"selections": [
|
||||
{
|
||||
"group": "segmentation",
|
||||
"module_id": "ddrnet",
|
||||
"module_sha256": ddrnet["module_sha256"],
|
||||
"parameters": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
first = client.post("/api/v1/observatory/ai-compositions", json=request)
|
||||
second = client.post("/api/v1/observatory/ai-compositions", json=request)
|
||||
assert first.status_code == second.status_code == 200
|
||||
assert first.json()["created"] is True and second.json()["created"] is False
|
||||
assert first.json()["composition_sha256"] == second.json()["composition_sha256"]
|
||||
assert first.json()["composition"]["outputs"] == ["segmentation.mask"]
|
||||
stored = list((tmp_path / "compositions").glob("*.json"))
|
||||
assert len(stored) == 1
|
||||
assert json.loads(stored[0].read_bytes())["execution"]["max_parallel_nodes"] == 1
|
||||
|
||||
|
||||
def test_server_rejects_two_segmentation_providers(tmp_path: Path) -> None:
|
||||
client, catalog = _client(tmp_path)
|
||||
modules = next(group for group in catalog["groups"] if group["group"] == "segmentation")[
|
||||
"modules"
|
||||
]
|
||||
response = client.post(
|
||||
"/api/v1/observatory/ai-compositions",
|
||||
json={
|
||||
"schema_version": COMPOSITION_SCHEMA,
|
||||
"source_session_id": "source-1",
|
||||
"idempotency_key": "ai-layer:source-1:both",
|
||||
"selections": [
|
||||
{
|
||||
"group": "segmentation",
|
||||
"module_id": module["module_id"],
|
||||
"module_sha256": module["module_sha256"],
|
||||
"parameters": {},
|
||||
}
|
||||
for module in modules
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 409
|
||||
assert response.json()["detail"] == "В одном слое можно выбрать только один AI-модуль."
|
||||
|
||||
|
||||
def test_tgs_browser_numeric_default_is_accepted_and_bad_value_is_explained(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, catalog = _client(tmp_path)
|
||||
tgs = next(
|
||||
module
|
||||
for group in catalog["groups"]
|
||||
for module in group["modules"]
|
||||
if module["module_id"] == "tgs"
|
||||
)
|
||||
|
||||
def request(value: int) -> dict:
|
||||
return {
|
||||
"schema_version": COMPOSITION_SCHEMA,
|
||||
"source_session_id": "source-1",
|
||||
"idempotency_key": f"ai-layer:source-1:tgs:{value}",
|
||||
"selections": [
|
||||
{
|
||||
"group": "geometry",
|
||||
"module_id": "tgs",
|
||||
"module_sha256": tgs["module_sha256"],
|
||||
"parameters": {"history-seconds": value},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
accepted = client.post("/api/v1/observatory/ai-compositions", json=request(1))
|
||||
assert accepted.status_code == 200
|
||||
|
||||
rejected = client.post("/api/v1/observatory/ai-compositions", json=request(2))
|
||||
assert rejected.status_code == 409
|
||||
assert rejected.json()["detail"] == (
|
||||
"Параметры выбранного AI-модуля устарели. "
|
||||
"Закройте окно, откройте его снова и повторите расчёт."
|
||||
)
|
||||
|
||||
|
||||
def test_tgs_and_object_distance_dispatch_as_two_independent_worker_jobs(tmp_path: Path) -> None:
|
||||
registry = ModuleRegistry.from_file(
|
||||
Path(__file__).parents[1] / "config" / "observatory-ai-modules.json"
|
||||
)
|
||||
|
||||
class Store:
|
||||
def get_session(self, session_id: str) -> object:
|
||||
return SimpleNamespace(summary=SimpleNamespace(session_id=session_id, lab=None))
|
||||
|
||||
class Definitions:
|
||||
def resolve_setup(self, setup_id: str) -> object:
|
||||
return SimpleNamespace(definition_sha256=(setup_id.encode().hex() + "0" * 64)[:64])
|
||||
|
||||
submitted: list[str] = []
|
||||
|
||||
class Binding:
|
||||
def check(self, **kwargs: object) -> object:
|
||||
return SimpleNamespace(check_sha256="c" * 64)
|
||||
|
||||
def submit(self, **kwargs: object) -> tuple[object, bool]:
|
||||
setup_id = str(kwargs["setup_id"])
|
||||
submitted.append(setup_id)
|
||||
return SimpleNamespace(as_dict=lambda: {"setup_id": setup_id}), True
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_modular_observatory_router(
|
||||
store=Store(), # type: ignore[arg-type]
|
||||
compositions=ModularCompositionStore(tmp_path / "compositions", registry),
|
||||
definitions=Definitions(), # type: ignore[arg-type]
|
||||
binding=Binding(), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
catalog = TestClient(app).get("/api/v1/observatory/ai-module-catalog").json()
|
||||
by_id = {
|
||||
module["module_id"]: {**module, "group": group["group"]}
|
||||
for group in catalog["groups"]
|
||||
for module in group["modules"]
|
||||
}
|
||||
response = TestClient(app).post(
|
||||
"/api/v1/observatory/ai-compositions",
|
||||
json={
|
||||
"schema_version": COMPOSITION_SCHEMA,
|
||||
"source_session_id": "source-1",
|
||||
"idempotency_key": "ai-layer:source-1:tgs-and-range",
|
||||
"selections": [
|
||||
{
|
||||
"group": by_id[module_id]["group"],
|
||||
"module_id": module_id,
|
||||
"module_sha256": by_id[module_id]["module_sha256"],
|
||||
"parameters": {},
|
||||
}
|
||||
for module_id in ("tgs", "rf-detr", "object-distance")
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["dispatch"]["ready"] is True
|
||||
assert submitted == ["m49-tgs-portable-v2", "ai-range-object-distance-v1"]
|
||||
assert response.json()["dispatch"]["setup_ids"] == submitted
|
||||
|
||||
|
||||
def test_existing_record_module_is_rejected_before_a_second_worker_submission(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry = ModuleRegistry.from_file(
|
||||
Path(__file__).parents[1] / "config" / "observatory-ai-modules.json"
|
||||
)
|
||||
tgs = next(module for module in registry.modules if module.module_id == "tgs")
|
||||
|
||||
class Store:
|
||||
def get_session(self, session_id: str) -> object:
|
||||
return SimpleNamespace(summary=SimpleNamespace(session_id=session_id, lab=None))
|
||||
|
||||
class Definitions:
|
||||
def resolve_setup(self, setup_id: str) -> object:
|
||||
return SimpleNamespace(definition_sha256="d" * 64)
|
||||
|
||||
class Binding:
|
||||
def check(self, **kwargs: object) -> object:
|
||||
raise AssertionError("duplicate must be rejected before source preparation")
|
||||
|
||||
def submit(self, **kwargs: object) -> tuple[object, bool]:
|
||||
raise AssertionError("duplicate must not reach Worker submission")
|
||||
|
||||
class Queue:
|
||||
def list_jobs(self, **kwargs: object) -> list[object]:
|
||||
assert kwargs == {
|
||||
"source_session_id": "source-1",
|
||||
"setup_id": "m49-tgs-portable-v2",
|
||||
"definition_sha256": "d" * 64,
|
||||
"limit": 20,
|
||||
}
|
||||
return [
|
||||
SimpleNamespace(
|
||||
job_id="already-calculated",
|
||||
state="succeeded",
|
||||
)
|
||||
]
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_modular_observatory_router(
|
||||
store=Store(), # type: ignore[arg-type]
|
||||
compositions=ModularCompositionStore(tmp_path / "compositions", registry),
|
||||
definitions=Definitions(), # type: ignore[arg-type]
|
||||
binding=Binding(), # type: ignore[arg-type]
|
||||
queue=Queue(), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
response = TestClient(app).post(
|
||||
"/api/v1/observatory/ai-compositions",
|
||||
json={
|
||||
"schema_version": COMPOSITION_SCHEMA,
|
||||
"source_session_id": "source-1",
|
||||
"idempotency_key": "ai-layer:source-1:tgs-again",
|
||||
"selections": [
|
||||
{
|
||||
"group": "geometry",
|
||||
"module_id": "tgs",
|
||||
"module_sha256": tgs.sha256,
|
||||
"parameters": {},
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 409
|
||||
assert response.json()["detail"] == (
|
||||
"Эта конфигурация уже рассчитана или поставлена в очередь. Выберите другую конфигурацию."
|
||||
)
|
||||
|
||||
|
||||
def test_existing_module_is_reused_inside_a_new_composition(tmp_path: Path) -> None:
|
||||
registry = ModuleRegistry.from_file(
|
||||
Path(__file__).parents[1] / "config" / "observatory-ai-modules.json"
|
||||
)
|
||||
|
||||
class Store:
|
||||
def get_session(self, session_id: str) -> object:
|
||||
return SimpleNamespace(summary=SimpleNamespace(session_id=session_id, lab=None))
|
||||
|
||||
class Definitions:
|
||||
def resolve_setup(self, setup_id: str) -> object:
|
||||
return SimpleNamespace(definition_sha256=(setup_id.encode().hex() + "0" * 64)[:64])
|
||||
|
||||
existing_tgs = SimpleNamespace(
|
||||
job_id="existing-tgs",
|
||||
setup_id="m49-tgs-portable-v2",
|
||||
state="succeeded",
|
||||
as_dict=lambda: {"setup_id": "m49-tgs-portable-v2", "reused": True},
|
||||
)
|
||||
submitted: list[str] = []
|
||||
|
||||
class Queue:
|
||||
def list_jobs(self, **kwargs: object) -> list[object]:
|
||||
if kwargs["setup_id"] == "m49-tgs-portable-v2":
|
||||
return [existing_tgs]
|
||||
return []
|
||||
|
||||
class Binding:
|
||||
def check(self, **kwargs: object) -> object:
|
||||
assert kwargs["setup_id"] == "ai-detection-rf-detr-v1"
|
||||
return SimpleNamespace(check_sha256="c" * 64)
|
||||
|
||||
def submit(self, **kwargs: object) -> tuple[object, bool]:
|
||||
setup_id = str(kwargs["setup_id"])
|
||||
submitted.append(setup_id)
|
||||
return SimpleNamespace(
|
||||
setup_id=setup_id,
|
||||
as_dict=lambda: {"setup_id": setup_id, "reused": False},
|
||||
), True
|
||||
|
||||
modules = {item.module_id: item for item in registry.modules}
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_modular_observatory_router(
|
||||
store=Store(), # type: ignore[arg-type]
|
||||
compositions=ModularCompositionStore(tmp_path / "compositions", registry),
|
||||
definitions=Definitions(), # type: ignore[arg-type]
|
||||
binding=Binding(), # type: ignore[arg-type]
|
||||
queue=Queue(), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
response = TestClient(app).post(
|
||||
"/api/v1/observatory/ai-compositions",
|
||||
json={
|
||||
"schema_version": COMPOSITION_SCHEMA,
|
||||
"source_session_id": "source-1",
|
||||
"idempotency_key": "ai-layer:source-1:tgs-and-rf-detr",
|
||||
"selections": [
|
||||
{
|
||||
"group": modules[module_id].group,
|
||||
"module_id": module_id,
|
||||
"module_sha256": modules[module_id].sha256,
|
||||
"parameters": {},
|
||||
}
|
||||
for module_id in ("tgs", "rf-detr")
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert submitted == ["ai-detection-rf-detr-v1"]
|
||||
assert response.json()["dispatch"]["setup_ids"] == [
|
||||
"m49-tgs-portable-v2",
|
||||
"ai-detection-rf-detr-v1",
|
||||
]
|
||||
assert [job["reused"] for job in response.json()["dispatch"]["jobs"]] == [True, False]
|
||||
assert "готовые результаты использованы повторно" in response.json()["dispatch"]["reason"]
|
||||
|
||||
|
||||
def test_composition_prepares_an_unopened_recording_before_submit(tmp_path: Path) -> None:
|
||||
registry = ModuleRegistry.from_file(
|
||||
Path(__file__).parents[1] / "config" / "observatory-ai-modules.json"
|
||||
)
|
||||
detector = next(module for module in registry.modules if module.module_id == "rf-detr")
|
||||
|
||||
class Store:
|
||||
def get_session(self, session_id: str) -> object:
|
||||
return SimpleNamespace(summary=SimpleNamespace(session_id=session_id, lab=None))
|
||||
|
||||
class Definitions:
|
||||
def resolve_setup(self, setup_id: str) -> object:
|
||||
return SimpleNamespace(definition_sha256="d" * 64)
|
||||
|
||||
calls: list[str] = []
|
||||
|
||||
class Binding:
|
||||
def check(self, **kwargs: object) -> object:
|
||||
calls.append("check")
|
||||
raise PortableSourceNotPreparedError("camera manifest is not prepared")
|
||||
|
||||
def prepare_check(self, **kwargs: object) -> object:
|
||||
calls.append("prepare-check")
|
||||
return SimpleNamespace(check_sha256="c" * 64)
|
||||
|
||||
def submit(self, **kwargs: object) -> tuple[object, bool]:
|
||||
calls.append("submit")
|
||||
return SimpleNamespace(as_dict=lambda: {"setup_id": kwargs["setup_id"]}), True
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_modular_observatory_router(
|
||||
store=Store(), # type: ignore[arg-type]
|
||||
compositions=ModularCompositionStore(tmp_path / "compositions", registry),
|
||||
definitions=Definitions(), # type: ignore[arg-type]
|
||||
binding=Binding(), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
response = TestClient(app).post(
|
||||
"/api/v1/observatory/ai-compositions",
|
||||
json={
|
||||
"schema_version": COMPOSITION_SCHEMA,
|
||||
"source_session_id": "source-1",
|
||||
"idempotency_key": "ai-layer:source-1:prepare-camera",
|
||||
"selections": [
|
||||
{
|
||||
"group": "detection",
|
||||
"module_id": "rf-detr",
|
||||
"module_sha256": detector.sha256,
|
||||
"parameters": {},
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["dispatch"]["ready"] is True
|
||||
assert calls == ["check", "prepare-check", "submit"]
|
||||
@@ -0,0 +1,221 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.modular_composition import (
|
||||
COMPOSITION_SCHEMA,
|
||||
CompositionError,
|
||||
ModuleRegistry,
|
||||
ModuleSpec,
|
||||
node_input_identity,
|
||||
)
|
||||
from k1link.observatory.modular_node_cache import ModularNodeCache
|
||||
|
||||
|
||||
def _digest(value: str) -> str:
|
||||
return hashlib.sha256(value.encode()).hexdigest()
|
||||
|
||||
|
||||
def _module(
|
||||
module_id: str,
|
||||
group: str,
|
||||
requires: tuple[str, ...],
|
||||
provides: tuple[str, ...],
|
||||
*,
|
||||
optional_inputs: tuple[str, ...] = (),
|
||||
state_policy: str = "stateless",
|
||||
) -> ModuleSpec:
|
||||
return ModuleSpec(
|
||||
module_id=module_id,
|
||||
label=module_id,
|
||||
group=group,
|
||||
image_sha256=_digest(module_id + "-image"),
|
||||
implementation_sha256=_digest(module_id + "-code"),
|
||||
model_sha256=(
|
||||
_digest(module_id + "-model") if group in {"segmentation", "detection"} else None
|
||||
),
|
||||
contract_sha256=_digest(module_id + "-contract"),
|
||||
requires=requires,
|
||||
provides=provides,
|
||||
optional_inputs=optional_inputs,
|
||||
parameter_choices_json=json.dumps({"cadence": [1, 2]}).encode(),
|
||||
defaults_json=json.dumps({"cadence": 1}).encode(),
|
||||
state_policy=state_policy,
|
||||
)
|
||||
|
||||
|
||||
def _registry() -> tuple[ModuleRegistry, ModuleSpec, ModuleSpec]:
|
||||
camera = _module("camera-source", "preparation", ("source.camera",), ("camera.frames",))
|
||||
ddrnet = _module("ddrnet", "segmentation", ("camera.frames",), ("segmentation.mask",))
|
||||
eomt = _module("eomt", "segmentation", ("camera.frames",), ("segmentation.mask",))
|
||||
detector = _module("rf-detr", "detection", ("camera.frames",), ("detections.2d",))
|
||||
distance = _module(
|
||||
"object-distance",
|
||||
"range",
|
||||
("detections.2d", "source.calibration", "source.lidar", "source.pose"),
|
||||
("objects.ranged",),
|
||||
)
|
||||
motion = _module(
|
||||
"object-motion",
|
||||
"motion",
|
||||
("objects.ranged",),
|
||||
("objects.motion",),
|
||||
state_policy="causal-reset-at-source-start",
|
||||
)
|
||||
policy = _module(
|
||||
"policy-shadow",
|
||||
"policy",
|
||||
("objects.motion",),
|
||||
("policy.observation",),
|
||||
optional_inputs=("segmentation.mask",),
|
||||
)
|
||||
return ModuleRegistry((camera, ddrnet, eomt, detector, distance, motion, policy)), ddrnet, eomt
|
||||
|
||||
|
||||
def _selection(*modules: ModuleSpec) -> dict:
|
||||
return {
|
||||
"schema_version": COMPOSITION_SCHEMA,
|
||||
"selections": [
|
||||
{
|
||||
"group": module.group,
|
||||
"module_id": module.module_id,
|
||||
"module_sha256": module.sha256,
|
||||
"parameters": {},
|
||||
}
|
||||
for module in modules
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_ddrnet_and_eomt_are_independent_alternative_compositions() -> None:
|
||||
registry, ddrnet, eomt = _registry()
|
||||
ddr = registry.compose(_selection(ddrnet))
|
||||
eom = registry.compose(_selection(eomt))
|
||||
assert [node.module.module_id for node in ddr.nodes] == ["camera-source", "ddrnet"]
|
||||
assert [node.module.module_id for node in eom.nodes] == ["camera-source", "eomt"]
|
||||
assert ddr.sha256 != eom.sha256
|
||||
assert ddr.outputs == eom.outputs == ("segmentation.mask",)
|
||||
assert ddrnet.docker_name == "ndc-mission-core-ai-module-ddrnet"
|
||||
assert eomt.docker_name == "ndc-mission-core-ai-module-eomt"
|
||||
|
||||
|
||||
def test_installed_tgs_is_an_independent_point_cloud_composition() -> None:
|
||||
registry = ModuleRegistry.from_file(
|
||||
Path(__file__).parents[1] / "config" / "observatory-ai-modules.json"
|
||||
)
|
||||
tgs = next(module for module in registry.modules if module.module_id == "tgs")
|
||||
composition = registry.compose(_selection(tgs))
|
||||
|
||||
assert [node.module.module_id for node in composition.nodes] == ["tgs"]
|
||||
assert composition.source_capabilities == (
|
||||
"source.calibration",
|
||||
"source.lidar",
|
||||
"source.pose",
|
||||
)
|
||||
assert composition.outputs == ("geometry.costmap", "geometry.ground")
|
||||
assert tgs.docker_name == "ndc-mission-core-ai-module-tgs"
|
||||
|
||||
|
||||
def test_installed_detection_range_and_tgs_keep_explicit_dependencies() -> None:
|
||||
registry = ModuleRegistry.from_file(
|
||||
Path(__file__).parents[1] / "config" / "observatory-ai-modules.json"
|
||||
)
|
||||
modules = {module.module_id: module for module in registry.modules}
|
||||
detector = registry.compose(_selection(modules["rf-detr"]))
|
||||
assert [node.module.module_id for node in detector.nodes] == ["camera-source", "rf-detr"]
|
||||
assert modules["rf-detr"].docker_name == "ndc-mission-core-ai-module-rf-detr"
|
||||
with pytest.raises(CompositionError, match="select a module providing detections.2d"):
|
||||
registry.compose(_selection(modules["object-distance"]))
|
||||
ranged = registry.compose(_selection(modules["rf-detr"], modules["object-distance"]))
|
||||
assert [node.module.module_id for node in ranged.nodes] == [
|
||||
"camera-source",
|
||||
"rf-detr",
|
||||
"object-distance",
|
||||
]
|
||||
assert modules["object-distance"].docker_name == ("ndc-mission-core-ai-module-object-distance")
|
||||
combined = registry.compose(
|
||||
_selection(modules["tgs"], modules["rf-detr"], modules["object-distance"])
|
||||
)
|
||||
assert [node.module.module_id for node in combined.nodes] == [
|
||||
"camera-source",
|
||||
"tgs",
|
||||
"rf-detr",
|
||||
"object-distance",
|
||||
]
|
||||
assert combined.source_capabilities == (
|
||||
"source.calibration",
|
||||
"source.camera",
|
||||
"source.lidar",
|
||||
"source.pose",
|
||||
)
|
||||
|
||||
|
||||
def test_two_segmenters_and_hidden_analytical_dependency_are_rejected() -> None:
|
||||
registry, ddrnet, eomt = _registry()
|
||||
with pytest.raises(CompositionError, match="only one provider"):
|
||||
registry.compose(_selection(ddrnet, eomt))
|
||||
policy = next(module for module in registry.modules if module.module_id == "policy-shadow")
|
||||
with pytest.raises(CompositionError, match="select a module providing objects.motion"):
|
||||
registry.compose(_selection(policy))
|
||||
|
||||
|
||||
def test_detector_distance_motion_policy_graph_is_topological_without_segmentation() -> None:
|
||||
registry, _, _ = _registry()
|
||||
chosen = [
|
||||
module
|
||||
for module in registry.modules
|
||||
if module.module_id in {"rf-detr", "object-distance", "object-motion", "policy-shadow"}
|
||||
]
|
||||
graph = registry.compose(_selection(*chosen))
|
||||
assert [node.module.module_id for node in graph.nodes] == [
|
||||
"camera-source",
|
||||
"rf-detr",
|
||||
"object-distance",
|
||||
"object-motion",
|
||||
"policy-shadow",
|
||||
]
|
||||
assert "source.lidar" in graph.source_capabilities
|
||||
assert "segmentation.mask" not in dict(graph.nodes[-1].inputs)
|
||||
|
||||
|
||||
def test_node_identity_ignores_other_composition_nodes_but_binds_temporal_state() -> None:
|
||||
registry, ddrnet, _ = _registry()
|
||||
graph = registry.compose(_selection(ddrnet))
|
||||
node = graph.nodes[-1]
|
||||
identity = node_input_identity(node, {"camera.frames": _digest("prepared-camera")})
|
||||
assert "job_id" not in json.dumps(identity)
|
||||
assert "composition" not in json.dumps(identity)
|
||||
assert identity == node_input_identity(node, {"camera.frames": _digest("prepared-camera")})
|
||||
motion = next(module for module in registry.modules if module.module_id == "object-motion")
|
||||
motion_graph = registry.compose(
|
||||
_selection(
|
||||
*(
|
||||
module
|
||||
for module in registry.modules
|
||||
if module.module_id in {"rf-detr", "object-distance", "object-motion"}
|
||||
)
|
||||
)
|
||||
)
|
||||
motion_node = next(node for node in motion_graph.nodes if node.module is motion)
|
||||
with pytest.raises(CompositionError, match="immutable digest"):
|
||||
node_input_identity(motion_node, {"objects.ranged": _digest("ranges")})
|
||||
|
||||
|
||||
def test_node_cache_reuses_exact_bytes_and_treats_corruption_as_miss(tmp_path: Path) -> None:
|
||||
output = tmp_path / "output"
|
||||
output.mkdir()
|
||||
(output / "result.json").write_text('{"ok":true}', encoding="utf-8")
|
||||
identity = {"schema_version": "test/v1", "input": _digest("source")}
|
||||
cache = ModularNodeCache(tmp_path / "cache")
|
||||
sealed = cache.seal(identity, output, metadata={"computed": True})
|
||||
hit = cache.lookup(identity)
|
||||
assert hit is not None and hit.result_sha256 == sealed.result_sha256
|
||||
assert hit.root.joinpath("result.json").read_text() == '{"ok":true}'
|
||||
hit.root.joinpath("result.json").chmod(0o644)
|
||||
hit.root.joinpath("result.json").write_text("damaged")
|
||||
assert cache.lookup(identity) is None
|
||||
assert output.joinpath("result.json").read_text() == '{"ok":true}'
|
||||
@@ -0,0 +1,84 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.domain_ontology import (
|
||||
ObservatoryDomainOntology,
|
||||
ObservatoryOntologyError,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def test_full_observatory_composition_projects_exact_label_and_pane_layers() -> None:
|
||||
ontology = ObservatoryDomainOntology.from_file(
|
||||
REPOSITORY_ROOT / "config" / "observatory-domain-ontology.json"
|
||||
)
|
||||
|
||||
projection = ontology.project_module_ids(("object-distance", "tgs", "rf-detr", "ddrnet"))
|
||||
|
||||
assert projection["configuration_label"] == (
|
||||
"DDRNet-39 · GOOSE · RF-DETR Large · TRAVEL TGS · Дистанция до объектов · K1 LiDAR"
|
||||
)
|
||||
assert [layer["layer_id"] for layer in projection["viewer_layers"]] == [
|
||||
"camera.source",
|
||||
"camera.ddrnet",
|
||||
"camera.detections",
|
||||
"spatial.source-points",
|
||||
"spatial.local-slam",
|
||||
"spatial.tgs",
|
||||
]
|
||||
entity_ids = {item["id"] for item in ontology.document["entities"]}
|
||||
relation_ids = {item["id"] for item in ontology.document["relations"]}
|
||||
assert {
|
||||
"mission.transport-unit",
|
||||
"mission.equipment-unit",
|
||||
"observatory.equipment-mount",
|
||||
"observatory.capture-profile",
|
||||
"observatory.recorded-session",
|
||||
"observatory.module-version",
|
||||
"observatory.container-image",
|
||||
"observatory.worker-node",
|
||||
"observatory.lab-view-profile",
|
||||
} <= entity_ids
|
||||
assert {
|
||||
"observatory.recorded-session.captured_on_transport",
|
||||
"observatory.recorded-session.captured_with_equipment",
|
||||
"observatory.recorded-session.uses_equipment_mount",
|
||||
"observatory.recorded-session.uses_capture_profile",
|
||||
"observatory.composition-run.uses_recorded_session",
|
||||
"observatory.module-version.implemented_by_image",
|
||||
"observatory.container-image.installed_on_worker",
|
||||
"observatory.recorded-job.executes_on_worker",
|
||||
"observatory.lab-projection.has_view_profile",
|
||||
} <= relation_ids
|
||||
|
||||
|
||||
def test_observatory_ontology_rejects_unprojected_module() -> None:
|
||||
ontology = ObservatoryDomainOntology.from_file(
|
||||
REPOSITORY_ROOT / "config" / "observatory-domain-ontology.json"
|
||||
)
|
||||
|
||||
with pytest.raises(ObservatoryOntologyError, match="has no ontology projection"):
|
||||
ontology.project_module_ids(("unknown-module",))
|
||||
|
||||
|
||||
def test_single_modules_project_only_the_viewports_they_own() -> None:
|
||||
ontology = ObservatoryDomainOntology.from_file(
|
||||
REPOSITORY_ROOT / "config" / "observatory-domain-ontology.json"
|
||||
)
|
||||
|
||||
ddrnet = ontology.project_module_ids(("ddrnet",))
|
||||
assert [layer["layer_id"] for layer in ddrnet["viewer_layers"]] == [
|
||||
"camera.source",
|
||||
"camera.ddrnet",
|
||||
]
|
||||
assert {layer["pane_id"] for layer in ddrnet["viewer_layers"]} == {"camera"}
|
||||
|
||||
tgs = ontology.project_module_ids(("tgs",))
|
||||
assert [layer["layer_id"] for layer in tgs["viewer_layers"]] == [
|
||||
"spatial.source-points",
|
||||
"spatial.local-slam",
|
||||
"spatial.tgs",
|
||||
]
|
||||
assert {layer["pane_id"] for layer in tgs["viewer_layers"]} == {"spatial"}
|
||||
@@ -46,9 +46,7 @@ def test_k1_equipment_and_capture_profiles_have_exact_content_identities() -> No
|
||||
assert capture.equipment == equipment
|
||||
for definition in _definition_registry().definitions:
|
||||
assert (
|
||||
registry.compatible_profile_for_requirements(
|
||||
definition.source_requirements.as_dict()
|
||||
)
|
||||
registry.compatible_profile_for_requirements(definition.source_requirements.as_dict())
|
||||
== capture
|
||||
)
|
||||
|
||||
@@ -111,12 +109,10 @@ def test_other_equipment_is_blocked_before_any_profile_probe() -> None:
|
||||
|
||||
catalog = projector.catalog(source)
|
||||
assert calls == []
|
||||
assert len(catalog["setups"]) == 2
|
||||
assert len(catalog["setups"]) == 6
|
||||
for setup in catalog["setups"]:
|
||||
assert setup["source_compatibility"]["compatible"] is False
|
||||
assert setup["source_compatibility"]["reason_code"] == (
|
||||
"equipment-model-mismatch"
|
||||
)
|
||||
assert setup["source_compatibility"]["reason_code"] == ("equipment-model-mismatch")
|
||||
assert setup["preflight"]["submission_allowed"] is False
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
"""No model/runtime side effects in installer admission tests."""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from test_observatory_source_reuse_install import FakeEngine as SourceEngine
|
||||
|
||||
SCRIPTS = Path(__file__).parents[1] / "experiments/perception/worker/observatory_portable"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def installer(monkeypatch):
|
||||
monkeypatch.syspath_prepend(str(SCRIPTS))
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"recorded_heartbeat_installer", SCRIPTS / "install_recorded_heartbeat_recovery.py"
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, spec.name, module)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class Engine(SourceEngine):
|
||||
busy = False
|
||||
|
||||
def execute_json(self, name, source):
|
||||
if source == self.installer.probe():
|
||||
return self.installer.BEFORE
|
||||
readiness = super().execute_json(name, source)
|
||||
if self.busy:
|
||||
readiness["recorded_jobs_by_state"]["reconciliation-required"] = 1
|
||||
return readiness
|
||||
|
||||
|
||||
def test_two_file_payload_is_hashed_and_cannot_gain_paths(installer, tmp_path):
|
||||
root = tmp_path / "payload"
|
||||
installer.pack(Path(__file__).parents[1], root)
|
||||
assert set(installer.payload_files(root)) == {"worker_agent.py", "worker_http_transport.py"}
|
||||
path = root / "payload.json"
|
||||
document = json.loads(path.read_bytes())
|
||||
document["files"]["../outside.py"] = "0" * 64
|
||||
path.write_text(json.dumps(document))
|
||||
with pytest.raises(ValueError, match="file set changed"):
|
||||
installer.payload_files(root)
|
||||
|
||||
|
||||
def test_plan_and_exact_cutover_fence_preserve_compute_and_resources(installer, tmp_path):
|
||||
root = tmp_path / "payload"
|
||||
installer.pack(Path(__file__).parents[1], root)
|
||||
engine = Engine(installer)
|
||||
plan = installer.plan(engine, root)
|
||||
assert plan["compute_packages_changed"] is False
|
||||
with pytest.raises(ValueError, match="plan changed"):
|
||||
installer.apply(engine, root, "0" * 64, tmp_path / "release")
|
||||
assert engine.requests == [] and not (tmp_path / "release").exists()
|
||||
target = plan["targets"][0]
|
||||
installer.fence(engine, target)
|
||||
engine.rows[target["name"]]["HostConfig"]["Memory"] = 999
|
||||
with pytest.raises(ValueError, match="changed since plan"):
|
||||
installer.fence(engine, target)
|
||||
|
||||
|
||||
def test_quarantined_owner_blocks_agent_replacement(installer, tmp_path):
|
||||
root = tmp_path / "payload"
|
||||
installer.pack(Path(__file__).parents[1], root)
|
||||
engine = Engine(installer)
|
||||
engine.busy = True
|
||||
with pytest.raises(ValueError, match="not idle"):
|
||||
installer.plan(engine, root)
|
||||
assert engine.requests == []
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Transient control-plane recovery never re-executes a model or changes owner."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from threading import Event, Thread
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from test_observatory_worker_agent import (
|
||||
BlockingExecutor,
|
||||
FakeTransport,
|
||||
_enqueue,
|
||||
_heartbeat_agent,
|
||||
_identity,
|
||||
_queue,
|
||||
)
|
||||
from test_observatory_worker_http_transport import (
|
||||
BEARER_TOKEN,
|
||||
CLAIM_TOKEN,
|
||||
JOB_ID,
|
||||
_cache_claim,
|
||||
_claim_response,
|
||||
)
|
||||
|
||||
from k1link.observatory.worker_agent import (
|
||||
ObservatoryWorkerClaimRejectedError,
|
||||
ObservatoryWorkerTransientTransportError,
|
||||
_ClaimHeartbeat,
|
||||
_RecordedJobPayload,
|
||||
_seal_job,
|
||||
)
|
||||
from k1link.observatory.worker_http_transport import (
|
||||
ObservatoryWorkerHttpError,
|
||||
ObservatoryWorkerHttpGateway,
|
||||
ObservatoryWorkerHttpTransientError,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lost_ack", [False, True])
|
||||
def test_transient_loss_retries_exact_sequence_without_rerunning_compute(tmp_path, lost_ack):
|
||||
queue = _queue(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
attempts = []
|
||||
accepted = Event()
|
||||
|
||||
class Transport(FakeTransport):
|
||||
def renew_claim(self, **kwargs):
|
||||
attempts.append(kwargs.copy())
|
||||
if len(attempts) == 1:
|
||||
if lost_ack:
|
||||
super().renew_claim(**kwargs)
|
||||
raise ObservatoryWorkerTransientTransportError("synthetic network loss")
|
||||
result = super().renew_claim(**kwargs)
|
||||
accepted.set()
|
||||
return result
|
||||
|
||||
entered, release = Event(), Event()
|
||||
transport = Transport(queue)
|
||||
agent = _heartbeat_agent(transport, BlockingExecutor(entered=entered, release=release))
|
||||
reports = []
|
||||
thread = Thread(target=lambda: reports.append(agent.run_once()))
|
||||
thread.start()
|
||||
try:
|
||||
assert entered.wait(2) and accepted.wait(2)
|
||||
finally:
|
||||
release.set()
|
||||
thread.join(2)
|
||||
assert not thread.is_alive()
|
||||
assert reports[0].state == "succeeded"
|
||||
assert attempts[0] == attempts[1]
|
||||
assert transport.starts == [job_id]
|
||||
assert len(transport.successes) == 1
|
||||
assert queue.get(job_id).claim_generation == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["expired", "forbidden", "stopping", "late-ack"])
|
||||
def test_uncertain_lease_never_becomes_success(tmp_path, failure):
|
||||
queue = _queue(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
transport = FakeTransport(queue)
|
||||
claim = transport.claim_next(
|
||||
claimant_id="worker-006",
|
||||
claim_request_id="heartbeat-budget",
|
||||
supported_executor_identities=(_identity(),),
|
||||
)
|
||||
started = transport.start(
|
||||
claimant_id="worker-006",
|
||||
job_id=job_id,
|
||||
claim_token=claim["claim_token"],
|
||||
)
|
||||
now, calls = [0.0], []
|
||||
heartbeat = _ClaimHeartbeat(
|
||||
transport=transport,
|
||||
job=_seal_job(_RecordedJobPayload.model_validate(started)),
|
||||
claim_token=claim["claim_token"],
|
||||
interval_seconds=0.01,
|
||||
stop_timeout_seconds=1.0,
|
||||
clock=lambda: now[0],
|
||||
)
|
||||
|
||||
class FailingTransport:
|
||||
def renew_claim(self, **kwargs):
|
||||
calls.append(kwargs)
|
||||
if failure == "forbidden":
|
||||
raise ObservatoryWorkerClaimRejectedError("changed generation")
|
||||
if failure == "stopping":
|
||||
heartbeat._stop.set()
|
||||
else:
|
||||
now[0] += 3601
|
||||
if failure == "late-ack":
|
||||
return transport.renew_claim(**kwargs)
|
||||
raise ObservatoryWorkerTransientTransportError("timeout")
|
||||
|
||||
heartbeat._transport = FailingTransport()
|
||||
heartbeat._run()
|
||||
assert heartbeat.failed
|
||||
assert len(calls) == 1
|
||||
assert transport.successes == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", [408, 429, 500, 502, 503, 504, "timeout", 401, 403, 409, 302])
|
||||
def test_http_renewal_classifies_retryable_failures_and_bounds_io(tmp_path: Path, status):
|
||||
attempts = []
|
||||
|
||||
def handle(request):
|
||||
if not request.url.path.endswith("/lease/renew"):
|
||||
return _claim_response()
|
||||
attempts.append(json.loads(request.content))
|
||||
assert request.extensions["timeout"] == dict.fromkeys(
|
||||
("connect", "read", "write", "pool"),
|
||||
5.0,
|
||||
)
|
||||
if status == "timeout":
|
||||
raise httpx.ReadTimeout("fixture", request=request)
|
||||
return httpx.Response(status, json={})
|
||||
|
||||
with ObservatoryWorkerHttpGateway(
|
||||
base_url="http://127.0.0.1:8000",
|
||||
bearer_token=BEARER_TOKEN,
|
||||
work_root=tmp_path,
|
||||
transport=httpx.MockTransport(handle),
|
||||
) as gateway:
|
||||
_cache_claim(gateway)
|
||||
with pytest.raises(ObservatoryWorkerHttpError) as error:
|
||||
gateway.renew_claim(
|
||||
claimant_id="worker-006",
|
||||
job_id=JOB_ID,
|
||||
claim_token=CLAIM_TOKEN,
|
||||
claim_generation=1,
|
||||
heartbeat_sequence=1,
|
||||
)
|
||||
assert isinstance(error.value, ObservatoryWorkerHttpTransientError) == (
|
||||
status in {408, 429, 500, 502, 503, 504, "timeout"}
|
||||
)
|
||||
# The transport itself must not replay uploads or other mutations.
|
||||
assert len(attempts) == 1
|
||||
@@ -258,8 +258,7 @@ def test_generic_fixed_stack_runs_topologically_and_returns_verified_package(
|
||||
]
|
||||
writer_launch = launches[-1]
|
||||
assert any(
|
||||
mount.container_path == "/missioncore/input/steps/compute-step"
|
||||
and mount.read_only
|
||||
mount.container_path == "/missioncore/input/steps/compute-step" and mount.read_only
|
||||
for mount in writer_launch.mounts
|
||||
)
|
||||
assert draft.result_id == "portable-result-generic-runner"
|
||||
@@ -268,6 +267,16 @@ def test_generic_fixed_stack_runs_topologically_and_returns_verified_package(
|
||||
for launch in launches:
|
||||
writable = [mount for mount in launch.mounts if not mount.read_only]
|
||||
assert [mount.container_path for mount in writable] == ["/missioncore/output"]
|
||||
plan_mount = next(
|
||||
mount for mount in launch.mounts if mount.container_path == "/missioncore/plan"
|
||||
)
|
||||
assert plan_mount.read_only
|
||||
plan_root = Path(plan_mount.engine_path)
|
||||
assert plan_root.is_dir()
|
||||
assert (
|
||||
json.loads((plan_root / "run-plan.json").read_bytes())["runtime_plan"]["job_id"]
|
||||
== plan.job_id
|
||||
)
|
||||
|
||||
|
||||
def test_generic_docker_launcher_uses_hardened_one_shot_contract() -> None:
|
||||
@@ -298,10 +307,10 @@ def test_generic_docker_launcher_uses_hardened_one_shot_contract() -> None:
|
||||
package_id=package.package_id,
|
||||
container=package.containers[0],
|
||||
mounts=(
|
||||
InstalledLabDockerMount("/engine/plan.json", "/missioncore/input/run-plan.json", True),
|
||||
InstalledLabDockerMount("/engine/source", "/missioncore/input/source", True),
|
||||
InstalledLabDockerMount("/engine/output", "/missioncore/output", False),
|
||||
InstalledLabDockerMount("/engine/asset", "/missioncore/package/assets/runner", True),
|
||||
InstalledLabDockerMount("/engine/plan", "/missioncore/plan", True),
|
||||
),
|
||||
labels={
|
||||
"com.nodedc.authority": "observation-only",
|
||||
@@ -309,8 +318,10 @@ def test_generic_docker_launcher_uses_hardened_one_shot_contract() -> None:
|
||||
"com.nodedc.definition-sha256": package.definition_sha256,
|
||||
"com.nodedc.job-id": f"observatory-run-{'1' * 32}",
|
||||
"com.nodedc.managed-by": "mission-core-worker",
|
||||
"com.nodedc.module-id": package.containers[0].container_id,
|
||||
"com.nodedc.package-sha256": package.package_sha256,
|
||||
"com.nodedc.product": "mission-core",
|
||||
"com.nodedc.role": "ai-module",
|
||||
"com.nodedc.stack": "observatory",
|
||||
},
|
||||
name_token="0123456789abcdef",
|
||||
@@ -324,6 +335,9 @@ def test_generic_docker_launcher_uses_hardened_one_shot_contract() -> None:
|
||||
"POST",
|
||||
"DELETE",
|
||||
]
|
||||
assert requests[1].url.params["name"] == (
|
||||
f"ndc-mission-core-ai-module-{package.containers[0].container_id}-0123456789abcdef"
|
||||
)
|
||||
assert create_document["Image"] == f"sha256:{package.containers[0].image_sha256}"
|
||||
assert create_document["NetworkDisabled"] is True
|
||||
host = cast(dict[str, object], create_document["HostConfig"])
|
||||
|
||||
@@ -104,7 +104,7 @@ def test_installed_package_binds_exact_ready_definition_and_runtime() -> None:
|
||||
assert document["container_io"] == {
|
||||
"schema_version": "missioncore.observatory-installed-lab-container-io/v2",
|
||||
"source_root": "/missioncore/input/source",
|
||||
"plan_path": "/missioncore/input/run-plan.json",
|
||||
"plan_path": "/missioncore/plan/run-plan.json",
|
||||
"step_input_root": "/missioncore/input/steps",
|
||||
"result_root": "/missioncore/output",
|
||||
"work_root": "/missioncore/work",
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.lab_view_profiles import (
|
||||
PROFILE_SCHEMA,
|
||||
LabSceneProfile,
|
||||
LabViewProfile,
|
||||
LabViewProfileError,
|
||||
LabViewProfileStore,
|
||||
)
|
||||
|
||||
|
||||
def _profile(result_id: str = "result-1") -> LabViewProfile:
|
||||
return LabViewProfile(
|
||||
result_id=result_id,
|
||||
scene_settings=LabSceneProfile(
|
||||
point_size=4.7,
|
||||
accumulation_seconds=8,
|
||||
color_mode="height",
|
||||
palette="viridis",
|
||||
show_grid=False,
|
||||
show_labels=True,
|
||||
show_camera_frustums=False,
|
||||
),
|
||||
updated_at_utc="2026-09-04T09:30:00.000Z",
|
||||
)
|
||||
|
||||
|
||||
def test_profile_store_round_trips_one_mutable_profile_per_result(tmp_path: Path) -> None:
|
||||
store = LabViewProfileStore(tmp_path / "profiles")
|
||||
assert store.get("result-1") is None
|
||||
|
||||
saved = store.save(_profile())
|
||||
assert store.get("result-1") == saved
|
||||
documents = list(store.root.glob("*.json"))
|
||||
assert len(documents) == 1
|
||||
assert documents[0].stat().st_mode & 0o777 == 0o600
|
||||
assert json.loads(documents[0].read_text())["schema_version"] == PROFILE_SCHEMA
|
||||
|
||||
updated = LabViewProfile(
|
||||
result_id="result-1",
|
||||
scene_settings=LabSceneProfile(
|
||||
point_size=2.0,
|
||||
accumulation_seconds=3,
|
||||
color_mode="distance",
|
||||
palette="turbo",
|
||||
show_grid=True,
|
||||
show_labels=False,
|
||||
show_camera_frustums=True,
|
||||
),
|
||||
updated_at_utc="2026-09-04T09:31:00.000Z",
|
||||
)
|
||||
store.save(updated)
|
||||
assert store.get("result-1") == updated
|
||||
assert len(list(store.root.glob("*.json"))) == 1
|
||||
|
||||
|
||||
def test_profile_store_preserves_unbounded_operator_values(tmp_path: Path) -> None:
|
||||
profile = LabViewProfile(
|
||||
result_id="result-extreme",
|
||||
scene_settings=LabSceneProfile(
|
||||
point_size=128.5,
|
||||
accumulation_seconds=50_000,
|
||||
color_mode="height",
|
||||
palette="turbo",
|
||||
show_grid=True,
|
||||
show_labels=False,
|
||||
show_camera_frustums=False,
|
||||
),
|
||||
updated_at_utc="2026-09-04T09:32:00.000Z",
|
||||
)
|
||||
store = LabViewProfileStore(tmp_path / "profiles")
|
||||
store.save(profile)
|
||||
assert store.get("result-extreme") == profile
|
||||
|
||||
|
||||
def test_profile_store_rejects_invalid_identity_and_tampered_document(tmp_path: Path) -> None:
|
||||
store = LabViewProfileStore(tmp_path / "profiles")
|
||||
with pytest.raises(LabViewProfileError, match="identity"):
|
||||
store.get("../foreign")
|
||||
|
||||
store.save(_profile())
|
||||
[document] = store.root.glob("*.json")
|
||||
payload = json.loads(document.read_text())
|
||||
payload["result_id"] = "different-result"
|
||||
document.write_text(json.dumps(payload), encoding="utf-8")
|
||||
with pytest.raises(LabViewProfileError, match="identity"):
|
||||
store.get("result-1")
|
||||
@@ -0,0 +1,103 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import runpy
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
_MODULE = runpy.run_path(
|
||||
str(
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "experiments/perception/worker/observatory_portable/run_ai_module_object_distance.py"
|
||||
)
|
||||
)
|
||||
_aligned_camera_seconds = _MODULE["_aligned_camera_seconds"]
|
||||
ObjectDistanceModuleError = _MODULE["ObjectDistanceModuleError"]
|
||||
_projection = _MODULE["_projection"]
|
||||
_LidarPack = _MODULE["_LidarPack"]
|
||||
|
||||
|
||||
def test_distance_joins_repaired_camera_and_lidar_by_frame_index() -> None:
|
||||
seconds = _aligned_camera_seconds(
|
||||
{"session_seconds": 136.002},
|
||||
{"frame_index": 1299, "session_seconds": 175.131263458},
|
||||
frame_index=1299,
|
||||
previous_camera_seconds=175.0,
|
||||
)
|
||||
assert seconds == 175.131263458
|
||||
|
||||
|
||||
def test_distance_rejects_frame_or_repaired_clock_regression() -> None:
|
||||
with pytest.raises(ObjectDistanceModuleError, match="frame identities"):
|
||||
_aligned_camera_seconds(
|
||||
{"session_seconds": 136.002},
|
||||
{"frame_index": 1298, "session_seconds": 175.131263458},
|
||||
frame_index=1299,
|
||||
previous_camera_seconds=175.0,
|
||||
)
|
||||
|
||||
|
||||
def test_distance_reads_calibration_from_full_e10_pack(tmp_path: Path) -> None:
|
||||
pack = tmp_path / "lidar-pack.npz"
|
||||
np.savez(
|
||||
pack,
|
||||
intrinsic_fx_fy_cx_cy=np.asarray([300.0, 301.0, 400.0, 300.0]),
|
||||
distortion_kb4=np.asarray([0.1, 0.01, 0.001, 0.0001]),
|
||||
t_camera_from_lidar=np.eye(4),
|
||||
cloud_points_map=np.zeros((100_000, 3), dtype=np.float64),
|
||||
)
|
||||
assert pack.stat().st_size > 1024 * 1024
|
||||
projection = _projection(pack)
|
||||
assert projection.width == 800
|
||||
assert projection.height == 600
|
||||
assert projection.intrinsic_fx_fy_cx_cy == (300.0, 301.0, 400.0, 300.0)
|
||||
|
||||
|
||||
def test_distance_accepts_sealed_lidar_pack_at_package_mount_name(tmp_path: Path) -> None:
|
||||
root = tmp_path / "lidar-pack"
|
||||
root.mkdir()
|
||||
arrays = root / "lidar-replay.npz"
|
||||
np.savez(
|
||||
arrays,
|
||||
point_offsets=np.asarray([0, 1]),
|
||||
point_xyz_map=np.asarray([[1.0, 2.0, 3.0]]),
|
||||
point_received_monotonic_ns=np.asarray([1], dtype=np.int64),
|
||||
pose_positions_map=np.asarray([[0.0, 0.0, 0.0]]),
|
||||
pose_quaternions_map_from_lidar=np.asarray([[0.0, 0.0, 0.0, 1.0]]),
|
||||
pose_received_monotonic_ns=np.asarray([1], dtype=np.int64),
|
||||
)
|
||||
digest = hashlib.sha256(arrays.read_bytes()).hexdigest()
|
||||
identity = "3" * 64
|
||||
(root / "manifest.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "missioncore.lidar-replay-pack/v2",
|
||||
"pack_id": f"lidar-replay-pack-{identity}",
|
||||
"identity_sha256": identity,
|
||||
"artifacts": [
|
||||
{
|
||||
"kind": "lidar-arrays",
|
||||
"path": "lidar-replay.npz",
|
||||
"byte_length": arrays.stat().st_size,
|
||||
"sha256": digest,
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
pack = _LidarPack(root)
|
||||
try:
|
||||
assert pack.pack_id == f"lidar-replay-pack-{identity}"
|
||||
assert pack.point_frame_count == 1
|
||||
finally:
|
||||
pack.close()
|
||||
with pytest.raises(ObjectDistanceModuleError, match="camera timeline"):
|
||||
_aligned_camera_seconds(
|
||||
{"session_seconds": 136.002},
|
||||
{"frame_index": 1299, "session_seconds": 174.0},
|
||||
frame_index=1299,
|
||||
previous_camera_seconds=175.0,
|
||||
)
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Recovery against the durable queue and real sealed-package publisher.
|
||||
|
||||
Only source/model payloads are synthetic. No Worker, GPU, or application server
|
||||
is started; reopening the same SQLite/CAS roots simulates process restart.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from test_observatory_portable_result_publisher import (
|
||||
NOW,
|
||||
RESULT_ID,
|
||||
_cache_fixture,
|
||||
_retry_intent,
|
||||
)
|
||||
from test_observatory_recorded_jobs import _intent, _queue, _running_job
|
||||
|
||||
from k1link.observatory.portable_publication_reconciler import PortablePublicationReconciler
|
||||
from k1link.observatory.portable_result_contract import PortableResultPublisherError
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueDuplicateError,
|
||||
RecordedRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.sessions import SessionStore
|
||||
|
||||
|
||||
def test_outbox_cursor_survives_publication_and_tied_creation_times(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
for index in range(7):
|
||||
job, claim = _running_job(
|
||||
queue,
|
||||
intent=_intent(idempotency_key=f"page-{index}", source_session_id=f"source-{index}"),
|
||||
claim_request_id=f"page-claim-{index}",
|
||||
)
|
||||
queue.complete_for_publication(
|
||||
job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
result_id=f"result-{index}",
|
||||
result_sha256="a" * 64,
|
||||
)
|
||||
expected = queue.pending_publications()
|
||||
first = queue.pending_publications(limit=3)
|
||||
assert first == expected[:3]
|
||||
cursor = (first[-1].created_at_utc, first[-1].job_id)
|
||||
for item in first:
|
||||
queue.mark_published(item.job_id)
|
||||
# Cursor need not point at a still-pending row; never use mutable offsets.
|
||||
assert queue.pending_publications(limit=3, after=cursor) == expected[3:6]
|
||||
assert (
|
||||
queue.pending_publications(
|
||||
limit=3, after=(expected[-1].created_at_utc, expected[-1].job_id)
|
||||
)
|
||||
== ()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("limit", [0, 101, True, 1.5])
|
||||
def test_outbox_page_rejects_invalid_bounds(tmp_path: Path, limit: Any) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
_queue(tmp_path).pending_publications(limit=limit)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure_after_publish", [False, True])
|
||||
def test_publication_recovers_after_restart_without_new_compute(
|
||||
tmp_path: Path,
|
||||
failure_after_publish: bool,
|
||||
) -> None:
|
||||
cache, queue, sessions, _, registry, definition, job, publisher, package = _cache_fixture(
|
||||
tmp_path,
|
||||
published=False,
|
||||
)
|
||||
current = datetime.fromisoformat(NOW.replace("Z", "+00:00"))
|
||||
|
||||
class Transport:
|
||||
def package_root_for_terminal(self, candidate: Any) -> Path:
|
||||
assert candidate.job_id == job.job_id
|
||||
assert candidate.result_sha256 == job.result_sha256
|
||||
return package
|
||||
|
||||
class InterruptedPublisher:
|
||||
def publish(self, **kwargs: Any) -> None:
|
||||
if failure_after_publish:
|
||||
publisher.publish(**kwargs)
|
||||
raise PortableResultPublisherError("temporary publication interruption")
|
||||
|
||||
first = PortablePublicationReconciler(
|
||||
queue=queue,
|
||||
artifact_transport=cast(Any, Transport()),
|
||||
result_publisher=cast(Any, InterruptedPublisher()),
|
||||
clock=lambda: current,
|
||||
).run_once()
|
||||
assert first.failed == 1
|
||||
failed = queue.get(job.job_id)
|
||||
assert failed.state == "succeeded" and failed.publication_state == "failed"
|
||||
assert failed.result_sha256 == job.result_sha256
|
||||
with pytest.raises(ObservatoryRecordedQueueDuplicateError):
|
||||
queue.submit(_retry_intent(job), reject_duplicate_computation=True)
|
||||
|
||||
restarted = ObservatoryRecordedJobQueue(
|
||||
sessions.data_dir,
|
||||
definitions=RecordedRunDefinitionRegistry(registry.ready_recorded_definitions()),
|
||||
clock=lambda: (current + timedelta(minutes=1)).isoformat().replace("+00:00", "Z"),
|
||||
)
|
||||
second = PortablePublicationReconciler(
|
||||
queue=restarted,
|
||||
artifact_transport=cast(Any, Transport()),
|
||||
result_publisher=publisher,
|
||||
clock=lambda: current + timedelta(minutes=1),
|
||||
).run_once()
|
||||
assert second.published == 1
|
||||
assert restarted.get(job.job_id).publication_state == "published"
|
||||
assert len(restarted.list_jobs()) == 1
|
||||
assert restarted.pending_publications() == ()
|
||||
assert [row["result_id"] for row in cache.find(job.source_session_id, definition)] == [
|
||||
RESULT_ID
|
||||
]
|
||||
reopened = SessionStore(sessions.repository_root, data_dir=sessions.data_dir)
|
||||
assert reopened.get_lab_instance(RESULT_ID) == sessions.get_lab_instance(RESULT_ID)
|
||||
# No second publication attempt once the outbox acknowledgement is durable.
|
||||
assert (
|
||||
PortablePublicationReconciler(
|
||||
queue=restarted,
|
||||
artifact_transport=cast(Any, Transport()),
|
||||
result_publisher=publisher,
|
||||
clock=lambda: datetime.now(UTC),
|
||||
)
|
||||
.run_once()
|
||||
.examined
|
||||
== 0
|
||||
)
|
||||
@@ -471,14 +471,10 @@ def test_component_adapter_accepts_only_exact_legacy_or_installed_package_layout
|
||||
component="ddrnet",
|
||||
expectations=expectations,
|
||||
)
|
||||
assert installed.request == Path(
|
||||
"/missioncore/input/steps/prepare/ddrnet-request.json"
|
||||
)
|
||||
assert installed.camera_job_root == Path(
|
||||
"/missioncore/input/steps/prepare/camera-job"
|
||||
)
|
||||
assert installed.request == Path("/missioncore/input/steps/prepare/ddrnet-request.json")
|
||||
assert installed.camera_job_root == Path("/missioncore/input/steps/prepare/camera-job")
|
||||
assert installed.output_root == Path("/missioncore/output")
|
||||
assert installed.eomt_result_root == Path("/missioncore/input/steps/eomt")
|
||||
assert installed.eomt_result_root == Path("/missioncore/input/steps/camera-source")
|
||||
|
||||
with pytest.raises(contract.ComponentAdapterError, match="accepts only"):
|
||||
contract.resolve_runtime_layout(
|
||||
@@ -969,10 +965,66 @@ def test_eomt_adapter_uses_only_fixed_legacy_argv_and_publishes_frames(
|
||||
|
||||
|
||||
def test_eomt_default_disk_floor_retains_large_post_run_reserve() -> None:
|
||||
assert eomt.DISK_FLOOR_BYTES == 350 * 1024**3
|
||||
assert eomt.DISK_FLOOR_BYTES == 250 * 1024**3
|
||||
full_record_working_set = 6_830 * 800 * 600 * 7 + 556_912_640
|
||||
assert full_record_working_set < 22 * 1024**3
|
||||
assert (eomt.DISK_FLOOR_BYTES + full_record_working_set) < 372 * 1024**3
|
||||
assert (eomt.DISK_FLOOR_BYTES + full_record_working_set) < 272 * 1024**3
|
||||
|
||||
|
||||
@pytest.mark.parametrize("floor", [-1, True, 250.5, eomt.DISK_FLOOR_BYTES])
|
||||
def test_eomt_disk_rejection_precedes_heavy_input_and_asset_validation(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
floor: int,
|
||||
) -> None:
|
||||
output = tmp_path / "output"
|
||||
output.mkdir()
|
||||
source = SimpleNamespace(frame_count=6_830, input_byte_length=556_912_640)
|
||||
reserve = source.frame_count * 800 * 600 * 7 + source.input_byte_length
|
||||
monkeypatch.setattr(
|
||||
eomt, "load_component_request", lambda *a, **kw: SimpleNamespace(source=source)
|
||||
)
|
||||
monkeypatch.setattr(eomt, "available_bytes", lambda _: eomt.DISK_FLOOR_BYTES + reserve - 1)
|
||||
|
||||
def heavy_work(*args: object, **kwargs: object) -> None:
|
||||
pytest.fail("disk admission must precede input hashing, asset hashing and inference")
|
||||
|
||||
monkeypatch.setattr(eomt, "validate_camera_compute_job", heavy_work)
|
||||
monkeypatch.setattr(eomt, "_validate_release_assets", heavy_work)
|
||||
layout = cast(contract.RuntimeLayout, SimpleNamespace(output_root=output))
|
||||
with pytest.raises(contract.ComponentAdapterError, match="disk reserve"):
|
||||
eomt.execute_eomt_component(
|
||||
request_path=tmp_path / "request.json",
|
||||
layout=layout,
|
||||
command_runner=heavy_work,
|
||||
disk_floor_bytes=floor,
|
||||
)
|
||||
assert not list(output.iterdir())
|
||||
|
||||
|
||||
def test_eomt_exact_disk_budget_still_requires_source_validation(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
output = tmp_path / "output"
|
||||
output.mkdir()
|
||||
source = SimpleNamespace(frame_count=6_830, input_byte_length=556_912_640)
|
||||
reserve = source.frame_count * 800 * 600 * 7 + source.input_byte_length
|
||||
monkeypatch.setattr(
|
||||
eomt, "load_component_request", lambda *a, **kw: SimpleNamespace(source=source)
|
||||
)
|
||||
monkeypatch.setattr(eomt, "available_bytes", lambda _: eomt.DISK_FLOOR_BYTES + reserve)
|
||||
|
||||
def invalid_source(*args: object) -> None:
|
||||
raise contract.ComponentAdapterError("source validation remains mandatory")
|
||||
|
||||
monkeypatch.setattr(eomt, "validate_camera_compute_job", invalid_source)
|
||||
layout = cast(
|
||||
contract.RuntimeLayout, SimpleNamespace(output_root=output, camera_job_root=tmp_path)
|
||||
)
|
||||
with pytest.raises(contract.ComponentAdapterError, match="source validation remains mandatory"):
|
||||
eomt.execute_eomt_component(request_path=tmp_path / "request.json", layout=layout)
|
||||
assert not list(output.iterdir())
|
||||
|
||||
|
||||
def _effective_ddrnet_config(
|
||||
|
||||
@@ -308,7 +308,7 @@ def test_duplicate_definition_and_incomplete_ready_executor_are_rejected(
|
||||
PortableRunDefinitionRegistry.from_file(_write(tmp_path, incomplete))
|
||||
|
||||
|
||||
def test_production_definitions_are_both_ready_and_convertible() -> None:
|
||||
def test_production_definitions_are_ready_and_convertible() -> None:
|
||||
registry = _registry()
|
||||
definition = registry.definitions[0]
|
||||
|
||||
@@ -325,6 +325,10 @@ def test_production_definitions_are_both_ready_and_convertible() -> None:
|
||||
assert tuple(row.setup_id for row in ready) == (
|
||||
"lab-v1-eomt-ddrnet-portable-v1",
|
||||
"m49-tgs-portable-v2",
|
||||
"ai-segmentation-ddrnet-v1",
|
||||
"ai-segmentation-eomt-v1",
|
||||
"ai-detection-rf-detr-v1",
|
||||
"ai-range-object-distance-v1",
|
||||
)
|
||||
assert registry.to_recorded_registry().definitions == ready
|
||||
|
||||
|
||||
@@ -140,7 +140,7 @@ class _GenericProbe:
|
||||
)
|
||||
|
||||
|
||||
def test_generic_catalog_projects_lab_v1_and_model_free_m49_independently() -> None:
|
||||
def test_generic_catalog_projects_every_registered_portable_setup_independently() -> None:
|
||||
registry = _registry()
|
||||
catalog = PortableSetupProjector(
|
||||
registry=registry,
|
||||
@@ -151,6 +151,10 @@ def test_generic_catalog_projects_lab_v1_and_model_free_m49_independently() -> N
|
||||
assert set(setups) == {
|
||||
"lab-v1-eomt-ddrnet-portable-v1",
|
||||
"m49-tgs-portable-v2",
|
||||
"ai-segmentation-ddrnet-v1",
|
||||
"ai-segmentation-eomt-v1",
|
||||
"ai-detection-rf-detr-v1",
|
||||
"ai-range-object-distance-v1",
|
||||
}
|
||||
m49 = setups["m49-tgs-portable-v2"]
|
||||
assert m49["display_name"] == PORTABLE_M49_DISPLAY_NAME
|
||||
|
||||
@@ -11,6 +11,10 @@ from k1link.observatory.m49_portable_result import (
|
||||
M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||
validate_m49_portable_result,
|
||||
)
|
||||
from k1link.observatory.modular_result import (
|
||||
MODULAR_RESULT_CONTRACT_SHA256,
|
||||
validate_modular_result,
|
||||
)
|
||||
from k1link.observatory.portable_artifact_transport import (
|
||||
PortableObservatoryArtifactTransport,
|
||||
)
|
||||
@@ -26,6 +30,14 @@ from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
AI_DDRNET_DISPLAY_NAME,
|
||||
AI_DDRNET_SETUP_ID,
|
||||
AI_EOMT_DISPLAY_NAME,
|
||||
AI_EOMT_SETUP_ID,
|
||||
AI_OBJECT_DISTANCE_DISPLAY_NAME,
|
||||
AI_OBJECT_DISTANCE_SETUP_ID,
|
||||
AI_RF_DETR_DISPLAY_NAME,
|
||||
AI_RF_DETR_SETUP_ID,
|
||||
PORTABLE_LAB_V1_DISPLAY_NAME,
|
||||
PORTABLE_LAB_V1_SETUP_ID,
|
||||
PORTABLE_M49_DISPLAY_NAME,
|
||||
@@ -53,30 +65,28 @@ def _definitions() -> PortableRunDefinitionRegistry:
|
||||
return PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||
|
||||
|
||||
def test_exact_validator_registry_covers_both_portable_profiles() -> None:
|
||||
def test_exact_validator_registry_covers_every_distinct_portable_contract() -> None:
|
||||
definitions = _definitions()
|
||||
|
||||
validators = portable_result_validator_registry(definitions)
|
||||
|
||||
assert validators.resolve(PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256) is validate_lab_v1_result_v2
|
||||
assert validators.resolve(M49_PORTABLE_RESULT_CONTRACT_SHA256) is validate_m49_portable_result
|
||||
assert validators.resolve(MODULAR_RESULT_CONTRACT_SHA256) is validate_modular_result
|
||||
assert len(validators.registrations) == 3
|
||||
|
||||
|
||||
def test_local_worker_gate_is_fail_closed_and_accepts_only_exact_one() -> None:
|
||||
assert observatory_worker_local_enabled({}) is False
|
||||
assert observatory_worker_local_enabled({OBSERVATORY_WORKER_LOCAL_ENABLED_ENV: ""}) is False
|
||||
assert observatory_worker_local_enabled(
|
||||
{OBSERVATORY_WORKER_LOCAL_ENABLED_ENV: "1"}
|
||||
) is True
|
||||
assert observatory_worker_local_enabled({OBSERVATORY_WORKER_LOCAL_ENABLED_ENV: "1"}) is True
|
||||
|
||||
for value in ("0", "true", " 1", "1 "):
|
||||
with pytest.raises(
|
||||
PortableWorkerIntegrationError,
|
||||
match="must be exactly 1 when enabled",
|
||||
):
|
||||
observatory_worker_local_enabled(
|
||||
{OBSERVATORY_WORKER_LOCAL_ENABLED_ENV: value}
|
||||
)
|
||||
observatory_worker_local_enabled({OBSERVATORY_WORKER_LOCAL_ENABLED_ENV: value})
|
||||
|
||||
|
||||
def test_validator_registry_selects_only_contracts_present_in_definitions() -> None:
|
||||
@@ -117,6 +127,10 @@ def test_server_integration_constructs_dormant_transport_and_publisher(
|
||||
assert integration.supported_setup_ids == (
|
||||
PORTABLE_LAB_V1_SETUP_ID,
|
||||
PORTABLE_M49_SETUP_ID,
|
||||
AI_DDRNET_SETUP_ID,
|
||||
AI_EOMT_SETUP_ID,
|
||||
AI_RF_DETR_SETUP_ID,
|
||||
AI_OBJECT_DISTANCE_SETUP_ID,
|
||||
)
|
||||
assert isinstance(
|
||||
integration.artifact_transport,
|
||||
@@ -132,6 +146,10 @@ def test_server_integration_constructs_dormant_transport_and_publisher(
|
||||
assert profiles == {
|
||||
PORTABLE_LAB_V1_SETUP_ID: PORTABLE_LAB_V1_DISPLAY_NAME,
|
||||
PORTABLE_M49_SETUP_ID: PORTABLE_M49_DISPLAY_NAME,
|
||||
AI_DDRNET_SETUP_ID: AI_DDRNET_DISPLAY_NAME,
|
||||
AI_EOMT_SETUP_ID: AI_EOMT_DISPLAY_NAME,
|
||||
AI_RF_DETR_SETUP_ID: AI_RF_DETR_DISPLAY_NAME,
|
||||
AI_OBJECT_DISTANCE_SETUP_ID: AI_OBJECT_DISTANCE_DISPLAY_NAME,
|
||||
}
|
||||
|
||||
|
||||
@@ -241,9 +259,7 @@ def test_explicit_volumes_storage_boundary_still_requires_a_mount(
|
||||
PortableWorkerIntegrationError,
|
||||
match="central artifact volume is not mounted: /Volumes/nodedc",
|
||||
):
|
||||
integration_module._require_mounted_volume(
|
||||
Path("/Volumes/nodedc/mission-core")
|
||||
)
|
||||
integration_module._require_mounted_volume(Path("/Volumes/nodedc/mission-core"))
|
||||
|
||||
assert mount_checks == [Path("/Volumes/nodedc")]
|
||||
|
||||
|
||||
@@ -53,8 +53,7 @@ def _runtime() -> PortableWorkerRuntimeRegistry:
|
||||
|
||||
def _blocked_candidate(candidate):
|
||||
phases = tuple(
|
||||
PortableWorkerRuntimePhase(phase.phase_id, "missing")
|
||||
for phase in candidate.phases
|
||||
PortableWorkerRuntimePhase(phase.phase_id, "missing") for phase in candidate.phases
|
||||
)
|
||||
blockers = ("executor-release-unsealed",)
|
||||
identity = candidate.identity_document()
|
||||
@@ -80,14 +79,25 @@ def _all_keys(value: object) -> set[str]:
|
||||
return set()
|
||||
|
||||
|
||||
def test_production_candidates_bind_exact_definitions_and_both_are_ready() -> None:
|
||||
def test_production_candidates_bind_every_exact_definition_and_are_ready() -> None:
|
||||
registry = _runtime()
|
||||
|
||||
assert {candidate.setup_id for candidate in registry.candidates} == {
|
||||
"lab-v1-eomt-ddrnet-portable-v1",
|
||||
"m49-tgs-portable-v2",
|
||||
"ai-segmentation-ddrnet-v1",
|
||||
"ai-segmentation-eomt-v1",
|
||||
"ai-detection-rf-detr-v1",
|
||||
"ai-range-object-distance-v1",
|
||||
}
|
||||
by_setup = {candidate.setup_id: candidate for candidate in registry.candidates}
|
||||
assert all(candidate.ready for candidate in registry.candidates)
|
||||
assert all(candidate.executor is not None for candidate in registry.candidates)
|
||||
assert all(candidate.blockers == () for candidate in registry.candidates)
|
||||
assert all(
|
||||
all(phase.state == "implemented" for phase in candidate.phases)
|
||||
for candidate in registry.candidates
|
||||
)
|
||||
lab_v1 = by_setup["lab-v1-eomt-ddrnet-portable-v1"]
|
||||
assert lab_v1.ready is True
|
||||
assert lab_v1.executor is not None
|
||||
@@ -228,10 +238,7 @@ def test_ready_lab_candidate_still_requires_complete_local_asset_admission() ->
|
||||
"ddrnet-portable-config": PortableWorkerLocalAssetBinding(
|
||||
asset_id="ddrnet-portable-config",
|
||||
file_path=(
|
||||
REPOSITORY_ROOT
|
||||
/ "config"
|
||||
/ "perception"
|
||||
/ "lab-v1-eomt-ddrnet-portable-v2.json"
|
||||
REPOSITORY_ROOT / "config" / "perception" / "lab-v1-eomt-ddrnet-portable-v2.json"
|
||||
),
|
||||
),
|
||||
"ddrnet-step-image": PortableWorkerLocalAssetBinding(
|
||||
@@ -370,9 +377,7 @@ def test_sealed_local_tree_uses_manifest_receipt_and_member_metadata(
|
||||
"model-bin": {
|
||||
"relative_path": "model.bin",
|
||||
"byte_length": 5,
|
||||
"sha256": (
|
||||
"9372c470eeadd5ec5f36cb0b9adf10545c93c5132503830bf1465fe7654b117b"
|
||||
),
|
||||
"sha256": ("9372c470eeadd5ec5f36cb0b9adf10545c93c5132503830bf1465fe7654b117b"),
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -473,10 +478,12 @@ def test_ready_local_adapter_composes_only_local_ports_and_exact_job(
|
||||
definition_sha256=canonical_sha256(definition_identity),
|
||||
)
|
||||
|
||||
blocked = _blocked_candidate(_runtime().resolve(
|
||||
"lab-v1-eomt-ddrnet-portable-v1",
|
||||
base_definition.definition_sha256,
|
||||
))
|
||||
blocked = _blocked_candidate(
|
||||
_runtime().resolve(
|
||||
"lab-v1-eomt-ddrnet-portable-v1",
|
||||
base_definition.definition_sha256,
|
||||
)
|
||||
)
|
||||
executor_seal = PortableWorkerExecutorSeal(
|
||||
release_id="lab-v1-portable-executor-v1",
|
||||
release_sha256="1" * 64,
|
||||
|
||||
@@ -5,6 +5,9 @@ from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.portable_artifact_transport import PortableArtifactTransportError
|
||||
from k1link.observatory.portable_publication_reconciler import (
|
||||
PortablePublicationReconciler,
|
||||
)
|
||||
@@ -18,8 +21,17 @@ class _Queue:
|
||||
self.published: list[str] = []
|
||||
self.failed: list[str] = []
|
||||
|
||||
def pending_publications(self) -> tuple[SimpleNamespace, ...]:
|
||||
return self.jobs
|
||||
def pending_publications(
|
||||
self,
|
||||
*,
|
||||
limit: int,
|
||||
after: tuple[str, str] | None = None,
|
||||
) -> tuple[SimpleNamespace, ...]:
|
||||
return tuple(
|
||||
job
|
||||
for job in sorted(self.jobs, key=lambda job: (job.created_at_utc, job.job_id))
|
||||
if after is None or (job.created_at_utc, job.job_id) > after
|
||||
)[:limit]
|
||||
|
||||
def mark_published(self, job_id: str) -> None:
|
||||
self.published.append(job_id)
|
||||
@@ -103,7 +115,67 @@ def _job(
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
job_id=f"observatory-run-{attempts:032x}",
|
||||
created_at_utc=updated_at,
|
||||
publication_state=publication_state,
|
||||
publication_attempts=attempts,
|
||||
updated_at_utc=updated_at,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("skip_kind", ["exhausted", "backoff"])
|
||||
def test_old_outbox_prefix_cannot_starve_ready_results_across_pages(
|
||||
tmp_path: Path,
|
||||
skip_kind: str,
|
||||
) -> None:
|
||||
jobs = []
|
||||
for index in range(70):
|
||||
skipped = index < 66
|
||||
job = _job(
|
||||
"failed" if skipped else "pending",
|
||||
attempts=(5 if skip_kind == "exhausted" else 1) if skipped else 0,
|
||||
updated_at="2026-09-01T11:59:50Z" if skipped else "2026-09-01T11:00:00Z",
|
||||
)
|
||||
job.created_at_utc = "2026-09-01T10:00:00Z"
|
||||
job.job_id = f"observatory-run-{index:032x}"
|
||||
jobs.append(job)
|
||||
queue = _Queue(tuple(jobs))
|
||||
publisher = _Publisher()
|
||||
reconciler = PortablePublicationReconciler(
|
||||
queue=cast(Any, queue),
|
||||
artifact_transport=cast(Any, _Transport(tmp_path)),
|
||||
result_publisher=cast(Any, publisher),
|
||||
clock=lambda: NOW,
|
||||
)
|
||||
|
||||
result = reconciler.run_once(limit=2)
|
||||
|
||||
assert result.examined == 68
|
||||
assert result.published == 2
|
||||
assert getattr(result, "exhausted" if skip_kind == "exhausted" else "deferred") == 66
|
||||
assert publisher.calls == [jobs[66].job_id, jobs[67].job_id]
|
||||
assert queue.failed == []
|
||||
|
||||
|
||||
def test_one_missing_package_does_not_block_the_next_publication(tmp_path: Path) -> None:
|
||||
first = _job("pending", attempts=0, updated_at="2026-09-01T10:00:00Z")
|
||||
second = _job("pending", attempts=0, updated_at="2026-09-01T11:00:00Z")
|
||||
second.job_id = f"observatory-run-{1:032x}"
|
||||
queue = _Queue((first, second))
|
||||
publisher = _Publisher()
|
||||
|
||||
class MissingFirst(_Transport):
|
||||
def package_root_for_terminal(self, job: SimpleNamespace) -> Path:
|
||||
if job.job_id == first.job_id:
|
||||
raise PortableArtifactTransportError("sealed package temporarily unavailable")
|
||||
return super().package_root_for_terminal(job)
|
||||
|
||||
result = PortablePublicationReconciler(
|
||||
queue=cast(Any, queue),
|
||||
artifact_transport=cast(Any, MissingFirst(tmp_path)),
|
||||
result_publisher=cast(Any, publisher),
|
||||
clock=lambda: NOW,
|
||||
).run_once(limit=2)
|
||||
|
||||
assert result.published == 1 and result.failed == 1
|
||||
assert queue.failed == [first.job_id]
|
||||
assert publisher.calls == [second.job_id]
|
||||
|
||||
@@ -99,6 +99,7 @@ def _queue(
|
||||
) -> ObservatoryRecordedJobQueue:
|
||||
preemptor = None
|
||||
if with_non_checkpointable_preemptor:
|
||||
|
||||
def preemptor(request):
|
||||
return ObservatoryNonCheckpointableCancellationReceipt.sealed(
|
||||
request,
|
||||
@@ -154,9 +155,7 @@ def _running_job(
|
||||
job, created = queue.submit(intent or _intent())
|
||||
assert created is True
|
||||
queue.enqueue(job.job_id)
|
||||
claim = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id=claim_request_id
|
||||
)
|
||||
claim = queue.claim_next(claimant_id="recorded-worker", claim_request_id=claim_request_id)
|
||||
assert claim is not None
|
||||
running = queue.start(job.job_id, claim_token=claim.claim_token)
|
||||
assert running.state == "running"
|
||||
@@ -207,7 +206,8 @@ def test_list_filters_definition_before_page_limit(tmp_path: Path) -> None:
|
||||
current = replace(previous, definition_version=2, definition_sha256="0" * 64)
|
||||
now = NOW
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path, definitions=RecordedRunDefinitionRegistry((previous, current)),
|
||||
tmp_path,
|
||||
definitions=RecordedRunDefinitionRegistry((previous, current)),
|
||||
clock=lambda: now,
|
||||
)
|
||||
first, _ = queue.submit(_intent())
|
||||
@@ -220,8 +220,10 @@ def test_list_filters_definition_before_page_limit(tmp_path: Path) -> None:
|
||||
)
|
||||
assert queue.list_jobs(limit=1)[0].definition_sha256 == current.definition_sha256
|
||||
assert queue.list_jobs(
|
||||
source_session_id=first.source_session_id, setup_id=previous.setup_id,
|
||||
definition_sha256=previous.definition_sha256, limit=1,
|
||||
source_session_id=first.source_session_id,
|
||||
setup_id=previous.setup_id,
|
||||
definition_sha256=previous.definition_sha256,
|
||||
limit=1,
|
||||
) == (first,)
|
||||
with pytest.raises(ValueError):
|
||||
queue.list_jobs(definition_sha256="not-a-digest")
|
||||
@@ -266,9 +268,6 @@ def test_portable_duplicate_guard_preserves_original_request(
|
||||
"field",
|
||||
[
|
||||
"source_session_id",
|
||||
"source_catalog_sha256",
|
||||
"source_bundle_sha256",
|
||||
"source_capability_manifest_sha256",
|
||||
"setup_id",
|
||||
],
|
||||
)
|
||||
@@ -280,9 +279,6 @@ def test_duplicate_guard_keeps_distinct_sources_and_profiles(
|
||||
queue.submit(_intent(), reject_duplicate_computation=True)
|
||||
values = {
|
||||
"source_session_id": "another-source",
|
||||
"source_catalog_sha256": "0" * 64,
|
||||
"source_bundle_sha256": "0" * 64,
|
||||
"source_capability_manifest_sha256": "0" * 64,
|
||||
"setup_id": "legacy-monolith-v1",
|
||||
}
|
||||
changes = {field: values[field]}
|
||||
@@ -293,6 +289,28 @@ def test_duplicate_guard_keeps_distinct_sources_and_profiles(
|
||||
assert created and len(queue.list_jobs()) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field",
|
||||
["source_catalog_sha256", "source_bundle_sha256", "source_capability_manifest_sha256"],
|
||||
)
|
||||
def test_duplicate_guard_does_not_recalculate_same_record_after_source_repair(
|
||||
tmp_path: Path,
|
||||
field: str,
|
||||
) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
first, _ = queue.submit(_intent(), reject_duplicate_computation=True)
|
||||
with pytest.raises(ObservatoryRecordedQueueDuplicateError) as duplicate:
|
||||
queue.submit(
|
||||
replace(
|
||||
_intent(idempotency_key=f"repaired-{field}"),
|
||||
**{field: "0" * 64},
|
||||
),
|
||||
reject_duplicate_computation=True,
|
||||
)
|
||||
assert duplicate.value.job_id == first.job_id
|
||||
assert len(queue.list_jobs()) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("publication_failed", [False, True])
|
||||
def test_duplicate_guard_never_recomputes_pending_publication(
|
||||
tmp_path: Path,
|
||||
@@ -321,7 +339,9 @@ def test_duplicate_guard_never_recomputes_pending_publication(
|
||||
assert len(queue.list_jobs()) == 1
|
||||
|
||||
|
||||
def test_duplicate_guard_allows_retry_after_computation_failure(tmp_path: Path) -> None:
|
||||
def test_duplicate_guard_allows_a_fresh_attempt_after_computation_failure(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job, claim = _running_job(queue)
|
||||
queue.fail(
|
||||
@@ -336,6 +356,8 @@ def test_duplicate_guard_allows_retry_after_computation_failure(tmp_path: Path)
|
||||
reject_duplicate_computation=True,
|
||||
)
|
||||
assert created and retried.job_id != job.job_id
|
||||
assert retried.state == "queued"
|
||||
assert len(queue.list_jobs()) == 2
|
||||
|
||||
|
||||
def test_duplicate_guard_is_atomic_across_two_queue_instances(tmp_path: Path) -> None:
|
||||
@@ -383,12 +405,14 @@ def test_duplicate_guard_keeps_new_version_of_same_setup(tmp_path: Path) -> None
|
||||
|
||||
def _capabilities() -> tuple[RecordedExecutorIdentity, ...]:
|
||||
definition = _definitions().definitions[0]
|
||||
return (RecordedExecutorIdentity(
|
||||
release_sha256=definition.executor_release_sha256,
|
||||
image_sha256=definition.executor_image_sha256,
|
||||
model_manifest_sha256=definition.model_manifest_sha256,
|
||||
resource_profile_sha256=definition.resource_profile_sha256,
|
||||
),)
|
||||
return (
|
||||
RecordedExecutorIdentity(
|
||||
release_sha256=definition.executor_release_sha256,
|
||||
image_sha256=definition.executor_image_sha256,
|
||||
model_manifest_sha256=definition.model_manifest_sha256,
|
||||
resource_profile_sha256=definition.resource_profile_sha256,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _claim_rows(queue: ObservatoryRecordedJobQueue, table: str) -> list[tuple]:
|
||||
@@ -399,8 +423,11 @@ def _claim_rows(queue: ObservatoryRecordedJobQueue, table: str) -> list[tuple]:
|
||||
|
||||
def test_v3_idle_does_not_consume_grant_quota_or_modify_legacy_receipts(tmp_path: Path) -> None:
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path, definitions=_definitions(), clock=lambda: NOW,
|
||||
max_claim_receipts=1, max_v3_claim_receipts=1,
|
||||
tmp_path,
|
||||
definitions=_definitions(),
|
||||
clock=lambda: NOW,
|
||||
max_claim_receipts=1,
|
||||
max_v3_claim_receipts=1,
|
||||
)
|
||||
args = {"claimant_id": "recorded-worker", "supported_executor_identities": _capabilities()}
|
||||
assert queue.claim_next(**args, claim_request_id="legacy-empty") is None
|
||||
@@ -411,9 +438,14 @@ def test_v3_idle_does_not_consume_grant_quota_or_modify_legacy_receipts(tmp_path
|
||||
with pytest.raises(ObservatoryRecordedQueueCapacityError):
|
||||
queue.claim_next(**args, claim_request_id="legacy-full")
|
||||
for number in range(8):
|
||||
assert queue.claim_next(
|
||||
**args, claim_request_id=f"idle-v3-{number}", protocol_version=3,
|
||||
) is None
|
||||
assert (
|
||||
queue.claim_next(
|
||||
**args,
|
||||
claim_request_id=f"idle-v3-{number}",
|
||||
protocol_version=3,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert _claim_rows(queue, "observatory_recorded_claim_grants_v3") == []
|
||||
assert _claim_rows(queue, "observatory_recorded_claim_receipts") == legacy
|
||||
|
||||
@@ -425,23 +457,30 @@ def test_v3_idle_does_not_consume_grant_quota_or_modify_legacy_receipts(tmp_path
|
||||
assert len(_claim_rows(queue, "observatory_recorded_claim_grants_v3")) == 1
|
||||
assert _claim_rows(queue, "observatory_recorded_claim_receipts") == legacy
|
||||
reopened = ObservatoryRecordedJobQueue(
|
||||
tmp_path, definitions=_definitions(), clock=lambda: NOW,
|
||||
max_claim_receipts=1, max_v3_claim_receipts=1,
|
||||
tmp_path,
|
||||
definitions=_definitions(),
|
||||
clock=lambda: NOW,
|
||||
max_claim_receipts=1,
|
||||
max_v3_claim_receipts=1,
|
||||
)
|
||||
assert reopened.claim_next(**args, claim_request_id="idle-v3-0", protocol_version=3) == claim
|
||||
# No second grant even if a client retries a v3 grant through legacy v2.
|
||||
assert reopened.claim_next(**args, claim_request_id="idle-v3-0") == claim
|
||||
with pytest.raises(ObservatoryRecordedQueueConflictError):
|
||||
reopened.claim_next(
|
||||
claimant_id="other-worker", supported_executor_identities=_capabilities(),
|
||||
claim_request_id="idle-v3-0", protocol_version=3,
|
||||
claimant_id="other-worker",
|
||||
supported_executor_identities=_capabilities(),
|
||||
claim_request_id="idle-v3-0",
|
||||
protocol_version=3,
|
||||
)
|
||||
assert reopened.claim_next(**args, claim_request_id="busy-v3", protocol_version=3) is None
|
||||
|
||||
reopened.start(job.job_id, claim_token=claim.claim_token)
|
||||
reopened.succeed(
|
||||
job.job_id, claim_token=claim.claim_token,
|
||||
result_id="result-first", result_sha256=RESULT_SHA,
|
||||
job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
result_id="result-first",
|
||||
result_sha256=RESULT_SHA,
|
||||
)
|
||||
with pytest.raises(ObservatoryRecordedQueueStaleClaimError):
|
||||
reopened.claim_next(**args, claim_request_id="idle-v3-0", protocol_version=3)
|
||||
@@ -457,15 +496,25 @@ def test_v3_idle_does_not_consume_grant_quota_or_modify_legacy_receipts(tmp_path
|
||||
def test_v3_empty_capabilities_and_live_lease_cannot_claim_work(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job, _ = queue.submit(_intent(), enqueue=True)
|
||||
assert queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="no-capabilities",
|
||||
supported_executor_identities=(), protocol_version=3,
|
||||
) is None
|
||||
assert (
|
||||
queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="no-capabilities",
|
||||
supported_executor_identities=(),
|
||||
protocol_version=3,
|
||||
)
|
||||
is None
|
||||
)
|
||||
queue.request_live(_live_intent())
|
||||
assert queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="live-has-priority",
|
||||
supported_executor_identities=_capabilities(), protocol_version=3,
|
||||
) is None
|
||||
assert (
|
||||
queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="live-has-priority",
|
||||
supported_executor_identities=_capabilities(),
|
||||
protocol_version=3,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert queue.get(job.job_id).state == "queued"
|
||||
assert _claim_rows(queue, "observatory_recorded_claim_grants_v3") == []
|
||||
|
||||
@@ -478,8 +527,10 @@ def test_v3_two_simultaneous_retries_receive_one_grant(tmp_path: Path) -> None:
|
||||
def claim(index: int):
|
||||
barrier.wait(timeout=5)
|
||||
return queues[index].claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="same-v3-id",
|
||||
supported_executor_identities=_capabilities(), protocol_version=3,
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="same-v3-id",
|
||||
supported_executor_identities=_capabilities(),
|
||||
protocol_version=3,
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
@@ -491,16 +542,26 @@ def test_v3_two_simultaneous_retries_receive_one_grant(tmp_path: Path) -> None:
|
||||
def test_v3_expired_grant_cannot_reclaim_or_change_capabilities(tmp_path: Path) -> None:
|
||||
clock = [NOW]
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path, definitions=_definitions(), clock=lambda: clock[0], claim_lease_seconds=10,
|
||||
tmp_path,
|
||||
definitions=_definitions(),
|
||||
clock=lambda: clock[0],
|
||||
claim_lease_seconds=10,
|
||||
)
|
||||
job, _ = queue.submit(_intent(), enqueue=True)
|
||||
args = {"claimant_id": "recorded-worker", "supported_executor_identities": _capabilities(),
|
||||
"protocol_version": 3}
|
||||
args = {
|
||||
"claimant_id": "recorded-worker",
|
||||
"supported_executor_identities": _capabilities(),
|
||||
"protocol_version": 3,
|
||||
}
|
||||
first = queue.claim_next(**args, claim_request_id="original-v3")
|
||||
assert first is not None
|
||||
with pytest.raises(ObservatoryRecordedQueueConflictError):
|
||||
queue.claim_next(claimant_id="recorded-worker", claim_request_id="original-v3",
|
||||
supported_executor_identities=(), protocol_version=3)
|
||||
queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="original-v3",
|
||||
supported_executor_identities=(),
|
||||
protocol_version=3,
|
||||
)
|
||||
clock[0] = "2026-08-30T21:00:11.000Z"
|
||||
with pytest.raises(ObservatoryRecordedQueueStaleClaimError):
|
||||
queue.claim_next(**args, claim_request_id="original-v3")
|
||||
@@ -515,8 +576,12 @@ def test_v3_expired_grant_cannot_reclaim_or_change_capabilities(tmp_path: Path)
|
||||
@pytest.mark.parametrize("version", [True, False, 1, 4, "3"])
|
||||
def test_claim_protocol_rejects_ambiguous_versions(tmp_path: Path, version) -> None:
|
||||
with pytest.raises(ValueError, match="protocol version"):
|
||||
_queue(tmp_path).claim_next(claimant_id="recorded-worker", claim_request_id="invalid",
|
||||
supported_executor_identities=(), protocol_version=version)
|
||||
_queue(tmp_path).claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="invalid",
|
||||
supported_executor_identities=(),
|
||||
protocol_version=version,
|
||||
)
|
||||
|
||||
|
||||
def test_submission_is_durable_exactly_idempotent_and_path_free(tmp_path: Path) -> None:
|
||||
@@ -548,15 +613,11 @@ def test_submission_is_durable_exactly_idempotent_and_path_free(tmp_path: Path)
|
||||
assert not hasattr(_intent(), "command")
|
||||
assert not hasattr(_intent(), "image")
|
||||
assert projection["source"]["bundle_sha256"] == SOURCE_BUNDLE_SHA
|
||||
assert projection["source"]["capability_manifest_sha256"] == (
|
||||
SOURCE_CAPABILITIES_SHA
|
||||
)
|
||||
assert projection["source"]["capability_manifest_sha256"] == (SOURCE_CAPABILITIES_SHA)
|
||||
assert projection["executor"]["release_sha256"] == EXECUTOR_RELEASE_SHA
|
||||
assert projection["executor"]["image_sha256"] == EXECUTOR_IMAGE_SHA
|
||||
assert projection["executor"]["model_manifest_sha256"] == MODEL_MANIFEST_SHA
|
||||
assert projection["executor"]["resource_profile_sha256"] == (
|
||||
RESOURCE_PROFILE_SHA
|
||||
)
|
||||
assert projection["executor"]["resource_profile_sha256"] == (RESOURCE_PROFILE_SHA)
|
||||
assert queue.database_path.name == RECORDED_JOB_DATABASE_NAME
|
||||
assert queue.database_path.stat().st_mode & 0o777 == 0o600
|
||||
|
||||
@@ -684,34 +745,23 @@ def test_success_requires_running_and_cannot_publish_during_preemption(
|
||||
def test_claim_is_exactly_idempotent_including_empty_result(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
|
||||
empty = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="empty-poll-001"
|
||||
)
|
||||
empty = queue.claim_next(claimant_id="recorded-worker", claim_request_id="empty-poll-001")
|
||||
assert empty is None
|
||||
job, _ = queue.submit(_intent())
|
||||
queue.enqueue(job.job_id)
|
||||
assert (
|
||||
queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="empty-poll-001"
|
||||
)
|
||||
is None
|
||||
queue.claim_next(claimant_id="recorded-worker", claim_request_id="empty-poll-001") is None
|
||||
)
|
||||
|
||||
claim = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="non-empty-poll-001"
|
||||
)
|
||||
retry = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="non-empty-poll-001"
|
||||
)
|
||||
claim = queue.claim_next(claimant_id="recorded-worker", claim_request_id="non-empty-poll-001")
|
||||
retry = queue.claim_next(claimant_id="recorded-worker", claim_request_id="non-empty-poll-001")
|
||||
assert claim is not None
|
||||
assert retry is not None
|
||||
assert retry.claim_token == claim.claim_token
|
||||
assert retry.job.job_id == job.job_id
|
||||
|
||||
with pytest.raises(ObservatoryRecordedQueueConflictError, match="claim"):
|
||||
queue.claim_next(
|
||||
claimant_id="another-worker", claim_request_id="non-empty-poll-001"
|
||||
)
|
||||
queue.claim_next(claimant_id="another-worker", claim_request_id="non-empty-poll-001")
|
||||
|
||||
|
||||
def test_capability_aware_claim_skips_incompatible_queued_job(tmp_path: Path) -> None:
|
||||
@@ -764,11 +814,14 @@ def test_capability_claim_identity_binds_snapshot_and_empty_snapshot_claims_noth
|
||||
queue = _queue(tmp_path)
|
||||
job, _ = queue.submit(_intent(), enqueue=True)
|
||||
|
||||
assert queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="capability-empty-poll",
|
||||
supported_executor_identities=(),
|
||||
) is None
|
||||
assert (
|
||||
queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="capability-empty-poll",
|
||||
supported_executor_identities=(),
|
||||
)
|
||||
is None
|
||||
)
|
||||
capability = RecordedExecutorIdentity(
|
||||
release_sha256=EXECUTOR_RELEASE_SHA,
|
||||
image_sha256=EXECUTOR_IMAGE_SHA,
|
||||
@@ -1037,9 +1090,7 @@ def test_operator_reconciliation_is_durable_idempotent_and_unblocks_queue(
|
||||
assert restored == receipt
|
||||
assert restored_queue.get_reconciliation(running.job_id) == receipt
|
||||
assert receipt.resource_release_attestation.resources_released is True
|
||||
assert receipt.resource_release_attestation.evidence_sha256 == (
|
||||
RESOURCE_RELEASE_EVIDENCE_SHA
|
||||
)
|
||||
assert receipt.resource_release_attestation.evidence_sha256 == (RESOURCE_RELEASE_EVIDENCE_SHA)
|
||||
assert receipt.expected_terminal_code == "claim-lease-expired"
|
||||
assert receipt.quarantined_terminal_message == (
|
||||
"Worker claim lease expired after execution started; "
|
||||
@@ -1089,14 +1140,19 @@ def test_cache_indexes_follow_legacy_publication_column_migration(tmp_path: Path
|
||||
connection.execute("DROP INDEX observatory_recorded_jobs_published_source")
|
||||
connection.execute("DROP INDEX observatory_recorded_jobs_computation")
|
||||
for column in (
|
||||
"publication_state", "publication_attempts", "publication_error", "published_at_utc",
|
||||
"publication_state",
|
||||
"publication_attempts",
|
||||
"publication_error",
|
||||
"published_at_utc",
|
||||
):
|
||||
connection.execute(f"ALTER TABLE observatory_recorded_jobs DROP COLUMN {column}")
|
||||
migrated = _queue(tmp_path)
|
||||
assert migrated.get(job.job_id) == job
|
||||
assert not migrated.published_results(
|
||||
source_session_id=job.source_session_id, source_catalog_sha256=job.source_catalog_sha256,
|
||||
setup_id=job.setup_id, definition_sha256=job.definition_sha256,
|
||||
source_session_id=job.source_session_id,
|
||||
source_catalog_sha256=job.source_catalog_sha256,
|
||||
setup_id=job.setup_id,
|
||||
definition_sha256=job.definition_sha256,
|
||||
)
|
||||
with sqlite3.connect(queue.database_path) as connection:
|
||||
names = {
|
||||
@@ -1123,9 +1179,7 @@ def test_legacy_sqlite_claim_schema_migrates_without_reusing_old_token(
|
||||
"claim_heartbeat_at_utc",
|
||||
"claim_renewal_count",
|
||||
):
|
||||
connection.execute(
|
||||
f"ALTER TABLE observatory_recorded_jobs DROP COLUMN {column}"
|
||||
)
|
||||
connection.execute(f"ALTER TABLE observatory_recorded_jobs DROP COLUMN {column}")
|
||||
connection.commit()
|
||||
|
||||
migrated_queue = _queue(tmp_path)
|
||||
@@ -1140,9 +1194,7 @@ def test_legacy_sqlite_claim_schema_migrates_without_reusing_old_token(
|
||||
with sqlite3.connect(migrated_queue.database_path) as connection:
|
||||
columns = {
|
||||
row[1]
|
||||
for row in connection.execute(
|
||||
"PRAGMA table_info(observatory_recorded_jobs)"
|
||||
).fetchall()
|
||||
for row in connection.execute("PRAGMA table_info(observatory_recorded_jobs)").fetchall()
|
||||
}
|
||||
assert {
|
||||
"claimed_at_utc",
|
||||
@@ -1171,9 +1223,7 @@ def test_legacy_sqlite_running_owner_migrates_to_reconciliation(
|
||||
"claim_heartbeat_at_utc",
|
||||
"claim_renewal_count",
|
||||
):
|
||||
connection.execute(
|
||||
f"ALTER TABLE observatory_recorded_jobs DROP COLUMN {column}"
|
||||
)
|
||||
connection.execute(f"ALTER TABLE observatory_recorded_jobs DROP COLUMN {column}")
|
||||
connection.commit()
|
||||
|
||||
migrated_queue = _queue(tmp_path)
|
||||
@@ -1201,15 +1251,10 @@ def test_single_worker_resource_has_only_one_recorded_owner(tmp_path: Path) -> N
|
||||
)
|
||||
queue.enqueue(first.job_id)
|
||||
queue.enqueue(second.job_id)
|
||||
first_claim = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="owner-poll-001"
|
||||
)
|
||||
first_claim = queue.claim_next(claimant_id="recorded-worker", claim_request_id="owner-poll-001")
|
||||
assert first_claim is not None
|
||||
assert (
|
||||
queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="owner-poll-002"
|
||||
)
|
||||
is None
|
||||
queue.claim_next(claimant_id="recorded-worker", claim_request_id="owner-poll-002") is None
|
||||
)
|
||||
|
||||
completed_job_id = first_claim.job.job_id
|
||||
@@ -1262,10 +1307,7 @@ def test_live_lease_cooperatively_pauses_and_resumes_recorded_job(tmp_path: Path
|
||||
)
|
||||
queue.enqueue(another.job_id)
|
||||
assert (
|
||||
queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="blocked-poll-001"
|
||||
)
|
||||
is None
|
||||
queue.claim_next(claimant_id="recorded-worker", claim_request_id="blocked-poll-001") is None
|
||||
)
|
||||
|
||||
completed = queue.finish_live(
|
||||
@@ -1319,9 +1361,7 @@ def test_live_request_pauses_claimed_job_before_execution(tmp_path: Path) -> Non
|
||||
queue = _queue(tmp_path)
|
||||
job, _ = queue.submit(_intent())
|
||||
queue.enqueue(job.job_id)
|
||||
claim = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="worker-claim-001"
|
||||
)
|
||||
claim = queue.claim_next(claimant_id="recorded-worker", claim_request_id="worker-claim-001")
|
||||
assert claim is not None
|
||||
|
||||
lease, _ = queue.request_live(_live_intent())
|
||||
@@ -1377,9 +1417,7 @@ def test_non_checkpointable_job_is_cancelled_and_restarts_from_zero_for_live(
|
||||
assert resumed.state == "queued"
|
||||
assert resumed.restart_from_zero is True
|
||||
|
||||
new_claim = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="worker-claim-002"
|
||||
)
|
||||
new_claim = queue.claim_next(claimant_id="recorded-worker", claim_request_id="worker-claim-002")
|
||||
assert new_claim is not None
|
||||
assert new_claim.job.job_id == running.job_id
|
||||
assert new_claim.job.restart_from_zero is True
|
||||
@@ -1520,15 +1558,16 @@ def test_reconciliation_required_quarantines_recorded_and_live_ownership(
|
||||
reason_code="worker-outcome-unknown",
|
||||
message="Worker ownership cannot be proven released.",
|
||||
)
|
||||
waiting, _ = queue.submit(
|
||||
_intent(idempotency_key="recorded-request-002")
|
||||
)
|
||||
waiting, _ = queue.submit(_intent(idempotency_key="recorded-request-002"))
|
||||
queue.enqueue(waiting.job_id)
|
||||
|
||||
assert queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="worker-claim-after-reconciliation",
|
||||
) is None
|
||||
assert (
|
||||
queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="worker-claim-after-reconciliation",
|
||||
)
|
||||
is None
|
||||
)
|
||||
lease, _ = queue.request_live(_live_intent())
|
||||
with pytest.raises(ObservatoryRecordedQueueBusyError, match="recorded work"):
|
||||
queue.activate_live(lease.lease_id)
|
||||
@@ -1558,8 +1597,7 @@ def test_queue_detects_mutated_immutable_identity(tmp_path: Path) -> None:
|
||||
job, _ = queue.submit(_intent())
|
||||
with sqlite3.connect(queue.database_path) as connection:
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs SET source_catalog_sha256 = ? "
|
||||
"WHERE job_id = ?",
|
||||
"UPDATE observatory_recorded_jobs SET source_catalog_sha256 = ? WHERE job_id = ?",
|
||||
("0" * 64, job.job_id),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Source-reuse installation admission only; no Docker/model runs."""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPTS = Path(__file__).parents[1] / "experiments/perception/worker/observatory_portable"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def installer(monkeypatch):
|
||||
monkeypatch.syspath_prepend(str(SCRIPTS))
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"recorded_source_reuse_installer", SCRIPTS / "install_recorded_source_reuse.py"
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, spec.name, module)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_pack_preserves_producer_and_rejects_payload_changes(installer, tmp_path):
|
||||
output = tmp_path / "payload"
|
||||
installer.pack(Path(__file__).parents[1], output)
|
||||
assert installer.PRODUCER not in installer.payload_files(output)
|
||||
assert len(installer.payload_files(output)) == 5
|
||||
path = output / "observatory/worker_source_cache.py"
|
||||
path.write_bytes(path.read_bytes() + b"\n# changed\n")
|
||||
with pytest.raises(ValueError, match="payload changed"):
|
||||
installer.payload_files(output)
|
||||
|
||||
|
||||
def test_source_reuse_manifest_cannot_add_paths(installer, tmp_path):
|
||||
output = tmp_path / "payload"
|
||||
installer.pack(Path(__file__).parents[1], output)
|
||||
path = output / "payload.json"
|
||||
manifest = json.loads(path.read_bytes())
|
||||
manifest["files"]["../../outside.py"] = "0" * 64
|
||||
path.write_text(json.dumps(manifest))
|
||||
with pytest.raises(ValueError, match="file set changed"):
|
||||
installer.payload_files(output)
|
||||
|
||||
|
||||
class FakeEngine:
|
||||
def __init__(self, installer):
|
||||
self.installer = installer
|
||||
self.requests = []
|
||||
self.volumes = []
|
||||
self.rows = {
|
||||
name: {
|
||||
"Id": str(index) * 64,
|
||||
"Name": "/" + name,
|
||||
"Image": "sha256:" + image,
|
||||
"State": {"Running": True},
|
||||
"Mounts": [],
|
||||
"HostConfig": {"ReadonlyRootfs": True, "NetworkMode": "bridge"},
|
||||
"Config": {"Labels": {"com.nodedc.authority": "observation-only"}, "Env": []},
|
||||
}
|
||||
for index, (name, image) in enumerate(installer.TARGETS.items(), 1)
|
||||
}
|
||||
|
||||
def inspect(self, name):
|
||||
return self.rows[name]
|
||||
|
||||
def request(self, method, path, body=None):
|
||||
self.requests.append((method, path))
|
||||
assert method == "GET"
|
||||
return {"Volumes": self.volumes}
|
||||
|
||||
def execute_json(self, name, source):
|
||||
if source == self.installer.probe():
|
||||
return {self.installer.PRODUCER: self.installer.PRODUCER_SHA, **self.installer.BEFORE}
|
||||
return {
|
||||
"schema_version": "missioncore.observatory-claim-readiness/v1",
|
||||
"open_live_lease_count": 0,
|
||||
"recorded_jobs_by_state": {"succeeded": 2},
|
||||
"protocols": [{}, {"grant_capacity_available": True}],
|
||||
}
|
||||
|
||||
|
||||
def test_plan_is_read_only_and_wrong_hash_cannot_cut_over(installer, tmp_path):
|
||||
output = tmp_path / "payload"
|
||||
installer.pack(Path(__file__).parents[1], output)
|
||||
engine = FakeEngine(installer)
|
||||
plan = installer.plan(engine, output)
|
||||
assert plan["compute_packages_changed"] is False
|
||||
assert plan["shared_cache"]["name"] == "ndc-observatory-source-cas-v1"
|
||||
assert plan["producer_sha256"] == installer.PRODUCER_SHA
|
||||
with pytest.raises(ValueError, match="plan changed"):
|
||||
installer.apply(engine, output, "0" * 64, tmp_path / "evidence")
|
||||
assert not (tmp_path / "evidence").exists()
|
||||
assert all(method == "GET" for method, _ in engine.requests)
|
||||
|
||||
|
||||
def test_fence_and_existing_volume_ownership_are_required(installer):
|
||||
engine = FakeEngine(installer)
|
||||
name = next(iter(installer.TARGETS))
|
||||
row = engine.inspect(name)
|
||||
target = {"name": name, "id": row["Id"], "create_sha256": installer.create_hash(row)}
|
||||
installer.fence(engine, target)
|
||||
row["HostConfig"]["Memory"] = 1024
|
||||
with pytest.raises(ValueError, match="changed since plan"):
|
||||
installer.fence(engine, target)
|
||||
engine.volumes = [{"Name": installer.VOLUME, "Driver": "local", "Labels": {}}]
|
||||
with pytest.raises(ValueError, match="another owner"):
|
||||
installer.volume_state(engine)
|
||||
@@ -0,0 +1,188 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import rerun as rr
|
||||
|
||||
from k1link.artifact_gateway import CentralArtifactStore
|
||||
from k1link.observatory.portable_object_replay import load_object_data, log_objects
|
||||
from k1link.observatory.portable_tgs_replay import PortableReplayError
|
||||
from k1link.perception.contracts import (
|
||||
BoundingRegion2D,
|
||||
EvidenceBasis,
|
||||
EvidenceCurrentness,
|
||||
MetricGeometry,
|
||||
ObjectProposal2D,
|
||||
ObstacleObservation,
|
||||
)
|
||||
|
||||
BUNDLE = "b" * 64
|
||||
|
||||
|
||||
def _fixture(tmp_path: Path, module_id: str, damage: str | None = None):
|
||||
store = CentralArtifactStore(tmp_path / "store", create=True)
|
||||
members: list[dict[str, object]] = []
|
||||
|
||||
def member(role: str, payload: object, media: str = "application/json") -> dict[str, object]:
|
||||
path = tmp_path / role
|
||||
encoded = payload if isinstance(payload, bytes) else json.dumps(payload).encode()
|
||||
path.write_bytes(encoded)
|
||||
published = store.publish_file(path)
|
||||
row = {
|
||||
"role": role,
|
||||
"sha256": published.sha256,
|
||||
"byte_length": published.byte_length,
|
||||
"media_type": media,
|
||||
}
|
||||
members.append(row)
|
||||
return row
|
||||
|
||||
proposals = []
|
||||
for index in range(2):
|
||||
proposal = ObjectProposal2D(
|
||||
proposal_id=f"proposal-{index}",
|
||||
source_id="recorded-k1",
|
||||
frame_id=f"frame-{index + 1:06d}",
|
||||
region=BoundingRegion2D(10.0, 20.0, 110.0, 220.0),
|
||||
objectness=0.9,
|
||||
provider_id="rf-detr-native",
|
||||
model_id="rf-detr-large",
|
||||
preprocess_id="kb4-native",
|
||||
semantic_hint="car",
|
||||
)
|
||||
proposals.append(proposal)
|
||||
detection_rows = [
|
||||
{
|
||||
"schema_version": "missioncore.observatory-ai-module-rf-detr-frame/v1",
|
||||
"frame_index": index,
|
||||
"session_seconds": 10.0 + index + (0.25 if damage == "clock" else 0),
|
||||
"proposals": [proposal.to_dict()],
|
||||
}
|
||||
for index, proposal in enumerate(proposals)
|
||||
]
|
||||
detections = member(
|
||||
"rf-detr-frame-detections",
|
||||
b"".join(json.dumps(row).encode() + b"\n" for row in detection_rows),
|
||||
"application/x-ndjson",
|
||||
)
|
||||
rf = member(
|
||||
"rf-detr-result-document",
|
||||
{
|
||||
"schema_version": "missioncore.observatory-ai-module-rf-detr-result/v1",
|
||||
"module_id": "rf-detr",
|
||||
"source": {"session_id": "source", "source_id": "recorded-k1"},
|
||||
"frame_count": 2,
|
||||
"detections_sha256": detections["sha256"],
|
||||
},
|
||||
)
|
||||
component = rf
|
||||
if module_id == "object-distance":
|
||||
observations = []
|
||||
for index, proposal in enumerate(proposals):
|
||||
observation = ObstacleObservation(
|
||||
observation_id=f"observation-{index}",
|
||||
occupancy_key=f"occupancy-{index}",
|
||||
source_id="recorded-k1",
|
||||
frame_id=proposal.frame_id,
|
||||
evidence_time_ns=(10 + index) * 1_000_000_000,
|
||||
basis=EvidenceBasis.FUSED,
|
||||
currentness=EvidenceCurrentness.CURRENT,
|
||||
occupied_support=True,
|
||||
source_point_ids=(index + 1,),
|
||||
metric_geometry=MetricGeometry(
|
||||
"camera", (0.0, 0.0, 12.5 + index), 12.5 + index, (0.1, 0.1, 0.1)
|
||||
),
|
||||
proposal_ids=(proposal.proposal_id,),
|
||||
semantic_hint="car",
|
||||
reason_codes=("qualified",),
|
||||
)
|
||||
observations.append(
|
||||
{
|
||||
"schema_version": "missioncore.observatory-ai-module-object-distance-frame/v1",
|
||||
"frame_index": index,
|
||||
"session_seconds": 10.0 + index,
|
||||
"observations": [observation.to_dict()],
|
||||
}
|
||||
)
|
||||
distance_rows = member(
|
||||
"object-distance-frame-observations",
|
||||
b"".join(json.dumps(row).encode() + b"\n" for row in observations),
|
||||
"application/x-ndjson",
|
||||
)
|
||||
component = member(
|
||||
"object-distance-result-document",
|
||||
{
|
||||
"schema_version": "missioncore.observatory-ai-module-object-distance-result/v1",
|
||||
"module_id": "object-distance",
|
||||
"source_session_id": "source",
|
||||
"frame_count": 2,
|
||||
"object_distances_sha256": distance_rows["sha256"],
|
||||
},
|
||||
)
|
||||
result_id = f"ai-layer-{module_id}-{'a' * 64}"
|
||||
view = {
|
||||
"result_id": result_id,
|
||||
"source_session_id": "source",
|
||||
"artifacts": members,
|
||||
"result_document": {
|
||||
"schema_version": "missioncore.recorded-ai-layer-review/v1",
|
||||
"result_id": result_id,
|
||||
"source": {
|
||||
"session_id": "source",
|
||||
"bundle_sha256": BUNDLE,
|
||||
"frame_count": 2,
|
||||
"timeline_start_seconds": 10.0,
|
||||
"timeline_end_seconds": 12.0,
|
||||
},
|
||||
"module": {
|
||||
"module_id": module_id,
|
||||
"component_result_sha256": component["sha256"],
|
||||
},
|
||||
},
|
||||
}
|
||||
return view, store
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module_id", ["rf-detr", "object-distance"])
|
||||
def test_object_replay_projects_boxes_and_optional_ranges(tmp_path: Path, module_id: str) -> None:
|
||||
view, store = _fixture(tmp_path, module_id)
|
||||
data = load_object_data(
|
||||
view, store, source_bundle_sha256=BUNDLE, starts=[10.0, 11.0], end_seconds=12.0
|
||||
)
|
||||
assert data.include_ranges is (module_id == "object-distance")
|
||||
assert data.frames[0].ranges_m == (
|
||||
{"proposal-0": 12.5} if module_id == "object-distance" else {}
|
||||
)
|
||||
path = tmp_path / "objects.rrd"
|
||||
recording = rr.RecordingStream("nodedc_mission_core_recorded", recording_id=module_id)
|
||||
recording.set_sinks(rr.FileSink(path, write_footer=True))
|
||||
calls: list[tuple[str, object]] = []
|
||||
native_log = recording.log
|
||||
|
||||
def capture(entity: str, value: object, **kwargs: object) -> None:
|
||||
calls.append((entity, value))
|
||||
native_log(entity, value, **kwargs)
|
||||
|
||||
recording.log = capture
|
||||
log_objects(recording, data)
|
||||
recording.flush()
|
||||
recording.disconnect()
|
||||
assert path.read_bytes().startswith(b"RRF2")
|
||||
assert sum(isinstance(value, rr.Boxes2D) for _, value in calls) == 2
|
||||
assert isinstance(calls[-1][1], rr.Clear)
|
||||
assert {entity for entity, _ in calls} == {"/perception/camera/detections"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("damage", ["clock", "bundle"])
|
||||
def test_object_replay_rejects_changed_source_or_clock(tmp_path: Path, damage: str) -> None:
|
||||
view, store = _fixture(tmp_path, "object-distance", damage)
|
||||
with pytest.raises(PortableReplayError):
|
||||
load_object_data(
|
||||
view,
|
||||
store,
|
||||
source_bundle_sha256="c" * 64 if damage == "bundle" else BUNDLE,
|
||||
starts=[10.0, 11.0],
|
||||
end_seconds=12.0,
|
||||
)
|
||||
@@ -210,7 +210,8 @@ def test_camera_proxy_keeps_demux_time_base_and_reuses_its_own_sealed_cache(tmp_
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def test_replay_api_get_never_prepares_and_range_is_generation_fenced(tmp_path):
|
||||
@pytest.mark.parametrize("prefix", ["m49-tgs-portable-review", "lab-v1-eomt-ddrnet"])
|
||||
def test_replay_api_get_never_prepares_and_range_is_generation_fenced(tmp_path, prefix):
|
||||
path = tmp_path / "replay.rrd"
|
||||
path.write_bytes(b"RRF2test")
|
||||
sha = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
@@ -228,7 +229,8 @@ def test_replay_api_get_never_prepares_and_range_is_generation_fenced(tmp_path):
|
||||
)
|
||||
)
|
||||
with TestClient(app) as client:
|
||||
url = f"/api/v1/observatory/portable-results/{RESULT}/replays/{BASE}/recording.rrd"
|
||||
result_id = prefix + "-" + "a" * 64
|
||||
url = f"/api/v1/observatory/portable-results/{result_id}/replays/{BASE}/recording.rrd"
|
||||
assert client.get(url, params={"generation": sha}).status_code == 409
|
||||
assert not prepared
|
||||
response = client.head(url)
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import tarfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import rerun as rr
|
||||
from PIL import Image
|
||||
|
||||
from k1link.artifact_gateway import CentralArtifactStore
|
||||
from k1link.observatory.portable_semantic_replay import (
|
||||
EOMT_LABELS,
|
||||
EOMT_PROFILE_SHA,
|
||||
RESULT_SCHEMA,
|
||||
load_semantic_data,
|
||||
log_semantics,
|
||||
)
|
||||
from k1link.observatory.portable_tgs_replay import PortableReplayError
|
||||
|
||||
RESULT = "lab-v1-eomt-ddrnet-" + "a" * 64
|
||||
BUNDLE = "b" * 64
|
||||
|
||||
|
||||
def fixture(tmp_path: Path, damage: str | None = None):
|
||||
store = CentralArtifactStore(tmp_path / "store", create=True)
|
||||
members = []
|
||||
|
||||
def member(role, payload, media="application/json"):
|
||||
path = tmp_path / role
|
||||
path.write_bytes(payload if isinstance(payload, bytes) else json.dumps(payload).encode())
|
||||
published = store.publish_file(path)
|
||||
value = {
|
||||
"role": role,
|
||||
"sha256": published.sha256,
|
||||
"byte_length": published.byte_length,
|
||||
"media_type": media,
|
||||
}
|
||||
members.append(value)
|
||||
return value
|
||||
|
||||
png = io.BytesIO()
|
||||
Image.fromarray(np.ones((600, 800), dtype=np.uint8)).save(png, format="PNG")
|
||||
archive = io.BytesIO()
|
||||
with tarfile.open(fileobj=archive, mode="w:gz") as stream:
|
||||
for index in (1, 0):
|
||||
if damage == "missing-mask" and index == 0:
|
||||
continue
|
||||
name = f"semantic-masks/frame-{index + 1:06d}.png"
|
||||
if damage == "unsafe-mask" and index == 0:
|
||||
name = "../frame-000001.png"
|
||||
if damage == "wrong-suffix" and index == 0:
|
||||
name = "semantic-masks/frame-000001Xpng"
|
||||
item = tarfile.TarInfo(name)
|
||||
item.size = len(png.getvalue())
|
||||
if damage == "oversized-mask" and index == 0:
|
||||
item.size = 1024 * 1024 + 1
|
||||
stream.addfile(item, io.BytesIO(b"x" * item.size))
|
||||
else:
|
||||
stream.addfile(item, io.BytesIO(png.getvalue()))
|
||||
member("eomt-panoptic-mask-archive", archive.getvalue(), "application/gzip")
|
||||
archive = io.BytesIO()
|
||||
with zipfile.ZipFile(archive, "w") as stream:
|
||||
for index in range(2):
|
||||
stream.writestr(f"masks/frame-{index + 1:06d}.png", png.getvalue())
|
||||
ddr_archive = member("ddrnet-semantic-mask-archive", archive.getvalue(), "application/zip")
|
||||
eomt = member(
|
||||
"eomt-result-document",
|
||||
{
|
||||
"result_id": "eomt-result",
|
||||
"frames_processed": 2,
|
||||
"session_id": "source",
|
||||
"input_sha256": "camera",
|
||||
"timestamp_basis": "session-time-seconds",
|
||||
"identity": {
|
||||
"configuration": {
|
||||
"profile_sha256": "d" * 64 if damage == "profile" else EOMT_PROFILE_SHA
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
ddrnet = member(
|
||||
"ddrnet-result-document",
|
||||
{
|
||||
"result_id": "ddrnet-result",
|
||||
"video_semantics": {
|
||||
"mask_archive": {
|
||||
**ddr_archive,
|
||||
"frame_count": 2,
|
||||
"width": 800,
|
||||
"height": 600,
|
||||
"encoding": "uint8-class-id-png",
|
||||
},
|
||||
"taxonomy": {
|
||||
"schema_version": "missioncore.lab-v1-vegetation-taxonomy/v1",
|
||||
"classes": [
|
||||
{"class_id": i, "label": str(i), "color_rgb": [64, 128, 192]}
|
||||
for i in range(64)
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
frames = [
|
||||
{
|
||||
"frame_index": i,
|
||||
"sequence": i + 1,
|
||||
"session_seconds": 10 + i + (0.1 if damage == "clock" else 0),
|
||||
"semantic_classes": [{"id": 1, "label": "car" if damage == "label" else "person"}],
|
||||
}
|
||||
for i in range(2)
|
||||
]
|
||||
if damage == "extra-frame":
|
||||
frames.append(frames[-1])
|
||||
member(
|
||||
"eomt-panoptic-frame-metadata",
|
||||
b"".join(json.dumps(row).encode() + b"\n" for row in frames),
|
||||
"application/x-ndjson",
|
||||
)
|
||||
view = {
|
||||
"result_id": RESULT,
|
||||
"source_session_id": "source",
|
||||
"artifacts": members,
|
||||
"result_document": {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"result_id": RESULT,
|
||||
"source": {
|
||||
"session_id": "source",
|
||||
"bundle_sha256": BUNDLE,
|
||||
"camera_input_sha256": "camera",
|
||||
"frame_count": 2,
|
||||
"timeline_start_seconds": 10.0,
|
||||
"timeline_end_seconds": 12.0,
|
||||
},
|
||||
"components": {
|
||||
"eomt": {
|
||||
"result_id": "eomt-result",
|
||||
"frames_processed": 2,
|
||||
"result_document_sha256": eomt["sha256"],
|
||||
},
|
||||
"ddrnet": {
|
||||
"result_id": "ddrnet-result",
|
||||
"frames_processed": 2,
|
||||
"result_document_sha256": ddrnet["sha256"],
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return view, store
|
||||
|
||||
|
||||
def test_target_labels_match_the_exact_checked_in_producer_profile():
|
||||
path = Path(__file__).parents[1] / "experiments/perception/worker/e3_k1_camera1_profile.json"
|
||||
payload = path.read_bytes()
|
||||
assert hashlib.sha256(payload).hexdigest() == EOMT_PROFILE_SHA
|
||||
assert tuple(json.loads(payload)["target_taxonomy"].values()) == EOMT_LABELS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("damage", ["clock", "profile", "label", "extra-frame", "bundle"])
|
||||
def test_mask_review_rejects_foreign_profile_source_or_clock(tmp_path, damage):
|
||||
view, store = fixture(tmp_path, damage)
|
||||
with pytest.raises(PortableReplayError):
|
||||
load_semantic_data(
|
||||
view,
|
||||
store,
|
||||
starts=[10.0, 11.0],
|
||||
end_seconds=12.0,
|
||||
source_bundle_sha256="c" * 64 if damage == "bundle" else BUNDLE,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"damage", [None, "missing-mask", "unsafe-mask", "oversized-mask", "wrong-suffix"]
|
||||
)
|
||||
def test_native_projection_is_complete_compressed_and_cleared_at_coverage_end(tmp_path, damage):
|
||||
view, store = fixture(tmp_path, damage)
|
||||
data = load_semantic_data(
|
||||
view, store, starts=[10.0, 11.0], end_seconds=12.0, source_bundle_sha256=BUNDLE
|
||||
)
|
||||
path = tmp_path / "recording.rrd"
|
||||
recording = rr.RecordingStream("nodedc_mission_core_recorded", recording_id="fixture")
|
||||
recording.set_sinks(rr.FileSink(path, write_footer=True))
|
||||
calls, times = [], []
|
||||
original_log, original_time = recording.log, recording.set_time
|
||||
|
||||
def log(entity, value, **kwargs):
|
||||
calls.append((entity, value))
|
||||
original_log(entity, value, **kwargs)
|
||||
|
||||
def timestamp(name, *, duration):
|
||||
times.append(int(duration.astype("timedelta64[ns]").astype(np.int64)))
|
||||
original_time(name, duration=duration)
|
||||
|
||||
recording.log, recording.set_time = log, timestamp
|
||||
try:
|
||||
if damage:
|
||||
with pytest.raises(PortableReplayError):
|
||||
log_semantics(recording, data)
|
||||
else:
|
||||
log_semantics(recording, data)
|
||||
recording.flush()
|
||||
assert path.read_bytes().startswith(b"RRF2")
|
||||
assert sum(isinstance(value, rr.EncodedImage) for _, value in calls) == 4
|
||||
assert times == [
|
||||
11_000_000_000,
|
||||
10_000_000_000,
|
||||
10_000_000_000,
|
||||
11_000_000_000,
|
||||
12_000_000_000,
|
||||
]
|
||||
assert isinstance(calls[-1][1], rr.Clear)
|
||||
assert all(entity.startswith("/perception/camera/segmentation") for entity, _ in calls)
|
||||
finally:
|
||||
recording.disconnect()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module_id", "result_role", "expected_layers"),
|
||||
[
|
||||
("eomt", "eomt-result-document", {"city"}),
|
||||
("ddrnet", "ddrnet-result-document", {"vegetation"}),
|
||||
],
|
||||
)
|
||||
def test_independent_ai_module_result_opens_only_its_own_semantic_layer(
|
||||
tmp_path: Path,
|
||||
module_id: str,
|
||||
result_role: str,
|
||||
expected_layers: set[str],
|
||||
) -> None:
|
||||
view, store = fixture(tmp_path)
|
||||
result_id = f"ai-layer-{module_id}-{'c' * 64}"
|
||||
result_artifact = next(item for item in view["artifacts"] if item["role"] == result_role)
|
||||
view["result_id"] = result_id
|
||||
view["result_document"] = {
|
||||
"schema_version": "missioncore.recorded-ai-layer-review/v1",
|
||||
"result_id": result_id,
|
||||
"source": {
|
||||
"session_id": "source",
|
||||
"bundle_sha256": BUNDLE,
|
||||
"camera_input_sha256": "camera",
|
||||
"frame_count": 2,
|
||||
"timeline_start_seconds": 10.0,
|
||||
"timeline_end_seconds": 12.0,
|
||||
},
|
||||
"module": {
|
||||
"module_id": module_id,
|
||||
"component_result_sha256": result_artifact["sha256"],
|
||||
},
|
||||
}
|
||||
data = load_semantic_data(
|
||||
view,
|
||||
store,
|
||||
starts=[10.0, 11.0],
|
||||
end_seconds=12.0,
|
||||
source_bundle_sha256=BUNDLE,
|
||||
)
|
||||
assert set(data.classes) == expected_layers
|
||||
assert (data.city_masks is not None) is (module_id == "eomt")
|
||||
assert (data.vegetation_masks is not None) is (module_id == "ddrnet")
|
||||
@@ -0,0 +1,116 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import threading
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.viewer.recorded_blueprint_lifecycle import (
|
||||
BlueprintSessionReleased,
|
||||
RecordedBlueprintSessions,
|
||||
)
|
||||
|
||||
KEY = ("app", "recording", "a" * 32)
|
||||
|
||||
|
||||
class Resource:
|
||||
def __init__(self) -> None:
|
||||
self.closes = 0
|
||||
|
||||
def close(self) -> None:
|
||||
self.closes += 1
|
||||
|
||||
|
||||
def test_release_drops_resource_and_fences_late_updates() -> None:
|
||||
sessions = RecordedBlueprintSessions[Resource]()
|
||||
resource = Resource()
|
||||
reference = weakref.ref(resource)
|
||||
assert (
|
||||
sessions.use(KEY, 0, lambda resource=resource: resource, lambda _: b"result") == b"result"
|
||||
)
|
||||
sessions.release(KEY)
|
||||
sessions.release(KEY)
|
||||
assert resource.closes == 1
|
||||
assert not sessions.renew(KEY)
|
||||
with pytest.raises(BlueprintSessionReleased):
|
||||
sessions.use(KEY, 0, Resource, lambda _: b"late")
|
||||
del resource
|
||||
gc.collect()
|
||||
assert reference() is None
|
||||
|
||||
|
||||
def test_renew_keeps_paused_viewport_but_abandoned_owner_expires() -> None:
|
||||
now = [0.0]
|
||||
sessions = RecordedBlueprintSessions[Resource](clock=lambda: now[0], ttl_seconds=300)
|
||||
resource = Resource()
|
||||
sessions.use(KEY, 0, lambda: resource, lambda _: None)
|
||||
for timestamp in [200, 400, 600]:
|
||||
now[0] = timestamp
|
||||
assert sessions.renew(KEY)
|
||||
sessions.expire()
|
||||
assert resource.closes == 0
|
||||
now[0] = 901
|
||||
sessions.expire()
|
||||
assert resource.closes == 1
|
||||
assert not sessions.renew(KEY)
|
||||
# An expired blueprint is rebuildable; this is not a recording/live stop.
|
||||
sessions.use(KEY, 0, Resource, lambda _: None)
|
||||
sessions.close()
|
||||
|
||||
|
||||
def test_failed_render_releases_partial_resource_and_can_retry() -> None:
|
||||
sessions = RecordedBlueprintSessions[Resource]()
|
||||
resource = Resource()
|
||||
|
||||
def fail(_: Resource) -> None:
|
||||
raise ValueError("failed serialization")
|
||||
|
||||
with pytest.raises(ValueError, match="failed serialization"):
|
||||
sessions.use(KEY, 0, lambda: resource, fail)
|
||||
assert resource.closes == 1
|
||||
assert not sessions.renew(KEY)
|
||||
sessions.use(KEY, 0, Resource, lambda _: None)
|
||||
sessions.close()
|
||||
|
||||
|
||||
def test_reset_lru_and_shutdown_release_resources_exactly_once() -> None:
|
||||
sessions = RecordedBlueprintSessions[Resource](max_entries=1)
|
||||
first, replacement, other = Resource(), Resource(), Resource()
|
||||
sessions.use(KEY, 0, lambda: first, lambda _: None)
|
||||
sessions.use(KEY, 1, lambda: replacement, lambda _: None)
|
||||
assert first.closes == 1
|
||||
sessions.use(("app", "other", "b" * 32), 0, lambda: other, lambda _: None)
|
||||
assert replacement.closes == 1
|
||||
sessions.close()
|
||||
sessions.close()
|
||||
assert other.closes == 1
|
||||
|
||||
|
||||
def test_release_waits_for_render_and_prevents_resurrection() -> None:
|
||||
sessions = RecordedBlueprintSessions[Resource]()
|
||||
resource = Resource()
|
||||
started, finish, released = threading.Event(), threading.Event(), threading.Event()
|
||||
|
||||
def render(_: Resource) -> None:
|
||||
started.set()
|
||||
assert finish.wait(2)
|
||||
assert resource.closes == 0
|
||||
|
||||
def release() -> None:
|
||||
sessions.release(KEY)
|
||||
released.set()
|
||||
|
||||
worker = threading.Thread(target=lambda: sessions.use(KEY, 0, lambda: resource, render))
|
||||
worker.start()
|
||||
assert started.wait(2)
|
||||
closer = threading.Thread(target=release)
|
||||
closer.start()
|
||||
assert not released.is_set()
|
||||
finish.set()
|
||||
worker.join(2)
|
||||
closer.join(2)
|
||||
assert not worker.is_alive() and not closer.is_alive()
|
||||
assert released.is_set() and resource.closes == 1
|
||||
with pytest.raises(BlueprintSessionReleased):
|
||||
sessions.use(KEY, 0, Resource, lambda _: None)
|
||||
@@ -0,0 +1,86 @@
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import rerun as rr
|
||||
|
||||
import k1link.viewer.recorded_camera_bounds as camera_bounds
|
||||
from k1link.viewer.recorded_camera_bounds import recorded_orbital_radius_limit
|
||||
|
||||
|
||||
def _recording(path: Path) -> None:
|
||||
recording = rr.RecordingStream("camera-bounds-test", recording_id="camera-bounds")
|
||||
recording.set_sinks(rr.FileSink(path, write_footer=True))
|
||||
try:
|
||||
recording.set_time("session_time", duration=np.timedelta64(10, "s"))
|
||||
recording.log("/world/points", rr.Points3D([[0, 0, 0], [3, 4, 0]]))
|
||||
recording.set_time("session_time", duration=np.timedelta64(20, "s"))
|
||||
recording.log("/world/points", rr.Points3D([[10, 0, 0], [10, 0, 12]]))
|
||||
recording.flush(timeout_sec=5)
|
||||
finally:
|
||||
recording.disconnect()
|
||||
|
||||
|
||||
def test_recorded_orbital_radius_uses_visible_accumulation_window(tmp_path: Path) -> None:
|
||||
path = tmp_path / "recording.rrd"
|
||||
_recording(path)
|
||||
|
||||
accumulated = recorded_orbital_radius_limit(
|
||||
path,
|
||||
current_time_ns=20_000_000_000,
|
||||
accumulation_seconds=15,
|
||||
show_points=True,
|
||||
show_trajectory=False,
|
||||
)
|
||||
latest = recorded_orbital_radius_limit(
|
||||
path,
|
||||
current_time_ns=20_000_000_000,
|
||||
accumulation_seconds=0,
|
||||
show_points=True,
|
||||
show_trajectory=False,
|
||||
)
|
||||
|
||||
assert accumulated == pytest.approx(5 * np.sqrt(10**2 + 4**2 + 12**2), rel=1e-6)
|
||||
assert latest == pytest.approx(60, rel=1e-6)
|
||||
|
||||
|
||||
def test_recorded_orbital_radius_is_absent_without_mapping_layers(tmp_path: Path) -> None:
|
||||
path = tmp_path / "recording.rrd"
|
||||
_recording(path)
|
||||
assert (
|
||||
recorded_orbital_radius_limit(
|
||||
path,
|
||||
current_time_ns=20_000_000_000,
|
||||
accumulation_seconds=15,
|
||||
show_points=False,
|
||||
show_trajectory=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_recorded_camera_queries_reuse_the_generation_bounds_index(tmp_path: Path) -> None:
|
||||
path = tmp_path / "recording.rrd"
|
||||
_recording(path)
|
||||
camera_bounds._recorded_spatial_bounds_index.cache_clear()
|
||||
|
||||
first = recorded_orbital_radius_limit(
|
||||
path,
|
||||
current_time_ns=10_000_000_000,
|
||||
accumulation_seconds=0,
|
||||
show_points=True,
|
||||
show_trajectory=False,
|
||||
)
|
||||
second = recorded_orbital_radius_limit(
|
||||
path,
|
||||
current_time_ns=20_000_000_000,
|
||||
accumulation_seconds=0,
|
||||
show_points=True,
|
||||
show_trajectory=False,
|
||||
)
|
||||
|
||||
cache = camera_bounds._recorded_spatial_bounds_index.cache_info()
|
||||
assert first == pytest.approx(25, rel=1e-6)
|
||||
assert second == pytest.approx(60, rel=1e-6)
|
||||
assert cache.misses == 1
|
||||
assert cache.hits == 1
|
||||
@@ -499,6 +499,30 @@ def test_recorded_follow_mode_tracks_sensor_pose_with_the_same_orbital_eye() ->
|
||||
}
|
||||
|
||||
|
||||
def test_recorded_layer_blueprint_can_carry_the_operator_eye() -> None:
|
||||
blueprint = viewer_recorded_blueprint(
|
||||
RerunSceneSettings(),
|
||||
include_initial_playback_state=False,
|
||||
unified_perception=True,
|
||||
update_eye_controls=False,
|
||||
eye_position=(3.0, 4.0, 5.0),
|
||||
eye_look_target=(1.0, 2.0, 0.0),
|
||||
eye_up=(0.0, 0.0, 1.0),
|
||||
)
|
||||
eye = blueprint.root_container.contents[1].properties["EyeControls3D"]
|
||||
components = {
|
||||
str(batch.component_descriptor()): batch.as_arrow_array().to_pylist()
|
||||
for batch in eye.as_component_batches()
|
||||
}
|
||||
assert components == {
|
||||
"EyeControls3D:kind": [2],
|
||||
"EyeControls3D:position": [[3.0, 4.0, 5.0]],
|
||||
"EyeControls3D:look_target": [[1.0, 2.0, 0.0]],
|
||||
"EyeControls3D:eye_up": [[0.0, 0.0, 1.0]],
|
||||
"EyeControls3D:tracking_entity": [""],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reactivate_updates", [False, True])
|
||||
def test_recorded_blueprint_updates_reuse_source_store_and_opt_in_to_activation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -601,9 +625,10 @@ def test_recorded_blueprint_updates_reuse_source_store_and_opt_in_to_activation(
|
||||
assert len(reset_store_ids) == 1
|
||||
assert reset_store_ids.isdisjoint(initial_store_ids)
|
||||
assert eye_control_updates == [True, False, True, False, True]
|
||||
assert explicit_presets == [
|
||||
False, reactivate_updates, reactivate_updates, reactivate_updates, False
|
||||
]
|
||||
# Reactivating a blueprint clone is required for visible layer changes, but
|
||||
# those changes must not write position/look-target/eye-up and reset the
|
||||
# operator's camera. Only the follow transition writes a spatial preset.
|
||||
assert explicit_presets == [False, False, True, False, False]
|
||||
assert [activation[1:] for activation in activations] == [
|
||||
(True, False),
|
||||
(reactivate_updates, False),
|
||||
@@ -644,18 +669,21 @@ def test_viewer_blueprint_unifies_original_video_and_independent_ai_layers() ->
|
||||
RerunSceneSettings(accumulation_seconds=12.0),
|
||||
include_initial_playback_state=False,
|
||||
unified_perception=True,
|
||||
unified_camera_share=0.73,
|
||||
show_camera_image=False,
|
||||
show_detections_2d=True,
|
||||
show_segmentation=True,
|
||||
show_cuboids_3d=False,
|
||||
)
|
||||
|
||||
assert type(blueprint.root_container).__name__ == "Horizontal"
|
||||
assert blueprint.root_container.column_shares == pytest.approx([0.73, 0.27])
|
||||
camera_view, spatial_view = blueprint.root_container.contents
|
||||
assert camera_view.origin == "/perception/camera"
|
||||
assert spatial_view.origin == "/world"
|
||||
assert camera_view.visualizer_overrides[
|
||||
"/perception/camera/image"
|
||||
].visible.as_arrow_array().to_pylist() == [True]
|
||||
].visible.as_arrow_array().to_pylist() == [False]
|
||||
assert camera_view.visualizer_overrides[
|
||||
"/perception/camera/detections"
|
||||
].visible.as_arrow_array().to_pylist() == [True]
|
||||
|
||||
+227
-42
@@ -9,6 +9,7 @@ import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@@ -41,6 +42,7 @@ from k1link.web.camera_archive import CameraArchiveWriter
|
||||
from k1link.web.observatory_api import build_observatory_router
|
||||
from k1link.web.session_api import (
|
||||
LayoutPutRequest,
|
||||
RecordedBlueprintLifecycleRequest,
|
||||
RecordedBlueprintRequest,
|
||||
RecordedPerceptionRequest,
|
||||
RecordedPointColorsRequest,
|
||||
@@ -441,20 +443,15 @@ def test_session_router_versions_capability_and_calculation_profile_projections(
|
||||
)["items"]
|
||||
v3_by_id = {item["id"]: item for item in v3_labs}
|
||||
assert v3_by_id[legacy.session_id]["lab"]["calculation_profile"] is None
|
||||
assert (
|
||||
v3_by_id[canonical.session_id]["lab"]["calculation_profile"]
|
||||
== calculation_profile
|
||||
)
|
||||
assert (
|
||||
v3_by_id[canonical.session_id]["lab"]["replay_capability"]
|
||||
== capability.as_dict()
|
||||
)
|
||||
assert v3_by_id[canonical.session_id]["lab"]["calculation_profile"] == calculation_profile
|
||||
assert v3_by_id[canonical.session_id]["lab"]["replay_capability"] == capability.as_dict()
|
||||
|
||||
application = FastAPI()
|
||||
application.include_router(router)
|
||||
assert TestClient(application).get(
|
||||
"/api/v1/observation-sessions?lab_contract=v4"
|
||||
).status_code == 422
|
||||
assert (
|
||||
TestClient(application).get("/api/v1/observation-sessions?lab_contract=v4").status_code
|
||||
== 422
|
||||
)
|
||||
|
||||
|
||||
def test_observatory_projection_api_renames_alias_and_deletes_only_projection(
|
||||
@@ -508,28 +505,37 @@ def test_observatory_projection_api_renames_alias_and_deletes_only_projection(
|
||||
client = TestClient(application)
|
||||
url = f"/api/v1/observatory/lab-projections/{projection.session_id}"
|
||||
|
||||
assert client.patch(
|
||||
url,
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-lab-projection-rename/v0",
|
||||
"display_name": "Неверная версия",
|
||||
},
|
||||
).status_code == 422
|
||||
assert client.patch(
|
||||
url,
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-lab-projection-rename/v1",
|
||||
"display_name": "Новый разбор",
|
||||
"unexpected": True,
|
||||
},
|
||||
).status_code == 422
|
||||
assert client.patch(
|
||||
url,
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-lab-projection-rename/v1",
|
||||
"display_name": " ",
|
||||
},
|
||||
).status_code == 422
|
||||
assert (
|
||||
client.patch(
|
||||
url,
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-lab-projection-rename/v0",
|
||||
"display_name": "Неверная версия",
|
||||
},
|
||||
).status_code
|
||||
== 422
|
||||
)
|
||||
assert (
|
||||
client.patch(
|
||||
url,
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-lab-projection-rename/v1",
|
||||
"display_name": "Новый разбор",
|
||||
"unexpected": True,
|
||||
},
|
||||
).status_code
|
||||
== 422
|
||||
)
|
||||
assert (
|
||||
client.patch(
|
||||
url,
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-lab-projection-rename/v1",
|
||||
"display_name": " ",
|
||||
},
|
||||
).status_code
|
||||
== 422
|
||||
)
|
||||
|
||||
renamed = client.patch(
|
||||
url,
|
||||
@@ -785,16 +791,16 @@ def test_canonical_lab_spatial_frame_uses_ready_immutable_recording(
|
||||
"GET",
|
||||
)
|
||||
try:
|
||||
response = asyncio.run(spatial_route(
|
||||
session_id=session.name,
|
||||
generation=generation,
|
||||
time_ns=500_000_000,
|
||||
profile="source-paced-ground-v3",
|
||||
))
|
||||
assert json.loads(response.body) == expected
|
||||
assert response.headers["etag"] == (
|
||||
f'"{generation}:source-paced-ground-v3:499000000"'
|
||||
response = asyncio.run(
|
||||
spatial_route(
|
||||
session_id=session.name,
|
||||
generation=generation,
|
||||
time_ns=500_000_000,
|
||||
profile="source-paced-ground-v3",
|
||||
)
|
||||
)
|
||||
assert json.loads(response.body) == expected
|
||||
assert response.headers["etag"] == (f'"{generation}:source-paced-ground-v3:499000000"')
|
||||
assert response.headers["cache-control"].endswith("immutable")
|
||||
finally:
|
||||
manager.close()
|
||||
@@ -1561,6 +1567,89 @@ def test_production_router_never_materializes_recording_inline(tmp_path: Path) -
|
||||
assert calls == 0
|
||||
|
||||
|
||||
def test_blueprint_lifecycle_releases_without_repreparing_source(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls: list[tuple[str, tuple[str, str, str]]] = []
|
||||
|
||||
class Sessions:
|
||||
def release(self, key: tuple[str, str, str]) -> None:
|
||||
calls.append(("release", key))
|
||||
|
||||
def renew(self, key: tuple[str, str, str]) -> bool:
|
||||
calls.append(("renew", key))
|
||||
return True
|
||||
|
||||
def forbidden_prepare(*args: object, **kwargs: object) -> None:
|
||||
pytest.fail("lifecycle must not reprepare a recording")
|
||||
|
||||
monkeypatch.setattr("k1link.web.session_api.recorded_blueprint_sessions", Sessions())
|
||||
monkeypatch.setattr("k1link.web.session_api._prepare_replay", forbidden_prepare)
|
||||
store = SessionStore(tmp_path / "repo", data_dir=tmp_path / "data")
|
||||
route = endpoint(
|
||||
build_session_router(store),
|
||||
"/api/v1/observation-sessions/{session_id}/blueprint-lifecycle",
|
||||
"POST",
|
||||
)
|
||||
for action in ("renew", "release"):
|
||||
request = RecordedBlueprintLifecycleRequest.model_validate(
|
||||
{
|
||||
"action": action,
|
||||
"application_id": "nodedc_mission_core_recorded",
|
||||
"recording_id": "recording",
|
||||
"blueprint_session_id": "f" * 32,
|
||||
}
|
||||
)
|
||||
response = asyncio.run(route(session_id="removed-source", request=request))
|
||||
assert response.status_code == 204
|
||||
assert response.headers["cache-control"] == "no-store"
|
||||
assert calls == [
|
||||
(action, ("nodedc_mission_core_recorded", "recording", "f" * 32))
|
||||
for action in ("renew", "release")
|
||||
]
|
||||
with pytest.raises(HTTPException) as error:
|
||||
asyncio.run(route(session_id="../unsafe", request=request))
|
||||
assert error.value.status_code == 422
|
||||
with pytest.raises(ValueError):
|
||||
RecordedBlueprintLifecycleRequest.model_validate({**request.model_dump(), "path": "/tmp"})
|
||||
|
||||
|
||||
def test_released_blueprint_update_returns_gone(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from k1link.viewer.recorded_blueprint_lifecycle import BlueprintSessionReleased
|
||||
|
||||
def released(*args: object, **kwargs: object) -> bytes:
|
||||
raise BlueprintSessionReleased("closed")
|
||||
|
||||
monkeypatch.setattr("k1link.web.session_api._prepare_replay", lambda *args: None)
|
||||
monkeypatch.setattr("k1link.web.session_api.recorded_blueprint_rrd", released)
|
||||
store = SessionStore(tmp_path / "repo", data_dir=tmp_path / "data")
|
||||
route = endpoint(
|
||||
build_session_router(store),
|
||||
"/api/v1/observation-sessions/{session_id}/blueprint.rrd",
|
||||
"POST",
|
||||
)
|
||||
with pytest.raises(HTTPException) as error:
|
||||
asyncio.run(
|
||||
route(
|
||||
session_id="source",
|
||||
request=RecordedBlueprintRequest(
|
||||
application_id="nodedc_mission_core_recorded",
|
||||
recording_id="recording",
|
||||
blueprint_session_id="f" * 32,
|
||||
accumulation_seconds=0.0,
|
||||
show_points=True,
|
||||
show_trajectory=False,
|
||||
show_grid=True,
|
||||
),
|
||||
)
|
||||
)
|
||||
assert error.value.status_code == 410
|
||||
|
||||
|
||||
def test_recorded_blueprint_endpoint_is_small_strict_and_session_scoped(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -1607,10 +1696,14 @@ def test_recorded_blueprint_endpoint_is_small_strict_and_session_scoped(
|
||||
active_view="perception",
|
||||
view_reset_generation=1,
|
||||
unified_perception=True,
|
||||
unified_camera_share=0.73,
|
||||
show_detections_2d=True,
|
||||
show_segmentation=True,
|
||||
show_cuboids_3d=True,
|
||||
follow_trajectory=True,
|
||||
eye_position=(3.0, 4.0, 5.0),
|
||||
eye_look_target=(1.0, 2.0, 0.0),
|
||||
eye_up=(0.0, 0.0, 1.0),
|
||||
),
|
||||
)
|
||||
)
|
||||
@@ -1629,10 +1722,14 @@ def test_recorded_blueprint_endpoint_is_small_strict_and_session_scoped(
|
||||
assert observed_kwargs[0]["active_view"] == "perception"
|
||||
assert observed_kwargs[0]["view_reset_generation"] == 1
|
||||
assert observed_kwargs[0]["unified_perception"] is True
|
||||
assert observed_kwargs[0]["unified_camera_share"] == pytest.approx(0.73)
|
||||
assert observed_kwargs[0]["show_detections_2d"] is True
|
||||
assert observed_kwargs[0]["show_segmentation"] is True
|
||||
assert observed_kwargs[0]["show_cuboids_3d"] is True
|
||||
assert observed_kwargs[0]["follow_trajectory"] is True
|
||||
assert observed_kwargs[0]["eye_position"] == (3.0, 4.0, 5.0)
|
||||
assert observed_kwargs[0]["eye_look_target"] == (1.0, 2.0, 0.0)
|
||||
assert observed_kwargs[0]["eye_up"] == (0.0, 0.0, 1.0)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
RecordedBlueprintRequest.model_validate(
|
||||
@@ -1644,6 +1741,32 @@ def test_recorded_blueprint_endpoint_is_small_strict_and_session_scoped(
|
||||
"source_url": "https://outside.invalid/recording.rrd",
|
||||
}
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
RecordedBlueprintRequest.model_validate(
|
||||
{
|
||||
"application_id": "nodedc_mission_core_recorded",
|
||||
"recording_id": "recording-001",
|
||||
"blueprint_session_id": "a" * 32,
|
||||
"accumulation_seconds": 12.0,
|
||||
"show_points": True,
|
||||
"show_trajectory": True,
|
||||
"show_grid": True,
|
||||
"unified_camera_share": 1.2,
|
||||
}
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
RecordedBlueprintRequest.model_validate(
|
||||
{
|
||||
"application_id": "nodedc_mission_core_recorded",
|
||||
"recording_id": "recording-001",
|
||||
"blueprint_session_id": "a" * 32,
|
||||
"accumulation_seconds": 12.0,
|
||||
"show_points": True,
|
||||
"show_trajectory": True,
|
||||
"show_grid": True,
|
||||
"eye_position": [3.0, 4.0, 5.0],
|
||||
}
|
||||
)
|
||||
with pytest.raises(HTTPException) as missing:
|
||||
asyncio.run(
|
||||
blueprint_route(
|
||||
@@ -1662,6 +1785,68 @@ def test_recorded_blueprint_endpoint_is_small_strict_and_session_scoped(
|
||||
assert missing.value.status_code == 404
|
||||
|
||||
|
||||
def test_recorded_blueprint_restores_camera_bounds_after_base_launch_lease_expires(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
repository = tmp_path / "repo"
|
||||
sessions = repository / "sessions"
|
||||
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
|
||||
store = SessionStore(repository, data_dir=tmp_path / "data")
|
||||
store.reconcile_archive(xgrids_k1_archive_source(sessions))
|
||||
recording_path = tmp_path / "published.rrd"
|
||||
calls: list[tuple[Path, dict[str, object]]] = []
|
||||
|
||||
class PublishedMaterializer:
|
||||
def restore_published(self, command: ReplayCommand) -> object:
|
||||
assert command.session_id == session.name
|
||||
return SimpleNamespace(path=recording_path)
|
||||
|
||||
def camera_bounds(path: Path, **kwargs: object) -> float:
|
||||
calls.append((path, kwargs))
|
||||
return 123.5
|
||||
|
||||
monkeypatch.setattr(
|
||||
"k1link.web.session_api.recorded_blueprint_rrd",
|
||||
lambda *_args, **_kwargs: b"RRF2",
|
||||
)
|
||||
monkeypatch.setattr("k1link.web.session_api.recorded_orbital_radius_limit", camera_bounds)
|
||||
route = endpoint(
|
||||
build_session_router(store, recording_materializer=PublishedMaterializer()), # type: ignore[arg-type]
|
||||
"/api/v1/observation-sessions/{session_id}/blueprint.rrd",
|
||||
"POST",
|
||||
)
|
||||
|
||||
response = asyncio.run(
|
||||
route(
|
||||
session_id=session.name,
|
||||
request=RecordedBlueprintRequest(
|
||||
application_id="nodedc_mission_core_recorded",
|
||||
recording_id="recording-camera-bounds",
|
||||
blueprint_session_id="b" * 32,
|
||||
accumulation_seconds=305.0,
|
||||
show_points=True,
|
||||
show_trajectory=False,
|
||||
show_grid=True,
|
||||
current_time_ns=39_215_000_000,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
assert response.headers["x-missioncore-camera-max-orbital-radius"] == "123.5"
|
||||
assert calls == [
|
||||
(
|
||||
recording_path,
|
||||
{
|
||||
"current_time_ns": 39_215_000_000,
|
||||
"accumulation_seconds": 305.0,
|
||||
"show_points": True,
|
||||
"show_trajectory": False,
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_recorded_perception_endpoint_returns_one_complete_optional_overlay(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from test_session_api import lab_method, make_legacy_session
|
||||
|
||||
from k1link.device_plugins.xgrids_k1 import xgrids_k1_archive_source
|
||||
from k1link.sessions import SessionStore
|
||||
from k1link.web.session_api import build_session_router
|
||||
|
||||
|
||||
def test_cursor_pages_keep_legacy_response_and_all_catalog_records(tmp_path: Path) -> None:
|
||||
repository = tmp_path / "repo"
|
||||
sessions = repository / "sessions"
|
||||
for index in range(102):
|
||||
make_legacy_session(sessions, f"source-{index:03}_viewer_live")
|
||||
store = SessionStore(repository, data_dir=tmp_path / "data")
|
||||
store.reconcile_archive(xgrids_k1_archive_source(sessions))
|
||||
for index in range(102):
|
||||
store.publish_lab_instance(
|
||||
session_id=f"lab-{index:03}",
|
||||
source_session_id="source-000_viewer_live",
|
||||
display_name=f"Historical LAB {index}",
|
||||
lab_id=f"LAB E{index}",
|
||||
result_kind="historical-evidence",
|
||||
result_id=f"result-{index:03}",
|
||||
run_created_at_utc="2026-07-23T15:55:15.548Z",
|
||||
provenance={"method": lab_method()},
|
||||
)
|
||||
app = FastAPI()
|
||||
app.include_router(build_session_router(store))
|
||||
with TestClient(app) as client:
|
||||
for scope in ("source", "laboratory"):
|
||||
base = {"scope": scope, "limit": 100, "lab_contract": "v3"}
|
||||
legacy = client.get("/api/v1/observation-sessions", params=base)
|
||||
assert legacy.status_code == 200
|
||||
assert set(legacy.json()) == {"items"}
|
||||
first = client.get(
|
||||
"/api/v1/observation-sessions", params={**base, "pagination": "cursor-v1"}
|
||||
).json()
|
||||
assert first["schema_version"] == "missioncore.observation-session-page/v1"
|
||||
assert first["items"] == legacy.json()["items"]
|
||||
assert first["next_cursor"] == first["items"][-1]["id"]
|
||||
second = client.get(
|
||||
"/api/v1/observation-sessions",
|
||||
params={**base, "pagination": "cursor-v1", "cursor": first["next_cursor"]},
|
||||
).json()
|
||||
assert len(second["items"]) == 2
|
||||
assert second["next_cursor"] is None
|
||||
assert len({item["id"] for item in first["items"] + second["items"]}) == 102
|
||||
# Neither paging nor the Observatory projection mutates the Legacy endpoint.
|
||||
assert client.get("/api/v1/observation-sessions", params=base).json() == legacy.json()
|
||||
assert client.get("/api/v1/observation-sessions?pagination=cursor-v2").status_code == 422
|
||||
assert (
|
||||
client.get(
|
||||
"/api/v1/observation-sessions?scope=source&pagination=cursor-v1&cursor=lab-099"
|
||||
).status_code
|
||||
== 404
|
||||
)
|
||||
|
||||
|
||||
def test_empty_and_exact_pages_end_explicitly(tmp_path: Path) -> None:
|
||||
repository = tmp_path / "repo"
|
||||
store = SessionStore(repository, data_dir=tmp_path / "data")
|
||||
app = FastAPI()
|
||||
app.include_router(build_session_router(store))
|
||||
with TestClient(app) as client:
|
||||
url = "/api/v1/observation-sessions?scope=source&pagination=cursor-v1&limit=1"
|
||||
empty = client.get(url).json()
|
||||
assert empty["items"] == []
|
||||
assert empty["next_cursor"] is None
|
||||
make_legacy_session(repository / "sessions", "source-one_viewer_live")
|
||||
store.reconcile_archive(xgrids_k1_archive_source(repository / "sessions"))
|
||||
full = client.get(url).json()
|
||||
assert len(full["items"]) == 1
|
||||
assert full["next_cursor"] is None
|
||||
@@ -21,6 +21,7 @@ import k1link.laboratory.vegetation_shadow_lab as vegetation_lab_module
|
||||
import k1link.web.vegetation_shadow_lab_api as vegetation_api_module
|
||||
from k1link.laboratory import LaboratoryEvidenceRegistry
|
||||
from k1link.laboratory.canonical_rerun_overlay import (
|
||||
CANONICAL_REPLAY_RESULT_ID,
|
||||
CanonicalLabOverlayArtifact,
|
||||
CanonicalLabReplayArtifact,
|
||||
_artifact_is_regular,
|
||||
@@ -69,10 +70,13 @@ def test_canonical_overlay_localizes_current_taxonomies() -> None:
|
||||
|
||||
def test_canonical_video_references_hold_only_missing_source_samples() -> None:
|
||||
session_times = np.arange(10, dtype=np.int64) * 100_000_000 + 39_000_000_000
|
||||
video_times = np.array(
|
||||
[0, 100, 200, 300, 400, 500, 600, 700, 900],
|
||||
dtype=np.int64,
|
||||
) * 1_000_000
|
||||
video_times = (
|
||||
np.array(
|
||||
[0, 100, 200, 300, 400, 500, 600, 700, 900],
|
||||
dtype=np.int64,
|
||||
)
|
||||
* 1_000_000
|
||||
)
|
||||
|
||||
references = _video_reference_timestamps(video_times, session_times)
|
||||
|
||||
@@ -219,6 +223,22 @@ def test_canonical_replay_merges_base_and_overlay_once(
|
||||
assert calls == 1
|
||||
|
||||
|
||||
def test_canonical_replay_accepts_each_published_portable_result_family() -> None:
|
||||
suffix = "a" * 64
|
||||
accepted = {
|
||||
"lab-v1-vegetation-shadow",
|
||||
"m49-tgs-portable-review",
|
||||
"lab-v1-eomt-ddrnet",
|
||||
"ai-layer-ddrnet",
|
||||
"ai-layer-eomt",
|
||||
"ai-layer-rf-detr",
|
||||
"ai-layer-object-distance",
|
||||
}
|
||||
|
||||
assert all(CANONICAL_REPLAY_RESULT_ID.fullmatch(f"{prefix}-{suffix}") for prefix in accepted)
|
||||
assert CANONICAL_REPLAY_RESULT_ID.fullmatch(f"ai-layer-unknown-{suffix}") is None
|
||||
|
||||
|
||||
def test_canonical_overlay_get_is_generation_bound_and_range_streamable(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
@@ -341,9 +361,7 @@ def test_canonical_overlay_get_is_generation_bound_and_range_streamable(
|
||||
)
|
||||
assert stale_overlay.status_code == 412
|
||||
|
||||
replay_endpoint = (
|
||||
f"/api/v1/laboratory/vegetation-shadow/{result_id}/canonical-replay.rrd"
|
||||
)
|
||||
replay_endpoint = f"/api/v1/laboratory/vegetation-shadow/{result_id}/canonical-replay.rrd"
|
||||
replay_descriptor = client.head(
|
||||
replay_endpoint,
|
||||
params={"base_generation": generation},
|
||||
@@ -496,7 +514,9 @@ def _worker_result(root: Path, *, candidate: str, mode: str, vegetation_iou: flo
|
||||
"truth_pixels": 16384,
|
||||
"truth_fraction": 0.0625,
|
||||
"stratum_rank": index + 1,
|
||||
} if mode == "goose" else None,
|
||||
}
|
||||
if mode == "goose"
|
||||
else None,
|
||||
"files": files,
|
||||
}
|
||||
)
|
||||
@@ -726,8 +746,7 @@ def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(
|
||||
},
|
||||
]
|
||||
(full_root / "result.json").write_text(
|
||||
json.dumps(full_manifest, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
+ "\n",
|
||||
json.dumps(full_manifest, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
for layer, sequence, expected in (
|
||||
@@ -735,27 +754,24 @@ def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(
|
||||
("vegetation", 1, full_archive_payloads[1]),
|
||||
):
|
||||
response = client.get(
|
||||
f"/api/v1/laboratory/vegetation-shadow/{full_result_id}"
|
||||
f"/route-masks/{layer}/{sequence}"
|
||||
f"/api/v1/laboratory/vegetation-shadow/{full_result_id}/route-masks/{layer}/{sequence}"
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.content == expected
|
||||
assert response.headers["cache-control"].endswith("immutable")
|
||||
assert client.get(
|
||||
f"/api/v1/laboratory/vegetation-shadow/{full_result_id}/route-masks/city/2"
|
||||
).status_code == 404
|
||||
timeline = client.get(
|
||||
f"/api/v1/laboratory/vegetation-shadow/{full_result_id}/route-timeline"
|
||||
assert (
|
||||
client.get(
|
||||
f"/api/v1/laboratory/vegetation-shadow/{full_result_id}/route-masks/city/2"
|
||||
).status_code
|
||||
== 404
|
||||
)
|
||||
timeline = client.get(f"/api/v1/laboratory/vegetation-shadow/{full_result_id}/route-timeline")
|
||||
assert timeline.status_code == 200
|
||||
assert timeline.content == full_timeline_payload
|
||||
assert timeline.headers["cache-control"].endswith("immutable")
|
||||
|
||||
(result_root / asset_path).write_bytes(b"tampered")
|
||||
assert (
|
||||
client.get(f"/api/v1/laboratory/vegetation-shadow/{result_root.name}").status_code
|
||||
== 503
|
||||
)
|
||||
assert client.get(f"/api/v1/laboratory/vegetation-shadow/{result_root.name}").status_code == 503
|
||||
|
||||
|
||||
def test_policy_review_reuses_sealed_video_and_links_yolox_tgs(
|
||||
@@ -837,9 +853,7 @@ def test_policy_review_reuses_sealed_video_and_links_yolox_tgs(
|
||||
assert route["linked_tgs_result_id"] == tgs_result_id
|
||||
assert route["fusion"]["pixel_raster_fusion"] is False
|
||||
assert route["fusion"]["camera_semantic_temporal_filter"] == "none"
|
||||
assert route["taxonomy"]["schema_version"] == (
|
||||
"missioncore.lab-v1-terrain-policy-taxonomy/v1"
|
||||
)
|
||||
assert route["taxonomy"]["schema_version"] == ("missioncore.lab-v1-terrain-policy-taxonomy/v1")
|
||||
assert len(route["taxonomy"]["classes"]) == 10
|
||||
assert route["valid_fov"]["outside_valid_fov_class_id"] == 9
|
||||
assert len(manifest["artifacts"]) == 80
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.worker_source_cache import WorkerSourceCache, WorkerSourceCacheError
|
||||
|
||||
|
||||
def test_cache_reuses_read_only_bytes_without_copy_or_retained_buffers(tmp_path: Path) -> None:
|
||||
source = tmp_path / "first-job-input"
|
||||
source.write_bytes(b"source-bytes")
|
||||
digest = hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
cache = WorkerSourceCache(tmp_path / "source-cache")
|
||||
assert cache.retain(source, sha256=digest, byte_length=12)
|
||||
destination = tmp_path / "second-job-input"
|
||||
assert cache.restore(destination, sha256=digest, byte_length=12)
|
||||
assert destination.read_bytes() == b"source-bytes"
|
||||
assert destination.stat().st_ino == source.stat().st_ino
|
||||
assert destination.stat().st_mode & 0o222 == 0
|
||||
assert set(vars(cache)) == {"root"}
|
||||
assert not list(tmp_path.rglob(".source-cache-*"))
|
||||
|
||||
|
||||
def test_bad_cache_is_miss_and_original_evidence_is_preserved(tmp_path: Path) -> None:
|
||||
good = b"good-source"
|
||||
digest = hashlib.sha256(good).hexdigest()
|
||||
cache = WorkerSourceCache(tmp_path / "source-cache")
|
||||
damaged = cache.root / digest
|
||||
damaged.write_bytes(b"bad--source")
|
||||
assert not cache.restore(tmp_path / "new-job", sha256=digest, byte_length=len(good))
|
||||
new_source = tmp_path / "fresh-download"
|
||||
new_source.write_bytes(good)
|
||||
assert not cache.retain(new_source, sha256=digest, byte_length=len(good))
|
||||
assert damaged.read_bytes() == b"bad--source"
|
||||
assert new_source.read_bytes() == good
|
||||
assert not (tmp_path / "new-job").exists()
|
||||
assert not list(tmp_path.rglob(".source-cache-*"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["symlink", "fifo", "directory"])
|
||||
def test_unsafe_cached_object_is_not_read(tmp_path: Path, kind: str) -> None:
|
||||
digest = hashlib.sha256(b"data").hexdigest()
|
||||
cache = WorkerSourceCache(tmp_path / "cache")
|
||||
cached = cache.root / digest
|
||||
if kind == "symlink":
|
||||
private = tmp_path / "not-source-evidence"
|
||||
private.write_bytes(b"data")
|
||||
cached.symlink_to(private)
|
||||
elif kind == "fifo":
|
||||
os.mkfifo(cached)
|
||||
else:
|
||||
cached.mkdir()
|
||||
assert not cache.restore(tmp_path / "new-job", sha256=digest, byte_length=4)
|
||||
assert not (tmp_path / "new-job").exists()
|
||||
|
||||
|
||||
def test_wrong_source_bytes_do_not_enter_cache(tmp_path: Path) -> None:
|
||||
source = tmp_path / "download"
|
||||
source.write_bytes(b"bad")
|
||||
cache = WorkerSourceCache(tmp_path / "cache")
|
||||
digest = hashlib.sha256(b"yes").hexdigest()
|
||||
assert not cache.retain(source, sha256=digest, byte_length=3)
|
||||
assert list(cache.root.iterdir()) == []
|
||||
assert source.read_bytes() == b"bad"
|
||||
|
||||
|
||||
def test_existing_destination_and_unsafe_identity_are_rejected(tmp_path: Path) -> None:
|
||||
source = tmp_path / "download"
|
||||
source.write_bytes(b"data")
|
||||
cache = WorkerSourceCache(tmp_path / "cache")
|
||||
digest = hashlib.sha256(b"data").hexdigest()
|
||||
cache.retain(source, sha256=digest, byte_length=4)
|
||||
with pytest.raises(WorkerSourceCacheError, match="not empty"):
|
||||
cache.restore(source, sha256=digest, byte_length=4)
|
||||
with pytest.raises(WorkerSourceCacheError, match="identity is invalid"):
|
||||
cache.restore(tmp_path / "other", sha256="../outside", byte_length=4)
|
||||
|
||||
|
||||
def test_unsupported_hardlinks_are_only_cache_misses(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
source = tmp_path / "download"
|
||||
source.write_bytes(b"data")
|
||||
cache = WorkerSourceCache(tmp_path / "cache")
|
||||
|
||||
def unsupported(*args: object, **kwargs: object) -> None:
|
||||
raise OSError("different filesystem")
|
||||
|
||||
monkeypatch.setattr(os, "link", unsupported)
|
||||
assert not cache.retain(source, sha256=hashlib.sha256(b"data").hexdigest(), byte_length=4)
|
||||
assert list(cache.root.iterdir()) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("corrupt", [False, True])
|
||||
def test_shared_cache_across_filesystems_copies_bounded_and_cleans_failures(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
corrupt: bool,
|
||||
) -> None:
|
||||
source = tmp_path / "download"
|
||||
payload = b"abcd" * 300_000
|
||||
source.write_bytes(payload)
|
||||
cache = WorkerSourceCache(tmp_path / "shared-cache")
|
||||
real_link = os.link
|
||||
real_read = os.read
|
||||
read_sizes = []
|
||||
|
||||
def link(src: Path, dst: Path, **kwargs: object) -> None:
|
||||
if Path(src) == source:
|
||||
raise OSError(errno.EXDEV, "separate agent mount")
|
||||
real_link(src, dst, **kwargs)
|
||||
|
||||
def read(fd: int, size: int) -> bytes:
|
||||
read_sizes.append(size)
|
||||
return real_read(fd, size)
|
||||
|
||||
monkeypatch.setattr(os, "link", link)
|
||||
monkeypatch.setattr(os, "read", read)
|
||||
digest = hashlib.sha256(b"other" if corrupt else payload).hexdigest()
|
||||
assert cache.retain(source, sha256=digest, byte_length=len(payload)) is not corrupt
|
||||
assert max(read_sizes) <= 1024 * 1024
|
||||
assert not list(tmp_path.rglob(".source-cache-*"))
|
||||
if not corrupt:
|
||||
assert (cache.root / digest).read_bytes() == payload
|
||||
assert (cache.root / digest).stat().st_ino != source.stat().st_ino
|
||||
@@ -0,0 +1,147 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from test_observatory_worker_http_transport import (
|
||||
BEARER_TOKEN,
|
||||
CLAIM_TOKEN,
|
||||
_cache_claim,
|
||||
_job,
|
||||
_source_contract,
|
||||
)
|
||||
|
||||
from k1link.observatory.worker_http_transport import (
|
||||
ObservatoryWorkerHttpError,
|
||||
ObservatoryWorkerHttpGateway,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"next_run", ["new-generation", "different-profile", "changed-camera", "bad-cache"]
|
||||
)
|
||||
def test_other_claim_reuses_bytes_but_obtains_its_own_manifest(
|
||||
tmp_path: Path, next_run: str
|
||||
) -> None:
|
||||
first = _job(
|
||||
bundle_sha256=hashlib.sha256(b"source-bundle").hexdigest(),
|
||||
capability_sha256=hashlib.sha256(b"source-capability").hexdigest(),
|
||||
)
|
||||
if next_run == "new-generation":
|
||||
second = replace(first, claim_generation=2)
|
||||
else:
|
||||
second = replace(
|
||||
first,
|
||||
job_id="observatory-run-" + "8" * 32,
|
||||
identity_sha256="9" * 64,
|
||||
setup_id="another-profile",
|
||||
definition_sha256="0" * 64,
|
||||
)
|
||||
requests: list[str] = []
|
||||
active = first
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
manifest, payloads = _source_contract(
|
||||
active,
|
||||
camera_segment_count=2 if active == second and next_run == "changed-camera" else 1,
|
||||
)
|
||||
if request.url.path.endswith("/claims"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"claim_token": CLAIM_TOKEN,
|
||||
"job": {
|
||||
"job_id": active.job_id,
|
||||
"claim_generation": active.claim_generation,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert request.headers["x-mission-core-claim-generation"] == str(active.claim_generation)
|
||||
assert active.job_id in request.url.path
|
||||
requests.append(request.url.path)
|
||||
if request.url.path.endswith("/source-materialization"):
|
||||
return httpx.Response(200, json=manifest)
|
||||
if request.url.path.endswith("/source-camera-epoch-archive"):
|
||||
return httpx.Response(404)
|
||||
payload = payloads[request.url.path.rsplit("/", 1)[-1]]
|
||||
return httpx.Response(
|
||||
200,
|
||||
content=payload,
|
||||
headers={
|
||||
"X-Mission-Core-Content-Sha256": hashlib.sha256(payload).hexdigest(),
|
||||
},
|
||||
)
|
||||
|
||||
arguments = dict(
|
||||
base_url="http://127.0.0.1:18080",
|
||||
bearer_token=BEARER_TOKEN,
|
||||
work_root=tmp_path / "worker",
|
||||
source_cache_root=tmp_path / "shared-source-cache",
|
||||
transport=httpx.MockTransport(handler),
|
||||
)
|
||||
with ObservatoryWorkerHttpGateway(**arguments) as gateway:
|
||||
_cache_claim(gateway)
|
||||
first_stage = gateway.materialize(first)
|
||||
if next_run == "bad-cache":
|
||||
cached = (
|
||||
tmp_path / "shared-source-cache" / hashlib.sha256(b"sealed-camera-segment").hexdigest()
|
||||
)
|
||||
cached.chmod(0o600)
|
||||
cached.write_bytes(b"corrupt-cache")
|
||||
requests.clear()
|
||||
active = second
|
||||
arguments["work_root"] = tmp_path / "other-agent-work"
|
||||
# Fresh process/client instance: no in-memory cache or claim is inherited.
|
||||
with ObservatoryWorkerHttpGateway(**arguments) as gateway:
|
||||
with pytest.raises(ObservatoryWorkerHttpError):
|
||||
gateway.materialize(second)
|
||||
_cache_claim(gateway)
|
||||
second_stage = gateway.materialize(second)
|
||||
assert second_stage.root != first_stage.root
|
||||
assert requests[0].endswith("/source-materialization")
|
||||
assert len(requests) == (2 if next_run in {"changed-camera", "bad-cache"} else 1)
|
||||
assert not any(path.endswith("/source-camera-epoch-archive") for path in requests)
|
||||
persisted = json.loads((second_stage.root / "materialization-manifest.json").read_bytes())
|
||||
assert persisted["job_id"] == second.job_id
|
||||
assert persisted["job_identity_sha256"] == second.identity_sha256
|
||||
assert persisted["claim_generation"] == second.claim_generation
|
||||
assert (
|
||||
second_stage.root / "camera/epoch-1/segments/1.m4s"
|
||||
).read_bytes() == b"sealed-camera-segment"
|
||||
assert not list(tmp_path.rglob(".source-cache-*"))
|
||||
|
||||
|
||||
def test_replayed_old_manifest_cannot_use_cached_source(tmp_path: Path) -> None:
|
||||
first = _job(bundle_sha256="1" * 64, capability_sha256="2" * 64)
|
||||
manifest, _ = _source_contract(first)
|
||||
second = replace(first, claim_generation=2)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/claims"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"claim_token": CLAIM_TOKEN,
|
||||
"job": {
|
||||
"job_id": second.job_id,
|
||||
"claim_generation": 2,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert request.url.path.endswith("/source-materialization")
|
||||
return httpx.Response(200, json=manifest)
|
||||
|
||||
with ObservatoryWorkerHttpGateway(
|
||||
base_url="http://127.0.0.1:18080",
|
||||
bearer_token=BEARER_TOKEN,
|
||||
work_root=tmp_path / "worker",
|
||||
transport=httpx.MockTransport(handler),
|
||||
) as gateway:
|
||||
_cache_claim(gateway)
|
||||
with pytest.raises(ObservatoryWorkerHttpError):
|
||||
gateway.materialize(second)
|
||||
assert not (tmp_path / "worker/sources").exists()
|
||||
Reference in New Issue
Block a user