fix(observatory): version non-binding idle claims and preserve grant history

tests/test_observatory_claim_transport_migration.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import copy
+import importlib.util
+import json
+from pathlib import Path
+
+import pytest
+
+SCRIPT = (
+    Path(__file__).resolve().parents[1] / "experiments/perception/worker/"
+    "observatory_portable/migrate_claim_transport_v3.py"
+)
+SPEC = importlib.util.spec_from_file_location("claim_migration", SCRIPT)
+assert SPEC and SPEC.loader
+migration = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(migration)
+
+
+def test_patch_is_exact_and_does_not_upgrade_other_transport_code(monkeypatch) -> None:
+    source = b'PROTOCOL = "missioncore.observatory-worker-claim-request/v2"\n'
+    monkeypatch.setattr(migration, "BEFORE_SHA", migration.sha(source))
+    assert migration.patch_source(source) == source.replace(migration.OLD, migration.NEW)
+    with pytest.raises(ValueError):
+        migration.patch_source(source + b"# unreviewed change\n")
+    with pytest.raises(ValueError):
+        migration.patch_source(source.replace(migration.OLD, migration.NEW))
+
+
+@pytest.mark.parametrize(
+    "state", ["queued", "accepted", "claimed", "running", "paused", "reconciliation-required"]
+)
+def test_migration_refuses_pending_or_active_work(state: str) -> None:
+    readiness = {
+        "schema_version": "missioncore.observatory-claim-readiness/v1",
+        "open_live_lease_count": 0,
+        "recorded_jobs_by_state": {state: 1},
+        "protocols": [{}, {"grant_capacity_available": True}],
+    }
+    with pytest.raises(ValueError, match="queue is not idle"):
+        migration.require_idle(readiness)
+    readiness["recorded_jobs_by_state"] = {"failed": 10, "succeeded": 1}
+    migration.require_idle(readiness)
+    readiness["open_live_lease_count"] = 1
+    with pytest.raises(ValueError, match="queue is not idle"):
+        migration.require_idle(readiness)
+
+
+def test_target_validation_is_pinned_and_refuses_gpu_or_inline_credentials() -> None:
+    name, image = next(iter(migration.TARGETS.items()))
+    inspection = {
+        "Name": "/" + name,
+        "Image": "sha256:" + image,
+        "State": {"Running": True},
+        "Config": {
+            "Labels": {
+                "com.nodedc.product": "mission-core",
+                "com.nodedc.authority": "observation-only",
+            },
+            "Env": ["MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE=/run/secrets/token"],
+        },
+        "HostConfig": {"ReadonlyRootfs": True, "NetworkMode": "bridge"},
+    }
+    migration.validate_target(name, inspection)
+    inspection["HostConfig"]["DeviceRequests"] = [{"Count": -1}]
+    with pytest.raises(ValueError):
+        migration.validate_target(name, inspection)
+    inspection["HostConfig"]["DeviceRequests"] = []
+    inspection["Config"]["Env"] = ["BEARER_TOKEN=synthetic-fixture"]
+    with pytest.raises(ValueError, match="inline credential"):
+        migration.validate_target(name, inspection)
+
+
+def test_evidence_is_immutable_and_control_probe_cannot_claim_models(tmp_path: Path) -> None:
+    target = tmp_path / "receipt.json"
+    migration.save(target, {"state": "original"})
+    with pytest.raises(FileExistsError):
+        migration.save(target, {"state": "overwritten"})
+    assert json.loads(target.read_bytes()) == {"state": "original"}
+    assert "supported_executor_identities=()" in migration.PROBE
+    assert "range(8)" in migration.PROBE
+    compile(migration.PROBE, "control-probe", "exec")
+    compile(migration.READINESS, "readiness", "exec")
+
+
+def test_transport_clone_retains_executor_configuration_and_rollback_snapshot() -> None:
+    inspection = {
+        "Config": {
+            "Image": "old",
+            "Cmd": ["-m", "existing-worker"],
+            "Labels": {"com.nodedc.package-sha256": "unchanged"},
+            "Env": ["DEFINITIONS=/release/definitions.json"],
+            "User": "0:0",
+        },
+        "HostConfig": {
+            "Mounts": [{"Source": "immutable-release", "Target": "/release"}],
+            "RestartPolicy": {"Name": "unless-stopped"},
+            "DeviceRequests": [],
+        },
+    }
+    before = copy.deepcopy(inspection)
+    body = migration.replacement_body(
+        inspection,
+        {
+            "image": "new",
+            "parent_image_sha256": "old",
+            "source_sha256": "patched-source",
+        },
+    )
+    assert inspection == before
+    assert body["HostConfig"] == before["HostConfig"]
+    assert body["Labels"]["com.nodedc.package-sha256"] == "unchanged"
+    for key in ("Cmd", "Env", "User"):
+        assert body[key] == before["Config"][key]
+    assert body["Image"] == "new"
+    body["HostConfig"]["RestartPolicy"]["Name"] = "no"
+    assert inspection == before
+
+
+def test_changed_plan_cannot_write_evidence_or_touch_containers(
+    tmp_path: Path, monkeypatch
+) -> None:
+    monkeypatch.setattr(migration, "plan", lambda engine: {"state": "changed"})
+    evidence = tmp_path / "not-created"
+    with pytest.raises(ValueError, match="plan changed"):
+        migration.apply(object(), "0" * 64, evidence)
+    assert not evidence.exists()

tests/test_observatory_recorded_jobs.py

@@ -18,6 +18,7 @@ from k1link.observatory.recorded_jobs import (
     ObservatoryRecordedJobQueue,
     ObservatoryRecordedPreemptionError,
     ObservatoryRecordedQueueBusyError,
+    ObservatoryRecordedQueueCapacityError,
     ObservatoryRecordedQueueConflictError,
     ObservatoryRecordedQueueDuplicateError,
     ObservatoryRecordedQueueIntegrityError,
@@ -355,6 +356,144 @@ def test_duplicate_guard_keeps_new_version_of_same_setup(tmp_path: Path) -> None
     assert created and len(reopened.list_jobs()) == 2
 
 
+def _capabilities() -> tuple[RecordedExecutorIdentity, ...]:
+    definition = _definitions().definitions[0]
+    return (RecordedExecutorIdentity(
+        release_sha256=definition.executor_release_sha256,
+        image_sha256=definition.executor_image_sha256,
+        model_manifest_sha256=definition.model_manifest_sha256,
+        resource_profile_sha256=definition.resource_profile_sha256,
+    ),)
+
+
+def _claim_rows(queue: ObservatoryRecordedJobQueue, table: str) -> list[tuple]:
+    assert table in {"observatory_recorded_claim_receipts", "observatory_recorded_claim_grants_v3"}
+    with sqlite3.connect(queue.database_path) as connection:
+        return connection.execute(f"SELECT * FROM {table} ORDER BY claim_request_id").fetchall()
+
+
+def test_v3_idle_does_not_consume_grant_quota_or_modify_legacy_receipts(tmp_path: Path) -> None:
+    queue = ObservatoryRecordedJobQueue(
+        tmp_path, definitions=_definitions(), clock=lambda: NOW,
+        max_claim_receipts=1, max_v3_claim_receipts=1,
+    )
+    args = {"claimant_id": "recorded-worker", "supported_executor_identities": _capabilities()}
+    assert queue.claim_next(**args, claim_request_id="legacy-empty") is None
+    legacy = _claim_rows(queue, "observatory_recorded_claim_receipts")
+    readiness = queue.claim_readiness()
+    assert readiness["protocols"][0]["grant_capacity_available"] is False
+    assert readiness["protocols"][1]["grant_capacity_available"] is True
+    with pytest.raises(ObservatoryRecordedQueueCapacityError):
+        queue.claim_next(**args, claim_request_id="legacy-full")
+    for number in range(8):
+        assert queue.claim_next(
+            **args, claim_request_id=f"idle-v3-{number}", protocol_version=3,
+        ) is None
+    assert _claim_rows(queue, "observatory_recorded_claim_grants_v3") == []
+    assert _claim_rows(queue, "observatory_recorded_claim_receipts") == legacy
+
+    job, _ = queue.submit(_intent(), enqueue=True)
+    # Old empty receipts stay empty even when retried via the new protocol.
+    assert queue.claim_next(**args, claim_request_id="legacy-empty", protocol_version=3) is None
+    claim = queue.claim_next(**args, claim_request_id="idle-v3-0", protocol_version=3)
+    assert claim is not None and claim.job.job_id == job.job_id
+    assert len(_claim_rows(queue, "observatory_recorded_claim_grants_v3")) == 1
+    assert _claim_rows(queue, "observatory_recorded_claim_receipts") == legacy
+    reopened = ObservatoryRecordedJobQueue(
+        tmp_path, definitions=_definitions(), clock=lambda: NOW,
+        max_claim_receipts=1, max_v3_claim_receipts=1,
+    )
+    assert reopened.claim_next(**args, claim_request_id="idle-v3-0", protocol_version=3) == claim
+    # No second grant even if a client retries a v3 grant through legacy v2.
+    assert reopened.claim_next(**args, claim_request_id="idle-v3-0") == claim
+    with pytest.raises(ObservatoryRecordedQueueConflictError):
+        reopened.claim_next(
+            claimant_id="other-worker", supported_executor_identities=_capabilities(),
+            claim_request_id="idle-v3-0", protocol_version=3,
+        )
+    assert reopened.claim_next(**args, claim_request_id="busy-v3", protocol_version=3) is None
+
+    reopened.start(job.job_id, claim_token=claim.claim_token)
+    reopened.succeed(
+        job.job_id, claim_token=claim.claim_token,
+        result_id="result-first", result_sha256=RESULT_SHA,
+    )
+    with pytest.raises(ObservatoryRecordedQueueStaleClaimError):
+        reopened.claim_next(**args, claim_request_id="idle-v3-0", protocol_version=3)
+    assert reopened.claim_next(**args, claim_request_id="empty-full-v3", protocol_version=3) is None
+    second, _ = reopened.submit(_intent(idempotency_key="second-job"), enqueue=True)
+    with pytest.raises(ObservatoryRecordedQueueCapacityError):
+        reopened.claim_next(**args, claim_request_id="grant-full-v3", protocol_version=3)
+    assert reopened.get(second.job_id).state == "queued"
+    assert reopened.get(second.job_id).claim_generation == 0
+    assert reopened.claim_readiness()["protocols"][1]["grant_capacity_available"] is False
+
+
+def test_v3_empty_capabilities_and_live_lease_cannot_claim_work(tmp_path: Path) -> None:
+    queue = _queue(tmp_path)
+    job, _ = queue.submit(_intent(), enqueue=True)
+    assert queue.claim_next(
+        claimant_id="recorded-worker", claim_request_id="no-capabilities",
+        supported_executor_identities=(), protocol_version=3,
+    ) is None
+    queue.request_live(_live_intent())
+    assert queue.claim_next(
+        claimant_id="recorded-worker", claim_request_id="live-has-priority",
+        supported_executor_identities=_capabilities(), protocol_version=3,
+    ) is None
+    assert queue.get(job.job_id).state == "queued"
+    assert _claim_rows(queue, "observatory_recorded_claim_grants_v3") == []
+
+
+def test_v3_two_simultaneous_retries_receive_one_grant(tmp_path: Path) -> None:
+    queues = (_queue(tmp_path), _queue(tmp_path))
+    queues[0].submit(_intent(), enqueue=True)
+    barrier = Barrier(2)
+
+    def claim(index: int):
+        barrier.wait(timeout=5)
+        return queues[index].claim_next(
+            claimant_id="recorded-worker", claim_request_id="same-v3-id",
+            supported_executor_identities=_capabilities(), protocol_version=3,
+        )
+
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        claims = tuple(executor.map(claim, (0, 1)))
+    assert claims[0] is not None and claims[0] == claims[1]
+    assert len(_claim_rows(queues[0], "observatory_recorded_claim_grants_v3")) == 1
+
+
+def test_v3_expired_grant_cannot_reclaim_or_change_capabilities(tmp_path: Path) -> None:
+    clock = [NOW]
+    queue = ObservatoryRecordedJobQueue(
+        tmp_path, definitions=_definitions(), clock=lambda: clock[0], claim_lease_seconds=10,
+    )
+    job, _ = queue.submit(_intent(), enqueue=True)
+    args = {"claimant_id": "recorded-worker", "supported_executor_identities": _capabilities(),
+            "protocol_version": 3}
+    first = queue.claim_next(**args, claim_request_id="original-v3")
+    assert first is not None
+    with pytest.raises(ObservatoryRecordedQueueConflictError):
+        queue.claim_next(claimant_id="recorded-worker", claim_request_id="original-v3",
+                         supported_executor_identities=(), protocol_version=3)
+    clock[0] = "2026-08-30T21:00:11.000Z"
+    with pytest.raises(ObservatoryRecordedQueueStaleClaimError):
+        queue.claim_next(**args, claim_request_id="original-v3")
+    second = queue.claim_next(**args, claim_request_id="new-v3")
+    assert second is not None and second.job.job_id == job.job_id
+    assert second.job.claim_generation == first.job.claim_generation + 1
+    assert second.claim_token != first.claim_token
+    with pytest.raises(ObservatoryRecordedQueueStaleClaimError):
+        queue.start(job.job_id, claim_token=first.claim_token)
+
+
+@pytest.mark.parametrize("version", [True, False, 1, 4, "3"])
+def test_claim_protocol_rejects_ambiguous_versions(tmp_path: Path, version) -> None:
+    with pytest.raises(ValueError, match="protocol version"):
+        _queue(tmp_path).claim_next(claimant_id="recorded-worker", claim_request_id="invalid",
+                                    supported_executor_identities=(), protocol_version=version)
+
+
 def test_submission_is_durable_exactly_idempotent_and_path_free(tmp_path: Path) -> None:
     queue = _queue(tmp_path)
     first, first_created = queue.submit(_intent())

tests/test_observatory_worker_api.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import hashlib
+import sqlite3
 from pathlib import Path
 from typing import Any, cast
 
@@ -170,6 +171,35 @@ def _services_with_transient_publisher(
     return TestClient(app), queue, publisher
 
 
+def test_v3_idle_is_explicit_and_grant_remains_idempotent(tmp_path: Path) -> None:
+    client, queue = _services(tmp_path)
+    readiness = client.get(
+        "/api/v1/worker/observatory/recorded-jobs/claim-readiness", headers=WORKER_HEADERS,
+    )
+    assert readiness.status_code == 200
+    assert readiness.json()["protocols"][1]["idle_policy"] == "non-binding-no-receipt"
+    assert client.get("/api/v1/worker/observatory/recorded-jobs/claim-readiness").status_code == 401
+    request = {**_claim_request("v3-client"), "schema_version": CLAIM_SCHEMA.replace("/v2", "/v3")}
+    path = "/api/v1/worker/observatory/recorded-jobs/claims"
+    idle = client.post(path, json=request, headers=WORKER_HEADERS)
+    assert idle.status_code == 204
+    assert idle.headers["X-Mission-Core-Idle-Claim"] == "non-binding"
+    with sqlite3.connect(queue.database_path) as connection:
+        assert connection.execute(
+            "SELECT COUNT(*) FROM observatory_recorded_claim_receipts"
+        ).fetchone()[0] == 0
+        assert connection.execute(
+            "SELECT COUNT(*) FROM observatory_recorded_claim_grants_v3"
+        ).fetchone()[0] == 0
+    job_id = _enqueue(queue)
+    grant = client.post(path, json=request, headers=WORKER_HEADERS)
+    assert grant.status_code == 200
+    assert grant.json()["job"]["job_id"] == job_id
+    retry = client.post(path, json=request, headers=WORKER_HEADERS)
+    assert retry.status_code == 200 and retry.json() == grant.json()
+    assert client.post(path, json=request).status_code == 401
+
+
 def _enqueue(
     queue: ObservatoryRecordedJobQueue,
     *,

tests/test_observatory_worker_http_transport.py

@@ -95,6 +95,24 @@ def _claim_response() -> httpx.Response:
     )
 
 
+def test_gateway_uses_v3_without_falling_back_to_idle_receipt_growth(tmp_path: Path) -> None:
+    def handle(request: httpx.Request) -> httpx.Response:
+        body = json.loads(request.content)
+        assert body["schema_version"] == "missioncore.observatory-worker-claim-request/v3"
+        assert body["supported_executor_identities"] == []
+        return httpx.Response(204, headers={"X-Mission-Core-Idle-Claim": "non-binding"})
+
+    with ObservatoryWorkerHttpGateway(
+        base_url="http://127.0.0.1:8000", bearer_token=BEARER_TOKEN,
+        work_root=tmp_path, transport=httpx.MockTransport(handle),
+    ) as gateway:
+        for number in range(2):
+            assert gateway.claim_next(
+                claimant_id=WORKER_006_CONTOUR_ID,
+                claim_request_id=f"v3-idle-{number}", supported_executor_identities=(),
+            ) is None
+
+
 def _cache_claim(gateway: ObservatoryWorkerHttpGateway) -> None:
     payload = gateway.claim_next(
         claimant_id=WORKER_006_CONTOUR_ID,