feat(observatory): reconcile quarantined recorded jobs

tests/test_observatory_recorded_jobs.py

@@ -19,6 +19,8 @@ from k1link.observatory.recorded_jobs import (
     ObservatoryRecordedQueueConflictError,
     ObservatoryRecordedQueueIntegrityError,
     ObservatoryRecordedQueueStaleClaimError,
+    ObservatoryRecordedReconciliationRequest,
+    ObservatoryRecordedResourceReleaseAttestation,
     RecordedRunDefinition,
     RecordedRunDefinitionRegistry,
 )
@@ -36,6 +38,7 @@ EXECUTOR_RELEASE_SHA = "3" * 64
 EXECUTOR_IMAGE_SHA = "4" * 64
 MODEL_MANIFEST_SHA = "5" * 64
 RESOURCE_PROFILE_SHA = "6" * 64
+RESOURCE_RELEASE_EVIDENCE_SHA = "7" * 64
 
 
 def _definitions() -> RecordedRunDefinitionRegistry:
@@ -155,6 +158,45 @@ def _running_job(
     return running, claim
 
 
+def _resource_release_attestation(
+    job_id: str,
+    claim_generation: int,
+    *,
+    resources_released: bool = True,
+) -> ObservatoryRecordedResourceReleaseAttestation:
+    return ObservatoryRecordedResourceReleaseAttestation(
+        attestation_id="worker-006-release-proof-001",
+        operator_id="missioncore-operator",
+        job_id=job_id,
+        claim_generation=claim_generation,
+        resources_released=resources_released,
+        evidence_sha256=RESOURCE_RELEASE_EVIDENCE_SHA,
+        attested_at_utc=NOW,
+    )
+
+
+def _reconciliation_request(
+    job_id: str,
+    claim_generation: int,
+    *,
+    reconciliation_id: str = "worker-006-reconciliation-001",
+    expected_terminal_code: str = "claim-lease-expired",
+    reason: str = "Worker 006 release was independently verified.",
+) -> ObservatoryRecordedReconciliationRequest:
+    return ObservatoryRecordedReconciliationRequest(
+        reconciliation_id=reconciliation_id,
+        job_id=job_id,
+        expected_claim_generation=claim_generation,
+        expected_terminal_code=expected_terminal_code,
+        resource_release_attestation=_resource_release_attestation(
+            job_id,
+            claim_generation,
+        ),
+        failure_code="operator-reconciled-failure",
+        reason=reason,
+    )
+
+
 def test_submission_is_durable_exactly_idempotent_and_path_free(tmp_path: Path) -> None:
     queue = _queue(tmp_path)
     first, first_created = queue.submit(_intent())
@@ -470,6 +512,131 @@ def test_expired_running_claim_is_quarantined_and_stale_terminal_is_fenced(
     )
 
 
+def test_operator_reconciliation_requires_exact_generation_code_and_release_proof(
+    tmp_path: Path,
+) -> None:
+    clock_value = [NOW]
+    queue = ObservatoryRecordedJobQueue(
+        tmp_path,
+        definitions=_definitions(),
+        clock=lambda: clock_value[0],
+        claim_lease_seconds=10,
+    )
+    running, _claim = _running_job(queue)
+    clock_value[0] = "2026-08-30T21:00:10.000Z"
+    [quarantined] = queue.recover_stale_claims()
+
+    with pytest.raises(ValueError, match="explicitly release resources"):
+        _resource_release_attestation(
+            quarantined.job_id,
+            quarantined.claim_generation,
+            resources_released=False,
+        )
+    with pytest.raises(ObservatoryRecordedQueueConflictError, match="generation"):
+        queue.reconcile_failed(
+            _reconciliation_request(
+                quarantined.job_id,
+                quarantined.claim_generation + 1,
+            )
+        )
+    with pytest.raises(ObservatoryRecordedQueueConflictError, match="terminal code"):
+        queue.reconcile_failed(
+            _reconciliation_request(
+                quarantined.job_id,
+                quarantined.claim_generation,
+                expected_terminal_code="claim-lease-migration",
+            )
+        )
+
+    unchanged = queue.get(running.job_id)
+    assert unchanged.state == "reconciliation-required"
+    assert unchanged.terminal_code == "claim-lease-expired"
+
+
+def test_operator_reconciliation_is_durable_idempotent_and_unblocks_queue(
+    tmp_path: Path,
+) -> None:
+    clock_value = [NOW]
+    queue = ObservatoryRecordedJobQueue(
+        tmp_path,
+        definitions=_definitions(),
+        clock=lambda: clock_value[0],
+        claim_lease_seconds=10,
+    )
+    running, _claim = _running_job(queue)
+    next_job, _ = queue.submit(
+        _intent(
+            idempotency_key="recorded-request-002",
+            source_session_id="another-session",
+        ),
+        enqueue=True,
+    )
+    clock_value[0] = "2026-08-30T21:00:10.000Z"
+    [quarantined] = queue.recover_stale_claims()
+    request = _reconciliation_request(
+        quarantined.job_id,
+        quarantined.claim_generation,
+    )
+
+    receipt = queue.reconcile_failed(request)
+    replayed = queue.reconcile_failed(request)
+    restored_queue = ObservatoryRecordedJobQueue(
+        tmp_path,
+        definitions=_definitions(),
+        clock=lambda: clock_value[0],
+        claim_lease_seconds=10,
+    )
+    restored = restored_queue.reconcile_failed(request)
+
+    assert replayed == receipt
+    assert restored == receipt
+    assert restored_queue.get_reconciliation(running.job_id) == receipt
+    assert receipt.resource_release_attestation.resources_released is True
+    assert receipt.resource_release_attestation.evidence_sha256 == (
+        RESOURCE_RELEASE_EVIDENCE_SHA
+    )
+    assert receipt.expected_terminal_code == "claim-lease-expired"
+    assert receipt.quarantined_terminal_message == (
+        "Worker claim lease expired after execution started; "
+        "physical resource ownership requires reconciliation."
+    )
+    assert receipt.quarantined_at_utc == "2026-08-30T21:00:10.000Z"
+    assert receipt.as_dict()["outcome"] == {
+        "state": "failed",
+        "code": "operator-reconciled-failure",
+        "reason": "Worker 006 release was independently verified.",
+    }
+    failed = restored_queue.get(running.job_id)
+    assert failed.state == "failed"
+    assert failed.terminal_code == "operator-reconciled-failure"
+    assert failed.terminal_message == "Worker 006 release was independently verified."
+    assert failed.terminal_claim_token_sha256 is None
+
+    with pytest.raises(ObservatoryRecordedQueueConflictError, match="another request"):
+        restored_queue.reconcile_failed(
+            _reconciliation_request(
+                quarantined.job_id,
+                quarantined.claim_generation,
+                reason="Conflicting operator explanation.",
+            )
+        )
+    with pytest.raises(ObservatoryRecordedQueueConflictError, match="another reconciliation"):
+        restored_queue.reconcile_failed(
+            _reconciliation_request(
+                quarantined.job_id,
+                quarantined.claim_generation,
+                reconciliation_id="worker-006-reconciliation-002",
+            )
+        )
+
+    replacement = restored_queue.claim_next(
+        claimant_id="recorded-worker",
+        claim_request_id="claim-after-operator-reconciliation",
+    )
+    assert replacement is not None
+    assert replacement.job.job_id == next_job.job_id
+
+
 def test_legacy_sqlite_claim_schema_migrates_without_reusing_old_token(
     tmp_path: Path,
 ) -> None: