SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(observatory): version non-binding idle claims and preserve grant history

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-09-02 23:40:11 +03:00
parent 049ed3eed4
commit cabd3081c7
8 changed files with 828 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/observatory/recorded_jobs.py
+103 -12
View File
@@ -44,6 +44,9 @@ OBSERVATORY_LIVE_LEASE_REQUEST_SCHEMA: Final = "missioncore.observatory-live-k1-
RECORDED_JOB_DATABASE_NAME: Final = "observatory-recorded-jobs.sqlite3"
MAX_RECORDED_JOBS: Final = 10_000
MAX_RECORDED_CLAIM_RECEIPTS: Final = 50_000
# The legacy ledger is immutable history, including successful empty polls.
# v3 has a separate bounded ledger containing only actual ownership grants.
MAX_RECORDED_V3_CLAIM_RECEIPTS: Final = 50_000
# Every capability expands to four SQLite bind parameters. Keeping the public
# bound at 128 stays comfortably below SQLite's traditional 999-variable limit
# even when the runtime was compiled with conservative defaults.
@@ -166,6 +169,16 @@ CREATE TABLE IF NOT EXISTS observatory_recorded_claim_receipts (
FOREIGN KEY (job_id) REFERENCES observatory_recorded_jobs(job_id)
);
CREATE TABLE IF NOT EXISTS observatory_recorded_claim_grants_v3 (
claim_request_id TEXT PRIMARY KEY,
request_sha256 TEXT NOT NULL,
claimant_id TEXT NOT NULL,
job_id TEXT NOT NULL,
claim_token TEXT NOT NULL,
created_at_utc TEXT NOT NULL,
FOREIGN KEY (job_id) REFERENCES observatory_recorded_jobs(job_id)
);
CREATE TABLE IF NOT EXISTS observatory_recorded_reconciliations (
reconciliation_id TEXT PRIMARY KEY,
request_sha256 TEXT NOT NULL,
@@ -1152,11 +1165,13 @@ class ObservatoryRecordedJobQueue:
non_checkpointable_preemptor: (ObservatoryNonCheckpointablePreemptor | None) = None,
max_jobs: int = MAX_RECORDED_JOBS,
max_claim_receipts: int = MAX_RECORDED_CLAIM_RECEIPTS,
max_v3_claim_receipts: int = MAX_RECORDED_V3_CLAIM_RECEIPTS,
max_live_leases: int = MAX_LIVE_LEASES,
claim_lease_seconds: int = DEFAULT_RECORDED_CLAIM_LEASE_SECONDS,
) -> None:
_validate_quota(max_jobs, MAX_RECORDED_JOBS, "recorded job")
_validate_quota(max_claim_receipts, MAX_RECORDED_CLAIM_RECEIPTS, "claim receipt")
_validate_quota(max_v3_claim_receipts, MAX_RECORDED_V3_CLAIM_RECEIPTS, "v3 claim grant")
_validate_quota(max_live_leases, MAX_LIVE_LEASES, "live lease")
_validate_claim_lease_seconds(claim_lease_seconds)
self.data_dir = data_dir.expanduser().resolve()
@@ -1166,6 +1181,7 @@ class ObservatoryRecordedJobQueue:
self._non_checkpointable_preemptor = non_checkpointable_preemptor
self._max_jobs = max_jobs
self._max_claim_receipts = max_claim_receipts
self._max_v3_claim_receipts = max_v3_claim_receipts
self._max_live_leases = max_live_leases
self._claim_lease_seconds = claim_lease_seconds
self._lock = threading.RLock()
@@ -1313,20 +1329,69 @@ class ObservatoryRecordedJobQueue:
return job
raise ObservatoryRecordedQueueConflictError("job cannot be enqueued")
def claim_readiness(self) -> dict[str, object]:
"""Report finite grant capacity separately from the legacy idle ledger."""
with self._read_connection() as connection:
legacy_count = connection.execute(
"SELECT COUNT(*) FROM observatory_recorded_claim_receipts"
).fetchone()[0]
grant_count = connection.execute(
"SELECT COUNT(*) FROM observatory_recorded_claim_grants_v3"
).fetchone()[0]
job_counts = dict(connection.execute(
"SELECT state, COUNT(*) FROM observatory_recorded_jobs GROUP BY state"
).fetchall())
live_count = connection.execute(
"SELECT COUNT(*) FROM observatory_live_leases "
"WHERE state IN ('pending', 'active')"
).fetchone()[0]
return {
"schema_version": "missioncore.observatory-claim-readiness/v1",
"protocols": [
{
"version": 2,
"idle_policy": "durable-receipt",
"receipt_count": legacy_count,
"receipt_limit": self._max_claim_receipts,
"grant_capacity_available": legacy_count < self._max_claim_receipts,
},
{
"version": 3,
"idle_policy": "non-binding-no-receipt",
"receipt_count": grant_count,
"receipt_limit": self._max_v3_claim_receipts,
"grant_capacity_available": grant_count < self._max_v3_claim_receipts,
},
],
"storage_limit_bytes": MAX_RECORDED_JOB_STORAGE_BYTES,
"recorded_jobs_by_state": job_counts,
"open_live_lease_count": live_count,
"authority": dict(_AUTHORITY),
}
def claim_next(
self,
*,
claimant_id: str,
claim_request_id: str,
supported_executor_identities: tuple[RecordedExecutorIdentity, ...] | None = None,
protocol_version: Literal[2, 3] = 2,
) -> ObservatoryRecordedClaim | None:
"""Claim one compatible job atomically; even an empty claim is idempotent.
"""Claim compatible work atomically with a versioned idle-poll policy.
``None`` preserves the legacy v1 claim semantics during rollout. A
concrete tuple is the capability-aware v2 contract; an empty tuple
deliberately claims nothing.
deliberately claims nothing. v2 records even idle polls. v3 idle is a
non-binding observation; only a real grant is persisted. A retry of a
granted request always returns the same grant (or a stale-claim error).
Both versions honor every previously persisted receipt.
"""
if isinstance(protocol_version, bool) or protocol_version not in (2, 3):
raise ValueError("claim protocol version is invalid")
if protocol_version == 3 and supported_executor_identities is None:
raise ValueError("v3 claims require an explicit capability snapshot")
_validate_pattern(claimant_id, _IDENTIFIER, "claimant id")
_validate_pattern(claim_request_id, _IDEMPOTENCY_KEY, "claim request id")
capabilities = _canonical_executor_capabilities(supported_executor_identities)
@@ -1338,22 +1403,33 @@ class ObservatoryRecordedJobQueue:
with self._transaction() as connection:
now = self._timestamp()
self._recover_stale_claims(connection, now=now)
receipt = connection.execute(
"SELECT * FROM observatory_recorded_claim_receipts WHERE claim_request_id = ?",
(claim_request_id,),
).fetchone()
receipts = connection.execute(
"SELECT * FROM observatory_recorded_claim_receipts WHERE claim_request_id = ? "
"UNION ALL SELECT * FROM observatory_recorded_claim_grants_v3 "
"WHERE claim_request_id = ?",
(claim_request_id, claim_request_id),
).fetchall()
if len(receipts) > 1:
raise ObservatoryRecordedQueueIntegrityError("claim id exists in both ledgers")
receipt = receipts[0] if receipts else None
if receipt is not None:
if receipt["request_sha256"] != request_sha256:
raise ObservatoryRecordedQueueConflictError(
"claim request id is bound to another claimant"
)
return self._claim_from_receipt(connection, receipt)
self._require_capacity(
connection,
table="observatory_recorded_claim_receipts",
limit=self._max_claim_receipts,
label="claim receipt",
receipt_table = (
"observatory_recorded_claim_grants_v3"
if protocol_version == 3
else "observatory_recorded_claim_receipts"
)
if protocol_version == 2:
self._require_capacity(
connection,
table=receipt_table,
limit=self._max_claim_receipts,
label="legacy claim receipt",
)
row = None
if self._open_live_lease_row(connection) is None:
active_owner = connection.execute(
@@ -1368,6 +1444,8 @@ class ObservatoryRecordedJobQueue:
supported_executor_identities=capabilities,
)
if row is None:
if protocol_version == 3:
return None
connection.execute(
"INSERT INTO observatory_recorded_claim_receipts "
"(claim_request_id, request_sha256, claimant_id, created_at_utc) "
@@ -1375,6 +1453,13 @@ class ObservatoryRecordedJobQueue:
(claim_request_id, request_sha256, claimant_id, now),
)
return None
if protocol_version == 3:
self._require_capacity(
connection,
table=receipt_table,
limit=self._max_v3_claim_receipts,
label="v3 claim grant",
)
job_id = str(row["job_id"])
claim_token = hashlib.sha256(
f"{uuid4().hex}:{job_id}:{claim_request_id}".encode()
@@ -1405,7 +1490,7 @@ class ObservatoryRecordedJobQueue:
"recorded-job claim lost its serialized state"
)
connection.execute(
"INSERT INTO observatory_recorded_claim_receipts "
f"INSERT INTO {receipt_table} "
"(claim_request_id, request_sha256, claimant_id, job_id, "
"claim_token, created_at_utc) VALUES (?, ?, ?, ?, ?, ?)",
(
@@ -2679,6 +2764,7 @@ class ObservatoryRecordedJobQueue:
expected = {
"observatory_recorded_jobs": 50,
"observatory_recorded_claim_receipts": 6,
"observatory_recorded_claim_grants_v3": 6,
"observatory_recorded_reconciliations": 18,
"observatory_live_leases": 13,
"observatory_recorded_preemptions": 14,
@@ -2819,6 +2905,11 @@ class ObservatoryRecordedJobQueue:
self._max_claim_receipts,
"claim receipt",
),
(
"observatory_recorded_claim_grants_v3",
self._max_v3_claim_receipts,
"v3 claim grant",
),
("observatory_live_leases", self._max_live_leases, "live lease"),
(
"observatory_recorded_preemptions",
src/k1link/observatory/worker_http_transport.py
+1 -1
View File
@@ -213,7 +213,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
"POST",
"/api/v1/worker/observatory/recorded-jobs/claims",
json_body={
"schema_version": "missioncore.observatory-worker-claim-request/v2",
"schema_version": "missioncore.observatory-worker-claim-request/v3",
"claim_request_id": claim_request_id,
"supported_executor_identities": [
identity.as_dict()
src/k1link/web/observatory_worker_api.py
+14 -2
View File
@@ -158,7 +158,10 @@ class ObservatoryWorkerExecutorCapability(_StrictWorkerRequest):
class ObservatoryWorkerClaimRequest(_StrictWorkerRequest):
schema_version: Literal["missioncore.observatory-worker-claim-request/v2"]
schema_version: Literal[
"missioncore.observatory-worker-claim-request/v2",
"missioncore.observatory-worker-claim-request/v3",
]
claim_request_id: str = Field(
min_length=1,
max_length=160,
@@ -276,6 +279,10 @@ def build_observatory_worker_router(
dependencies=[Depends(require_configured_worker)],
)
@router.get("/recorded-jobs/claim-readiness")
def claim_readiness() -> dict[str, object]:
return _queue_call(queue.claim_readiness)
@router.post("/recorded-jobs/claims", response_model=None)
def claim_next(
request: ObservatoryWorkerClaimRequest,
@@ -288,10 +295,15 @@ def build_observatory_worker_router(
capability.recorded_identity()
for capability in request.supported_executor_identities
),
protocol_version=(3 if request.schema_version.endswith("/v3") else 2),
)
)
if claim is None:
return Response(status_code=204)
return Response(status_code=204, headers={
"X-Mission-Core-Idle-Claim": (
"non-binding" if request.schema_version.endswith("/v3") else "durable"
),
})
return claim.as_dict()
@router.get("/recorded-jobs/{job_id}")