fix(observatory): reject duplicate active portable computations atomically

This commit is contained in:
DCCONSTRUCTIONS
2026-09-02 23:15:39 +03:00
parent 941183d624
commit 35c1249631
6 changed files with 226 additions and 3 deletions
@@ -214,6 +214,7 @@ class PortableRecordedQueueBindingService:
return self._queue.submit(
preparation.intent(idempotency_key=idempotency_key),
enqueue=enqueue,
reject_duplicate_computation=True,
)
def _resolve_definition(
+27 -2
View File
@@ -239,6 +239,14 @@ class ObservatoryRecordedQueueConflictError(ObservatoryRecordedQueueError):
"""An idempotency identity or state transition conflicts with history."""
class ObservatoryRecordedQueueDuplicateError(ObservatoryRecordedQueueConflictError):
"""Another request already owns this exact recorded computation."""
def __init__(self, job_id: str) -> None:
self.job_id = job_id
super().__init__("an identical recorded computation already exists")
class ObservatoryRecordedQueueNotFoundError(ObservatoryRecordedQueueError):
"""A requested job or live lease does not exist."""
@@ -1177,6 +1185,7 @@ class ObservatoryRecordedJobQueue:
intent: ObservatoryRecordedJobIntent,
*,
enqueue: bool = False,
reject_duplicate_computation: bool = False,
) -> tuple[ObservatoryRecordedJob, bool]:
"""Accept one immutable identity and optionally queue it atomically."""
@@ -1199,16 +1208,32 @@ class ObservatoryRecordedJobQueue:
)
record = self._get_job(connection, record.job_id)
return record, False
definition = self._definitions.resolve(intent.setup_id, intent.definition_sha256)
identity_sha256 = _job_identity_sha256(intent, definition)
if reject_duplicate_computation:
# This check shares the INSERT transaction: two clients with
# different idempotency keys cannot race into duplicate jobs.
# A sealed result awaiting publication is not a reason to run
# the models again. Published-cache validity is a separate gate.
duplicate = connection.execute(
"SELECT job_id FROM observatory_recorded_jobs "
"WHERE identity_sha256 = ? AND ("
"state IN ('accepted', 'queued', 'claimed', 'running', 'paused', "
"'preemption-pending', 'reconciliation-required') OR "
"(state = 'succeeded' AND publication_state IN ('pending', 'failed'))) "
"ORDER BY created_at_utc DESC, job_id DESC LIMIT 1",
(identity_sha256,),
).fetchone()
if duplicate is not None:
raise ObservatoryRecordedQueueDuplicateError(duplicate["job_id"])
self._require_capacity(
connection,
table="observatory_recorded_jobs",
limit=self._max_jobs,
label="recorded job",
)
definition = self._definitions.resolve(intent.setup_id, intent.definition_sha256)
now = self._timestamp()
job_id = f"observatory-run-{uuid4().hex}"
identity_sha256 = _job_identity_sha256(intent, definition)
receipt_sha256 = _submission_receipt_sha256(
job_id=job_id,
request_sha256=intent.request_sha256,
+9
View File
@@ -53,6 +53,7 @@ from k1link.observatory.recorded_jobs import (
ObservatoryRecordedJobQueue,
ObservatoryRecordedQueueCapacityError,
ObservatoryRecordedQueueConflictError,
ObservatoryRecordedQueueDuplicateError,
ObservatoryRecordedQueueError,
ObservatoryRecordedQueueNotFoundError,
)
@@ -929,6 +930,14 @@ def build_observatory_router(
status_code=409,
detail="Portable-привязка источника не прошла проверку целостности.",
) from exc
except ObservatoryRecordedQueueDuplicateError as exc:
raise HTTPException(
status_code=409,
detail=(
"Такой расчёт уже выполняется или ожидает публикации. "
"Обновите список расчётов; повторный запуск не создан."
),
) from exc
except ObservatoryRecordedQueueConflictError as exc:
try:
raced = recorded_job_queue.get_by_idempotency_key(request.idempotency_key)