fix(observatory): reject duplicate active portable computations atomically
This commit is contained in:
@@ -214,6 +214,7 @@ class PortableRecordedQueueBindingService:
|
||||
return self._queue.submit(
|
||||
preparation.intent(idempotency_key=idempotency_key),
|
||||
enqueue=enqueue,
|
||||
reject_duplicate_computation=True,
|
||||
)
|
||||
|
||||
def _resolve_definition(
|
||||
|
||||
@@ -239,6 +239,14 @@ class ObservatoryRecordedQueueConflictError(ObservatoryRecordedQueueError):
|
||||
"""An idempotency identity or state transition conflicts with history."""
|
||||
|
||||
|
||||
class ObservatoryRecordedQueueDuplicateError(ObservatoryRecordedQueueConflictError):
|
||||
"""Another request already owns this exact recorded computation."""
|
||||
|
||||
def __init__(self, job_id: str) -> None:
|
||||
self.job_id = job_id
|
||||
super().__init__("an identical recorded computation already exists")
|
||||
|
||||
|
||||
class ObservatoryRecordedQueueNotFoundError(ObservatoryRecordedQueueError):
|
||||
"""A requested job or live lease does not exist."""
|
||||
|
||||
@@ -1177,6 +1185,7 @@ class ObservatoryRecordedJobQueue:
|
||||
intent: ObservatoryRecordedJobIntent,
|
||||
*,
|
||||
enqueue: bool = False,
|
||||
reject_duplicate_computation: bool = False,
|
||||
) -> tuple[ObservatoryRecordedJob, bool]:
|
||||
"""Accept one immutable identity and optionally queue it atomically."""
|
||||
|
||||
@@ -1199,16 +1208,32 @@ class ObservatoryRecordedJobQueue:
|
||||
)
|
||||
record = self._get_job(connection, record.job_id)
|
||||
return record, False
|
||||
definition = self._definitions.resolve(intent.setup_id, intent.definition_sha256)
|
||||
identity_sha256 = _job_identity_sha256(intent, definition)
|
||||
if reject_duplicate_computation:
|
||||
# This check shares the INSERT transaction: two clients with
|
||||
# different idempotency keys cannot race into duplicate jobs.
|
||||
# A sealed result awaiting publication is not a reason to run
|
||||
# the models again. Published-cache validity is a separate gate.
|
||||
duplicate = connection.execute(
|
||||
"SELECT job_id FROM observatory_recorded_jobs "
|
||||
"WHERE identity_sha256 = ? AND ("
|
||||
"state IN ('accepted', 'queued', 'claimed', 'running', 'paused', "
|
||||
"'preemption-pending', 'reconciliation-required') OR "
|
||||
"(state = 'succeeded' AND publication_state IN ('pending', 'failed'))) "
|
||||
"ORDER BY created_at_utc DESC, job_id DESC LIMIT 1",
|
||||
(identity_sha256,),
|
||||
).fetchone()
|
||||
if duplicate is not None:
|
||||
raise ObservatoryRecordedQueueDuplicateError(duplicate["job_id"])
|
||||
self._require_capacity(
|
||||
connection,
|
||||
table="observatory_recorded_jobs",
|
||||
limit=self._max_jobs,
|
||||
label="recorded job",
|
||||
)
|
||||
definition = self._definitions.resolve(intent.setup_id, intent.definition_sha256)
|
||||
now = self._timestamp()
|
||||
job_id = f"observatory-run-{uuid4().hex}"
|
||||
identity_sha256 = _job_identity_sha256(intent, definition)
|
||||
receipt_sha256 = _submission_receipt_sha256(
|
||||
job_id=job_id,
|
||||
request_sha256=intent.request_sha256,
|
||||
|
||||
@@ -53,6 +53,7 @@ from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueCapacityError,
|
||||
ObservatoryRecordedQueueConflictError,
|
||||
ObservatoryRecordedQueueDuplicateError,
|
||||
ObservatoryRecordedQueueError,
|
||||
ObservatoryRecordedQueueNotFoundError,
|
||||
)
|
||||
@@ -929,6 +930,14 @@ def build_observatory_router(
|
||||
status_code=409,
|
||||
detail="Portable-привязка источника не прошла проверку целостности.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueDuplicateError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
"Такой расчёт уже выполняется или ожидает публикации. "
|
||||
"Обновите список расчётов; повторный запуск не создан."
|
||||
),
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueConflictError as exc:
|
||||
try:
|
||||
raced = recorded_job_queue.get_by_idempotency_key(request.idempotency_key)
|
||||
|
||||
Reference in New Issue
Block a user