feat(observatory): expose attempt-bound recorded progress

This commit is contained in:
DCCONSTRUCTIONS
2026-09-03 11:17:56 +03:00
parent 4e0565eb4b
commit bee8552003
17 changed files with 960 additions and 11 deletions
@@ -47,6 +47,7 @@ from k1link.observatory.portable_worker_runtime import (
PortableWorkerSourceStage,
inspect_runtime_candidate,
)
from k1link.observatory.recorded_progress import report_recorded_progress
from k1link.observatory.worker_agent import ObservatoryWorkerExecutorRegistration
if TYPE_CHECKING:
@@ -467,7 +468,9 @@ class InstalledLabPackageProfileRunner:
}
),
)
for container in _topological_containers(self.package):
containers = _topological_containers(self.package)
report_recorded_progress("computing", 0, len(containers), "steps")
for step_index, container in enumerate(containers):
container_output_root = output_root
if container.role == "step":
container_output_root = steps_root / container.container_id
@@ -483,6 +486,8 @@ class InstalledLabPackageProfileRunner:
name_token=attempt_token,
)
)
report_recorded_progress("computing", step_index + 1, len(containers), "steps")
report_recorded_progress("result-assembly", unit="steps")
return _read_result_draft(
output_root,
plan=plan,
@@ -19,7 +19,9 @@ import shutil
import stat
import subprocess
import tempfile
import time
from collections.abc import Callable
from contextlib import suppress
from dataclasses import dataclass
from pathlib import Path
from typing import Final, Protocol, cast
@@ -36,6 +38,7 @@ from k1link.observatory.m49_portable_source import (
materialize_m49_portable_source_from_worker_stage,
validate_m49_portable_source_stage_binding,
)
from k1link.observatory.m49_timing_progress import M49TimingProgress
from k1link.observatory.portable_result_contract import (
OBSERVATION_ONLY_AUTHORITY,
canonical_json,
@@ -53,6 +56,7 @@ from k1link.observatory.portable_worker_runtime import (
PortableWorkerSourceMaterializer,
PortableWorkerSourceStage,
)
from k1link.observatory.recorded_progress import report_recorded_progress
from k1link.observatory.worker_agent import SealedObservatoryRecordedJob
M49_COMPILED_RUNNER_BUILD_SCHEMA: Final = "missioncore.m49-tgs-portable-compiled-runner-build/v1"
@@ -257,6 +261,7 @@ class M49PortableProfileRunnerAdapter:
try:
output = workspace / "outputs"
timing = workspace / "timing.tsv"
report_recorded_progress("computing", 0, stage.timeline_frame_count)
invoker = self.invoker or _invoke_exact_runner
invoker(
binary=self.installation.runner_binary_path,
@@ -267,6 +272,7 @@ class M49PortableProfileRunnerAdapter:
workspace=workspace,
timeout_seconds=self.installation.timeout_seconds,
)
report_recorded_progress("result-assembly", unit="steps")
package = assemble_m49_portable_result(
source_stage=stage,
runner_output_root=output,
@@ -335,19 +341,32 @@ def _invoke_exact_runner(
stderr = workspace / "runner.stderr.log"
try:
with stdout.open("xb") as stdout_stream, stderr.open("xb") as stderr_stream:
completed = subprocess.run(
started = time.monotonic()
progress = M49TimingProgress(timing)
process = subprocess.Popen(
[str(binary), str(sequence), str(schedule), str(output), str(timing)],
cwd=workspace,
env={"LANG": "C", "LC_ALL": "C", "TZ": "UTC"},
stdin=subprocess.DEVNULL,
stdout=stdout_stream,
stderr=stderr_stream,
check=False,
timeout=timeout_seconds,
)
try:
while process.poll() is None:
report_recorded_progress("computing", progress.poll())
remaining = timeout_seconds - (time.monotonic() - started)
if remaining <= 0:
raise subprocess.TimeoutExpired(str(binary), timeout_seconds)
with suppress(subprocess.TimeoutExpired):
process.wait(timeout=min(0.5, remaining))
report_recorded_progress("computing", progress.poll())
finally:
if process.poll() is None:
process.kill()
process.wait()
except (OSError, subprocess.SubprocessError) as exc:
raise M49PortableExecutorError("portable M4.9 runner invocation failed") from exc
if completed.returncode != 0:
if process.returncode != 0:
raise M49PortableExecutorError("portable M4.9 runner rejected its exact source stage")
@@ -34,6 +34,7 @@ import numpy.typing as npt
from k1link.compute.lidar_replay import LidarReplayPackV2, build_lidar_replay_pack_v2
from k1link.observatory.portable_result_contract import canonical_json
from k1link.observatory.recorded_progress import report_recorded_progress
from k1link.observatory.source_admission import (
PORTABLE_SOURCE_BUNDLE_SCHEMA,
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
@@ -175,6 +176,7 @@ def materialize_m49_portable_source_from_worker_stage(
or worker_stage.source_adapter_sha256 != job.source_adapter_sha256
):
raise M49PortableSourceError("Worker source stage belongs to another job")
report_recorded_progress("source-preparation")
root = _safe_directory(worker_stage.root, "Worker source stage")
manifest_payload, manifest = _read_canonical_document(
root / "materialization-manifest.json",
@@ -487,6 +489,7 @@ def _materialize_stage(
index_rows: list[dict[str, object]] = []
available_slot = 0
sequence_logical = hashlib.sha256()
report_recorded_progress("source-preparation", 0, len(anchors))
for anchor in anchors:
point_index = int(
np.searchsorted(point_times, anchor.session_seconds, side="right") - 1
@@ -534,6 +537,9 @@ def _materialize_stage(
f"{anchor.timeline_frame_index}\t{anchor.source_frame_index}"
f"\t{anchor.session_seconds:.9f}\t-1\t0"
)
report_recorded_progress(
"source-preparation", anchor.timeline_frame_index + 1, len(anchors),
)
continue
start_seconds = anchor.session_seconds - profile.history_seconds
@@ -593,6 +599,9 @@ def _materialize_stage(
f"\t{anchor.session_seconds:.9f}\t{available_slot}\t{native.shape[0]}"
)
available_slot += 1
report_recorded_progress(
"source-preparation", anchor.timeline_frame_index + 1, len(anchors),
)
if available_slot < 1:
raise M49PortableSourceError("portable K1 source has no admissible LiDAR frames")
@@ -0,0 +1,43 @@
"""Incremental observation of flushed TGS timing rows, not result validation."""
from pathlib import Path
class M49TimingProgress:
def __init__(self, path: Path) -> None:
self.path = path
self.offset = 0
self.pending = b""
self.header = False
self.completed = 0
self.invalid = False
def poll(self) -> int:
if self.invalid:
return self.completed
try:
with self.path.open("rb") as stream:
stream.seek(self.offset)
data = stream.read(64 * 1024)
self.offset += len(data)
except OSError:
return self.completed
lines = (self.pending + data).split(b"\n")
self.pending = lines.pop()
if len(self.pending) > 2048:
self.invalid = True
self.pending = b""
return self.completed
for line in lines:
if not self.header:
self.header = line.startswith(b"timeline_frame_index\tsource_frame_index\t")
if not self.header:
self.invalid = True
break
continue
fields = line.split(b"\t")
if len(fields) != 10 or fields[0] != str(self.completed).encode():
self.invalid = True
break
self.completed += 1
return self.completed
@@ -27,6 +27,7 @@ from k1link.observatory.portable_run_definitions import (
PortableRunDefinitionRegistry,
canonical_sha256,
)
from k1link.observatory.recorded_progress import report_recorded_progress
from k1link.observatory.worker_agent import (
ObservatoryWorkerExecutionResult,
ObservatoryWorkerExecutorIdentity,
@@ -732,6 +733,7 @@ class PortableWorkerExecutorAdapter:
job: SealedObservatoryRecordedJob,
) -> ObservatoryWorkerExecutionResult:
self._verify_job(job)
report_recorded_progress("source-transfer", unit="members")
source = self.source_materializer.materialize(job)
if (
source.source_bundle_sha256 != job.source_bundle_sha256
@@ -764,11 +766,13 @@ class PortableWorkerExecutorAdapter:
result_contract_sha256=self.candidate.result_contract_sha256,
phases=tuple(phase.phase_id for phase in self.candidate.phases),
)
report_recorded_progress("computing", unit="steps")
draft = self.runner.run(plan, source)
if draft.result_contract_sha256 != self.candidate.result_contract_sha256:
raise PortableWorkerRuntimeJobRejectedError(
"runtime result uses another result contract"
)
report_recorded_progress("result-transfer", unit="members")
published = self.publisher.publish(job, draft)
if (
published.result_id != draft.result_id
+75
View File
@@ -32,6 +32,7 @@ from typing import Final, Literal
from uuid import uuid4
from k1link.artifacts import utc_now_iso
from k1link.observatory.recorded_progress import RecordedProgress
OBSERVATORY_RECORDED_JOB_SCHEMA: Final = "missioncore.observatory-recorded-job/v1"
OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA: Final = "missioncore.observatory-recorded-job-request/v1"
@@ -159,6 +160,12 @@ CREATE TABLE IF NOT EXISTS observatory_recorded_jobs (
CREATE INDEX IF NOT EXISTS observatory_recorded_jobs_queue_order
ON observatory_recorded_jobs (state, priority_rank, created_at_utc, job_id);
CREATE TABLE IF NOT EXISTS observatory_recorded_progress (
job_id TEXT PRIMARY KEY REFERENCES observatory_recorded_jobs(job_id),
snapshot_json TEXT NOT NULL CHECK (length(snapshot_json) <= 2048),
received_at_utc TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS observatory_recorded_claim_receipts (
claim_request_id TEXT PRIMARY KEY,
request_sha256 TEXT NOT NULL,
@@ -1614,6 +1621,73 @@ class ObservatoryRecordedJobQueue:
)
return self._get_job(connection, job_id)
def report_progress(
self, job_id: str, *, claim_token: str, claimant_id: str,
progress: RecordedProgress,
) -> None:
"""Replace one small observation, fenced in the ownership transaction."""
_validate_pattern(job_id, _JOB_ID, "recorded job id")
_validate_pattern(claim_token, _TOKEN, "claim token")
payload = progress.model_dump_json()
with self._transaction() as connection:
job = self._get_job(connection, job_id)
now = self._timestamp()
self._require_active_claim(job, claim_token, now=now)
if (
job.claim_generation != progress.claim_generation
or job.active_claimant_id != claimant_id
):
raise ObservatoryRecordedQueueStaleClaimError("progress owner is stale")
if job.state != "running":
raise ObservatoryRecordedQueueConflictError("progress requires running execution")
row = connection.execute(
"SELECT snapshot_json FROM observatory_recorded_progress WHERE job_id = ?",
(job_id,),
).fetchone()
if row is not None:
previous = RecordedProgress.model_validate_json(row["snapshot_json"])
if previous.claim_generation == progress.claim_generation:
if previous == progress:
return
if not progress.follows(previous):
raise ObservatoryRecordedQueueConflictError("progress snapshot regressed")
connection.execute(
"INSERT INTO observatory_recorded_progress VALUES (?, ?, ?) "
"ON CONFLICT(job_id) DO UPDATE SET snapshot_json = excluded.snapshot_json, "
"received_at_utc = excluded.received_at_utc",
(job_id, payload, now),
)
def progress(self, job_id: str) -> dict[str, object]:
"""Read-only projection; never claims, renews, publishes or computes."""
_validate_pattern(job_id, _JOB_ID, "recorded job id")
with self._read_connection() as connection:
job = self._get_job(connection, job_id)
row = connection.execute(
"SELECT snapshot_json, received_at_utc FROM observatory_recorded_progress "
"WHERE job_id = ?", (job_id,),
).fetchone()
progress = None if row is None else RecordedProgress.model_validate_json(
row["snapshot_json"]
)
if progress is not None and progress.claim_generation != job.claim_generation:
progress = None
return {
"schema_version": "missioncore.observatory-recorded-progress-view/v1",
"job_id": job.job_id,
"source_session_id": job.source_session_id,
"setup_id": job.setup_id,
"definition_sha256": job.definition_sha256,
"claim_generation": job.claim_generation,
"state": job.state,
"received_at_utc": None if progress is None else row["received_at_utc"],
"age_seconds": None if progress is None else max(0.0, (
_parse_timestamp(self._timestamp(), "clock")
- _parse_timestamp(row["received_at_utc"], "progress receipt")
).total_seconds()),
"progress": None if progress is None else progress.model_dump(mode="json"),
}
def authorize_claim_access(
self,
job_id: str,
@@ -2816,6 +2890,7 @@ class ObservatoryRecordedJobQueue:
def _validate_schema(self, connection: sqlite3.Connection) -> None:
expected = {
"observatory_recorded_progress": 3,
"observatory_recorded_jobs": 50,
"observatory_recorded_claim_receipts": 6,
"observatory_recorded_claim_grants_v3": 6,
+166
View File
@@ -0,0 +1,166 @@
"""Bounded, attempt-scoped observation of recorded execution, never authority.
The execution thread owns counters; a separate bounded sender samples them.
Progress loss cannot cancel compute or renew a claim. No frame event journal,
estimated completion, filesystem paths, or model-dependent UI contract.
"""
from __future__ import annotations
import math
import threading
import time
from collections.abc import Callable, Iterator
from contextlib import contextmanager, suppress
from contextvars import ContextVar
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, model_validator
type ProgressPhase = Literal[
"source-transfer",
"source-preparation",
"computing",
"result-assembly",
"result-transfer",
]
type ProgressUnit = Literal["frames", "members", "steps"]
class RecordedProgress(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
schema_version: Literal["missioncore.observatory-recorded-progress/v1"] = (
"missioncore.observatory-recorded-progress/v1"
)
claim_generation: int = Field(ge=1, le=2**53 - 1)
sequence: int = Field(ge=1, le=2**53 - 1)
phase_index: int = Field(ge=0, le=2**53 - 1)
phase: ProgressPhase
unit: ProgressUnit
completed: int = Field(ge=0, le=2**53 - 1)
total: int | None = Field(default=None, ge=1, le=2**53 - 1)
elapsed_seconds: float = Field(ge=0, allow_inf_nan=False)
phase_elapsed_seconds: float = Field(ge=0, allow_inf_nan=False)
@model_validator(mode="after")
def validate_counts(self) -> RecordedProgress:
if self.total is not None and self.completed > self.total:
raise ValueError("completed exceeds the observed total")
if self.phase_elapsed_seconds > self.elapsed_seconds:
raise ValueError("phase duration exceeds execution duration")
return self
def follows(self, previous: RecordedProgress) -> bool:
if self.claim_generation != previous.claim_generation:
return False
if self.sequence <= previous.sequence or self.phase_index < previous.phase_index:
return False
if self.elapsed_seconds < previous.elapsed_seconds:
return False
if self.phase_index != previous.phase_index:
return True
return (
self.phase == previous.phase
and self.unit == previous.unit
and self.completed >= previous.completed
and (previous.total is None or self.total == previous.total)
and self.phase_elapsed_seconds >= previous.phase_elapsed_seconds
)
class RecordedProgressTracker:
def __init__(self, generation: int, *, clock: Callable[[], float] = time.monotonic):
self.generation = generation
self.clock = clock
self.started = clock()
self.phase_started = self.started
self.lock = threading.Lock()
self.phase: ProgressPhase = "source-transfer"
self.unit: ProgressUnit = "members"
self.completed = 0
self.total: int | None = None
self.phase_index = 0
self.sequence = 0
def update(
self,
phase: ProgressPhase,
completed: int = 0,
total: int | None = None,
unit: ProgressUnit = "frames",
) -> None:
with self.lock:
if phase != self.phase or unit != self.unit:
self.phase_index += 1
self.phase_started = self.clock()
elif completed < self.completed:
raise ValueError("progress counter moved backwards within one phase")
elif total is None:
total = self.total
self.phase, self.unit = phase, unit
self.completed, self.total = completed, total
def sample(self) -> RecordedProgress:
with self.lock:
now = self.clock()
self.sequence += 1
return RecordedProgress(
claim_generation=self.generation,
sequence=self.sequence,
phase_index=self.phase_index,
phase=self.phase,
unit=self.unit,
completed=self.completed,
total=self.total,
elapsed_seconds=max(0.0, now - self.started),
phase_elapsed_seconds=max(0.0, now - self.phase_started),
)
_CURRENT: ContextVar[RecordedProgressTracker | None] = ContextVar(
"observatory_recorded_progress",
default=None,
)
def report_recorded_progress(
phase: ProgressPhase,
completed: int = 0,
total: int | None = None,
unit: ProgressUnit = "frames",
) -> None:
tracker = _CURRENT.get()
if tracker is not None:
tracker.update(phase, completed, total, unit)
@contextmanager
def observe_recorded_execution(
generation: int,
send: Callable[[RecordedProgress], None],
*,
interval_seconds: float = 1.0,
) -> Iterator[RecordedProgressTracker]:
if not math.isfinite(interval_seconds) or not 0.01 <= interval_seconds <= 30:
raise ValueError("progress sampling interval is invalid")
tracker = RecordedProgressTracker(generation)
stop = threading.Event()
def pump() -> None:
while not stop.is_set():
# Progress is secondary observation, not lease or result authority.
with suppress(Exception):
send(tracker.sample())
stop.wait(interval_seconds)
token = _CURRENT.set(tracker)
thread = threading.Thread(target=pump, name="observatory-progress", daemon=True)
thread.start()
try:
yield tracker
finally:
_CURRENT.reset(token)
stop.set()
# The production sender reads no body and uses 2-second I/O timeouts.
thread.join(timeout=3.0)
+14 -1
View File
@@ -18,6 +18,7 @@ import json
import re
import threading
from collections.abc import Callable, Mapping
from contextlib import nullcontext
from dataclasses import dataclass
from datetime import datetime
from typing import Annotated, Final, Literal, Protocol
@@ -31,6 +32,7 @@ from k1link.observatory.recorded_jobs import (
OBSERVATORY_RECORDED_JOB_SCHEMA,
RecordedExecutorIdentity,
)
from k1link.observatory.recorded_progress import observe_recorded_execution
WORKER_006_CONTOUR_ID: Final = "worker-006"
MAX_EXECUTOR_FAILURE_MESSAGE_LENGTH: Final = 512
@@ -498,7 +500,18 @@ class ObservatoryWorkerAgent:
heartbeat.start()
try:
result = adapter.execute(active_job)
send_progress = getattr(self._transport, "report_progress", None)
observation = (
observe_recorded_execution(
active_job.claim_generation,
lambda snapshot: send_progress(
job_id=active_job.job_id, claim_token=claim.claim_token,
progress=snapshot,
),
) if callable(send_progress) else nullcontext()
)
with observation:
result = adapter.execute(active_job)
if not isinstance(result, ObservatoryWorkerExecutionResult):
raise TypeError("executor returned an unknown result contract")
except Exception as exc:
@@ -47,6 +47,7 @@ from k1link.observatory.portable_worker_runtime import (
PortableWorkerResultDraft,
PortableWorkerSourceStage,
)
from k1link.observatory.recorded_progress import RecordedProgress, report_recorded_progress
from k1link.observatory.source_admission import (
PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID,
PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE,
@@ -279,6 +280,23 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
},
)
def report_progress(
self, *, job_id: str, claim_token: str, progress: RecordedProgress,
) -> None:
context = self._require_cached_claim(job_id, claim_token)
if progress.claim_generation != context.claim_generation:
raise ObservatoryWorkerHttpError("progress generation changed")
# No response body is needed. Bounded I/O cannot stall the execution thread.
with self._client.stream(
"POST", self._job_path(job_id, "progress"),
json={
"schema_version": "missioncore.observatory-worker-progress-request/v1",
"claim_token": claim_token, "progress": progress.model_dump(mode="json"),
}, timeout=httpx.Timeout(2.0),
) as response:
if response.status_code != 204:
raise ObservatoryWorkerHttpError("progress observation was not accepted")
def succeed(
self,
*,
@@ -336,6 +354,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
headers=headers,
)
members = _source_members(manifest, job)
report_recorded_progress("source-transfer", 0, len(members), "members")
root = _secure_directory(
self._work_root
/ "sources"
@@ -368,10 +387,16 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
members=camera_members,
layout=layout,
)
completed_members = len(camera_members) if camera_epoch_ready else 0
report_recorded_progress("source-transfer", completed_members, len(members), "members")
for destination, member in destinations.items():
if camera_epoch_ready and member.kind in {"camera-init", "camera-segment"}:
continue
if _matches_file(destination, member.sha256, member.byte_length):
completed_members += 1
report_recorded_progress(
"source-transfer", completed_members, len(members), "members",
)
continue
self._download_member(
job_id=job.job_id,
@@ -379,6 +404,8 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
member=member,
destination=destination,
)
completed_members += 1
report_recorded_progress("source-transfer", completed_members, len(members), "members")
manifest_path = root / "materialization-manifest.json"
_write_local_exact(manifest_path, canonical_json(manifest))
return PortableWorkerSourceStage(
@@ -440,7 +467,8 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
or {member.role for member in upload_members} != set(artifacts)
):
raise ObservatoryWorkerHttpError("Worker result artifact roles are not unique")
for member in upload_members:
report_recorded_progress("result-transfer", 0, len(upload_members), "members")
for member_index, member in enumerate(upload_members):
artifact = artifacts.get(member.role)
expected_member_id = hashlib.sha256(
canonical_json(
@@ -461,6 +489,9 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
"result upload plan differs from the local manifest"
)
if member.uploaded:
report_recorded_progress(
"result-transfer", member_index + 1, len(upload_members), "members",
)
continue
relative = relative_artifact_path(artifact.relative_path)
source = _confined_local_member(draft.root, relative.parts)
@@ -489,6 +520,9 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
raise ObservatoryWorkerHttpError(
"result upload acknowledgement did not seal its member"
)
report_recorded_progress(
"result-transfer", member_index + 1, len(upload_members), "members",
)
receipt = self._required_json_request(
"POST",
self._job_path(
+11
View File
@@ -1151,6 +1151,17 @@ def build_observatory_router(
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
}
@router.get("/api/v1/observatory/runs/{job_id}/progress")
def get_observatory_progress(
job_id: str = ApiPath(pattern=r"^observatory-run-[a-f0-9]{32}$"),
) -> dict[str, object]:
try:
return recorded_job_queue.progress(job_id)
except ObservatoryRecordedQueueNotFoundError as exc:
raise HTTPException(status_code=404, detail="Расчёт не найден.") from exc
except (ObservatoryRecordedQueueError, ValueError) as exc:
raise HTTPException(status_code=503, detail="Прогресс недоступен.") from exc
@router.get("/api/v1/observatory/runs/{job_id}")
def get_observatory_recorded_run(
job_id: str = ApiPath(
+18
View File
@@ -51,6 +51,7 @@ from k1link.observatory.recorded_jobs import (
ObservatoryRecordedQueueStaleClaimError,
RecordedExecutorIdentity,
)
from k1link.observatory.recorded_progress import RecordedProgress
OBSERVATORY_WORKER_CAPABILITY_CLAIM_REQUEST_SCHEMA: Final = (
"missioncore.observatory-worker-claim-request/v2"
@@ -194,6 +195,12 @@ class ObservatoryWorkerRenewRequest(_StrictWorkerRequest):
heartbeat_sequence: int = Field(ge=1)
class ObservatoryWorkerProgressRequest(_StrictWorkerRequest):
schema_version: Literal["missioncore.observatory-worker-progress-request/v1"]
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
progress: RecordedProgress
class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest):
schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"]
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
@@ -333,6 +340,17 @@ def build_observatory_worker_router(
)
).as_dict()
@router.post("/recorded-jobs/{job_id}/progress", status_code=204)
def report_job_progress(
request: ObservatoryWorkerProgressRequest,
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
) -> Response:
_queue_call(lambda: queue.report_progress(
job_id, claim_token=request.claim_token,
claimant_id=authentication.contour_id, progress=request.progress,
))
return Response(status_code=204)
@router.post("/recorded-jobs/{job_id}/checkpoint")
def checkpoint_job(
request: ObservatoryWorkerCheckpointRequest,