feat(observatory): expose attempt-bound recorded progress

This commit is contained in:
DCCONSTRUCTIONS
2026-09-03 11:17:56 +03:00
parent 4e0565eb4b
commit bee8552003
17 changed files with 960 additions and 11 deletions
@@ -26,6 +26,7 @@ export interface ObservatoryRecordedJob {
readonly sourceSessionId: string;
readonly setupId: string;
readonly definitionSha256: string;
readonly claimGeneration: number;
readonly state: ObservatoryRecordedJobState;
readonly restartFromZero: boolean;
readonly resultId: string | null;
@@ -229,6 +230,7 @@ function decodeJob(value: unknown): ObservatoryRecordedJob {
sourceSessionId: text(source.session_id, "source.session_id"),
setupId: text(setup.setup_id, "setup.setup_id"),
definitionSha256: String(setup.definition_sha256),
claimGeneration: nonNegativeInteger(row.claim_generation, "claim_generation"),
state,
restartFromZero: boolean(row.restart_from_zero, "restart_from_zero"),
resultId: result === null ? null : text(result.result_id, "result.result_id"),
@@ -0,0 +1,118 @@
import type { ObservatoryRecordedJob } from "./recordedJobs";
const PHASES = {
"source-transfer": "Передаём входные данные",
"source-preparation": "Подготавливаем вход",
computing: "Обрабатываем запись",
"result-assembly": "Собираем результат",
"result-transfer": "Передаём результат",
} as const;
const UNITS = { frames: "кадров", members: "файлов", steps: "операций" } as const;
type Phase = keyof typeof PHASES;
type Unit = keyof typeof UNITS;
export interface RecordedProgressView {
readonly jobId: string;
readonly claimGeneration: number;
readonly sequence: number;
readonly phase: Phase;
readonly unit: Unit;
readonly completed: number;
readonly total: number | null;
readonly ageSeconds: number;
}
export async function fetchRecordedProgress(
job: ObservatoryRecordedJob,
{ signal, fetcher = globalThis.fetch }: {
signal?: AbortSignal;
fetcher?: typeof globalThis.fetch;
} = {},
): Promise<RecordedProgressView | null> {
const request = new AbortController();
const abort = () => request.abort();
signal?.addEventListener("abort", abort, { once: true });
if (signal?.aborted) request.abort();
const timer = globalThis.setTimeout(abort, 2_500);
try {
const response = await fetcher(
`/api/v1/observatory/runs/${encodeURIComponent(job.jobId)}/progress`,
{ signal: request.signal, headers: { Accept: "application/json" } },
);
if (!response.ok) throw new Error("Прогресс расчёта недоступен.");
return decodeRecordedProgress(await response.json(), job);
} finally {
globalThis.clearTimeout(timer);
signal?.removeEventListener("abort", abort);
}
}
export function decodeRecordedProgress(
value: unknown, job: ObservatoryRecordedJob,
): RecordedProgressView | null {
const row = object(value);
keys(row, ["schema_version", "job_id", "source_session_id", "setup_id",
"definition_sha256", "claim_generation", "state", "received_at_utc", "age_seconds", "progress"]);
if (row.schema_version !== "missioncore.observatory-recorded-progress-view/v1"
|| row.job_id !== job.jobId || row.source_session_id !== job.sourceSessionId
|| row.setup_id !== job.setupId || row.definition_sha256 !== job.definitionSha256
|| row.claim_generation !== job.claimGeneration) throw new Error("Прогресс другой попытки.");
if (row.progress === null || row.state !== job.state) return null;
const progress = object(row.progress);
keys(progress, ["schema_version", "claim_generation", "sequence", "phase_index",
"phase", "unit", "completed", "total", "elapsed_seconds", "phase_elapsed_seconds"]);
if (progress.schema_version !== "missioncore.observatory-recorded-progress/v1"
|| progress.claim_generation !== row.claim_generation
|| !(typeof progress.phase === "string" && Object.hasOwn(PHASES, progress.phase))
|| !(typeof progress.unit === "string" && Object.hasOwn(UNITS, progress.unit))
|| typeof row.received_at_utc !== "string" || !Number.isFinite(Date.parse(row.received_at_utc))) {
throw new Error("Некорректный прогресс.");
}
const completed = integer(progress.completed);
const total = progress.total === null ? null : integer(progress.total, 1);
const elapsed = finite(progress.elapsed_seconds);
if ((total !== null && completed > total) || finite(progress.phase_elapsed_seconds) > elapsed) {
throw new Error("Некорректные счётчики прогресса.");
}
integer(progress.phase_index);
return {
jobId: job.jobId, claimGeneration: integer(progress.claim_generation, 1),
sequence: integer(progress.sequence, 1), phase: progress.phase as Phase,
unit: progress.unit as Unit, completed, total, ageSeconds: finite(row.age_seconds),
};
}
export function recordedProgressLabel(
job: ObservatoryRecordedJob | null, progress: RecordedProgressView | null,
): string {
if (job?.publication.state === "pending") return "Сохраняем результат";
if (!job || job.state === "accepted" || job.state === "queued") return "Ожидаем расчёт";
if (job.state === "paused" || job.state === "preemption-pending") return "Расчёт приостановлен";
if (job.state === "reconciliation-required") return "Проверяем состояние расчёта";
if (!progress || progress.jobId !== job.jobId
|| progress.claimGeneration !== job.claimGeneration) return "Ожидаем данные расчёта";
if (progress.ageSeconds > 15) return "Ожидаем обновление прогресса";
const count = progress.total === null
? (progress.completed > 0 ? ` · ${progress.completed.toLocaleString("ru-RU")} ${UNITS[progress.unit]}` : "")
: ` · ${progress.completed.toLocaleString("ru-RU")} / ${progress.total.toLocaleString("ru-RU")} ${UNITS[progress.unit]}`;
return PHASES[progress.phase] + count;
}
function object(value: unknown): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Прогресс: ожидался объект.");
return value as Record<string, unknown>;
}
function keys(row: Record<string, unknown>, expected: string[]): void {
if (Object.keys(row).length !== expected.length || expected.some((key) => !Object.hasOwn(row, key))) {
throw new Error("Прогресс: неизвестные поля.");
}
}
function finite(value: unknown): number {
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new Error("Прогресс: некорректное число.");
return value;
}
function integer(value: unknown, minimum = 0): number {
const number = finite(value);
if (!Number.isSafeInteger(number) || number < minimum) throw new Error("Прогресс: некорректный счётчик.");
return number;
}
@@ -1,4 +1,5 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { fetchRecordedProgress, type RecordedProgressView } from "./recordedProgress";
import {
fetchObservatoryRecordedJobs,
@@ -21,6 +22,7 @@ interface JobSnapshot {
readonly jobs: readonly ObservatoryRecordedJob[];
readonly state: RecordedJobsState;
readonly error: string | null;
readonly progress?: RecordedProgressView | null;
}
const EMPTY_JOBS = [] as const;
const OPEN_STATES = new Set([
@@ -63,14 +65,19 @@ export function useObservatoryRecordedJobs(
setSnapshot({
selectionKey, jobs,
state: jobs.length > 0 ? "refreshing" : "loading", error: null,
progress: current?.progress ?? null,
});
void fetchObservatoryRecordedJobs(sourceSessionId, setupId, {
definitionSha256, signal: request.signal,
}).then((next) => {
}).then(async (next) => {
if (request.signal.aborted || requestSequence.current !== sequence) return;
const pending = next.find((job) => OPEN_STATES.has(job.state));
const progress = pending
? await fetchRecordedProgress(pending, { signal: request.signal }).catch(() => null)
: null;
if (request.signal.aborted || requestSequence.current !== sequence) return;
if (pending) observedJobId.current = pending.jobId;
setSnapshot({ selectionKey, jobs: next, state: "ready", error: null });
setSnapshot({ selectionKey, jobs: next, state: "ready", error: null, progress });
}).catch((caught: unknown) => {
if (request.signal.aborted || requestSequence.current !== sequence) return;
setSnapshot({
@@ -95,7 +102,7 @@ export function useObservatoryRecordedJobs(
&& latestJob.jobId === observedJobId.current;
useEffect(() => {
if (state !== "ready" || (!activeJob && !publicationPending)) return;
if ((state !== "ready" && state !== "error") || (!activeJob && !publicationPending)) return;
const timer = globalThis.setTimeout(
() => setRevision((value) => value + 1),
POLL_INTERVAL_MS,
@@ -185,6 +192,7 @@ export function useObservatoryRecordedJobs(
return {
jobs, latestJob, activeJob, publicationPending, computationFailed,
progress: current?.progress ?? null,
state, error, refresh, submit, retryPublication,
};
}
@@ -33,6 +33,7 @@ import {
import { useObservatoryCatalog } from "../../core/observatory/useObservatoryCatalog";
import { useObservatoryLaboratorySetups } from "../../core/observatory/useObservatoryLaboratorySetups";
import { useObservatoryRecordedJobs } from "../../core/observatory/useObservatoryRecordedJobs";
import { recordedProgressLabel } from "../../core/observatory/recordedProgress";
import type { WorkspaceDefinition } from "../../productModel";
const EMPTY_OBSERVATORY_ITEMS = [] as const;
@@ -466,7 +467,12 @@ export function ObservatoryWorkspace({
</Button>
<div className="observatory-catalog-bar__run" aria-busy={calculationPending}>
{calculationPending ? (
<ActivityIndicator size="compact" label="Ожидание результата расчёта" />
<>
<ActivityIndicator size="compact" />
<span role="status">
{recordedProgressLabel(presentedJob, recordedJobsController.progress)}
</span>
</>
) : null}
{showCalculate ? (
<Button
@@ -0,0 +1,79 @@
import assert from "node:assert/strict";
import { after, before, test } from "node:test";
import { createServer } from "vite";
let server;
let decode;
let label;
let fetchProgress;
const job = {
jobId: "observatory-run-" + "1".repeat(32),
sourceSessionId: "source-a", setupId: "m49-tgs",
definitionSha256: "a".repeat(64), claimGeneration: 2, state: "running",
publication: { state: "not-required" },
};
function view() {
return {
schema_version: "missioncore.observatory-recorded-progress-view/v1",
job_id: job.jobId, source_session_id: job.sourceSessionId, setup_id: job.setupId,
definition_sha256: job.definitionSha256, claim_generation: 2, state: "running",
received_at_utc: "2026-09-03T12:00:00Z", age_seconds: 0.5,
progress: {
schema_version: "missioncore.observatory-recorded-progress/v1",
claim_generation: 2, sequence: 4, phase_index: 2,
phase: "computing", unit: "frames", completed: 3, total: 10,
elapsed_seconds: 8, phase_elapsed_seconds: 5,
},
};
}
before(async () => {
server = await createServer({ configFile: false, logLevel: "silent", server: { middlewareMode: true } });
const module = await server.ssrLoadModule("/src/core/observatory/recordedProgress.ts");
decode = module.decodeRecordedProgress;
label = module.recordedProgressLabel;
fetchProgress = module.fetchRecordedProgress;
});
after(async () => { await server?.close(); });
test("actual frame count is separate from completion and publication", () => {
const progress = decode(view(), job);
assert.equal(label(job, progress), "Обрабатываем запись · 3 / 10 кадров");
assert.equal(label({ ...job, publication: { state: "pending" } }, progress), "Сохраняем результат");
assert.doesNotMatch(label(job, progress), /%|завершён|FPS|ETA/);
});
test("unknown total never becomes a synthetic percentage", () => {
const value = view();
value.progress.total = null;
assert.equal(label(job, decode(value, job)), "Обрабатываем запись · 3 кадров");
value.progress.completed = 0;
assert.equal(label(job, decode(value, job)), "Обрабатываем запись");
});
test("old source definition or attempt cannot flash as current progress", () => {
for (const delta of [
{ job_id: "other" }, { source_session_id: "other" }, { setup_id: "other" },
{ definition_sha256: "b".repeat(64) }, { claim_generation: 1 },
]) assert.throws(() => decode({ ...view(), ...delta }, job));
assert.equal(decode({ ...view(), state: "succeeded" }, job), null);
assert.equal(label({ ...job, claimGeneration: 3 }, decode(view(), job)), "Ожидаем данные расчёта");
});
test("invalid counters, invented phase and extra fields fail closed", () => {
for (const delta of [
{ completed: true }, { completed: 11 }, { completed: -1 }, { total: 0 },
{ completed: Number.MAX_SAFE_INTEGER + 1 }, { phase: "done" }, { path: "/private" },
{ phase_elapsed_seconds: 9 }, { claim_generation: 3 },
]) assert.throws(() => decode({ ...view(), progress: { ...view().progress, ...delta } }, job));
});
test("missing and stale telemetry show no fabricated advancement", () => {
assert.equal(decode({ ...view(), progress: null, age_seconds: null, received_at_utc: null }, job), null);
assert.equal(label(job, null), "Ожидаем данные расчёта");
assert.equal(label(job, decode({ ...view(), age_seconds: 30 }, job)), "Ожидаем обновление прогресса");
});
test("progress is a read-only request bound to the active job", async () => {
const calls = [];
const progress = await fetchProgress(job, { fetcher: async (path, init) => {
calls.push([path, init.method ?? "GET"]);
return new Response(JSON.stringify(view()));
} });
assert.deepEqual(calls, [[`/api/v1/observatory/runs/${job.jobId}/progress`, "GET"]]);
assert.equal(progress.completed, 3);
});
@@ -47,6 +47,7 @@ from k1link.observatory.portable_worker_runtime import (
PortableWorkerSourceStage,
inspect_runtime_candidate,
)
from k1link.observatory.recorded_progress import report_recorded_progress
from k1link.observatory.worker_agent import ObservatoryWorkerExecutorRegistration
if TYPE_CHECKING:
@@ -467,7 +468,9 @@ class InstalledLabPackageProfileRunner:
}
),
)
for container in _topological_containers(self.package):
containers = _topological_containers(self.package)
report_recorded_progress("computing", 0, len(containers), "steps")
for step_index, container in enumerate(containers):
container_output_root = output_root
if container.role == "step":
container_output_root = steps_root / container.container_id
@@ -483,6 +486,8 @@ class InstalledLabPackageProfileRunner:
name_token=attempt_token,
)
)
report_recorded_progress("computing", step_index + 1, len(containers), "steps")
report_recorded_progress("result-assembly", unit="steps")
return _read_result_draft(
output_root,
plan=plan,
@@ -19,7 +19,9 @@ import shutil
import stat
import subprocess
import tempfile
import time
from collections.abc import Callable
from contextlib import suppress
from dataclasses import dataclass
from pathlib import Path
from typing import Final, Protocol, cast
@@ -36,6 +38,7 @@ from k1link.observatory.m49_portable_source import (
materialize_m49_portable_source_from_worker_stage,
validate_m49_portable_source_stage_binding,
)
from k1link.observatory.m49_timing_progress import M49TimingProgress
from k1link.observatory.portable_result_contract import (
OBSERVATION_ONLY_AUTHORITY,
canonical_json,
@@ -53,6 +56,7 @@ from k1link.observatory.portable_worker_runtime import (
PortableWorkerSourceMaterializer,
PortableWorkerSourceStage,
)
from k1link.observatory.recorded_progress import report_recorded_progress
from k1link.observatory.worker_agent import SealedObservatoryRecordedJob
M49_COMPILED_RUNNER_BUILD_SCHEMA: Final = "missioncore.m49-tgs-portable-compiled-runner-build/v1"
@@ -257,6 +261,7 @@ class M49PortableProfileRunnerAdapter:
try:
output = workspace / "outputs"
timing = workspace / "timing.tsv"
report_recorded_progress("computing", 0, stage.timeline_frame_count)
invoker = self.invoker or _invoke_exact_runner
invoker(
binary=self.installation.runner_binary_path,
@@ -267,6 +272,7 @@ class M49PortableProfileRunnerAdapter:
workspace=workspace,
timeout_seconds=self.installation.timeout_seconds,
)
report_recorded_progress("result-assembly", unit="steps")
package = assemble_m49_portable_result(
source_stage=stage,
runner_output_root=output,
@@ -335,19 +341,32 @@ def _invoke_exact_runner(
stderr = workspace / "runner.stderr.log"
try:
with stdout.open("xb") as stdout_stream, stderr.open("xb") as stderr_stream:
completed = subprocess.run(
started = time.monotonic()
progress = M49TimingProgress(timing)
process = subprocess.Popen(
[str(binary), str(sequence), str(schedule), str(output), str(timing)],
cwd=workspace,
env={"LANG": "C", "LC_ALL": "C", "TZ": "UTC"},
stdin=subprocess.DEVNULL,
stdout=stdout_stream,
stderr=stderr_stream,
check=False,
timeout=timeout_seconds,
)
try:
while process.poll() is None:
report_recorded_progress("computing", progress.poll())
remaining = timeout_seconds - (time.monotonic() - started)
if remaining <= 0:
raise subprocess.TimeoutExpired(str(binary), timeout_seconds)
with suppress(subprocess.TimeoutExpired):
process.wait(timeout=min(0.5, remaining))
report_recorded_progress("computing", progress.poll())
finally:
if process.poll() is None:
process.kill()
process.wait()
except (OSError, subprocess.SubprocessError) as exc:
raise M49PortableExecutorError("portable M4.9 runner invocation failed") from exc
if completed.returncode != 0:
if process.returncode != 0:
raise M49PortableExecutorError("portable M4.9 runner rejected its exact source stage")
@@ -34,6 +34,7 @@ import numpy.typing as npt
from k1link.compute.lidar_replay import LidarReplayPackV2, build_lidar_replay_pack_v2
from k1link.observatory.portable_result_contract import canonical_json
from k1link.observatory.recorded_progress import report_recorded_progress
from k1link.observatory.source_admission import (
PORTABLE_SOURCE_BUNDLE_SCHEMA,
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
@@ -175,6 +176,7 @@ def materialize_m49_portable_source_from_worker_stage(
or worker_stage.source_adapter_sha256 != job.source_adapter_sha256
):
raise M49PortableSourceError("Worker source stage belongs to another job")
report_recorded_progress("source-preparation")
root = _safe_directory(worker_stage.root, "Worker source stage")
manifest_payload, manifest = _read_canonical_document(
root / "materialization-manifest.json",
@@ -487,6 +489,7 @@ def _materialize_stage(
index_rows: list[dict[str, object]] = []
available_slot = 0
sequence_logical = hashlib.sha256()
report_recorded_progress("source-preparation", 0, len(anchors))
for anchor in anchors:
point_index = int(
np.searchsorted(point_times, anchor.session_seconds, side="right") - 1
@@ -534,6 +537,9 @@ def _materialize_stage(
f"{anchor.timeline_frame_index}\t{anchor.source_frame_index}"
f"\t{anchor.session_seconds:.9f}\t-1\t0"
)
report_recorded_progress(
"source-preparation", anchor.timeline_frame_index + 1, len(anchors),
)
continue
start_seconds = anchor.session_seconds - profile.history_seconds
@@ -593,6 +599,9 @@ def _materialize_stage(
f"\t{anchor.session_seconds:.9f}\t{available_slot}\t{native.shape[0]}"
)
available_slot += 1
report_recorded_progress(
"source-preparation", anchor.timeline_frame_index + 1, len(anchors),
)
if available_slot < 1:
raise M49PortableSourceError("portable K1 source has no admissible LiDAR frames")
@@ -0,0 +1,43 @@
"""Incremental observation of flushed TGS timing rows, not result validation."""
from pathlib import Path
class M49TimingProgress:
def __init__(self, path: Path) -> None:
self.path = path
self.offset = 0
self.pending = b""
self.header = False
self.completed = 0
self.invalid = False
def poll(self) -> int:
if self.invalid:
return self.completed
try:
with self.path.open("rb") as stream:
stream.seek(self.offset)
data = stream.read(64 * 1024)
self.offset += len(data)
except OSError:
return self.completed
lines = (self.pending + data).split(b"\n")
self.pending = lines.pop()
if len(self.pending) > 2048:
self.invalid = True
self.pending = b""
return self.completed
for line in lines:
if not self.header:
self.header = line.startswith(b"timeline_frame_index\tsource_frame_index\t")
if not self.header:
self.invalid = True
break
continue
fields = line.split(b"\t")
if len(fields) != 10 or fields[0] != str(self.completed).encode():
self.invalid = True
break
self.completed += 1
return self.completed
@@ -27,6 +27,7 @@ from k1link.observatory.portable_run_definitions import (
PortableRunDefinitionRegistry,
canonical_sha256,
)
from k1link.observatory.recorded_progress import report_recorded_progress
from k1link.observatory.worker_agent import (
ObservatoryWorkerExecutionResult,
ObservatoryWorkerExecutorIdentity,
@@ -732,6 +733,7 @@ class PortableWorkerExecutorAdapter:
job: SealedObservatoryRecordedJob,
) -> ObservatoryWorkerExecutionResult:
self._verify_job(job)
report_recorded_progress("source-transfer", unit="members")
source = self.source_materializer.materialize(job)
if (
source.source_bundle_sha256 != job.source_bundle_sha256
@@ -764,11 +766,13 @@ class PortableWorkerExecutorAdapter:
result_contract_sha256=self.candidate.result_contract_sha256,
phases=tuple(phase.phase_id for phase in self.candidate.phases),
)
report_recorded_progress("computing", unit="steps")
draft = self.runner.run(plan, source)
if draft.result_contract_sha256 != self.candidate.result_contract_sha256:
raise PortableWorkerRuntimeJobRejectedError(
"runtime result uses another result contract"
)
report_recorded_progress("result-transfer", unit="members")
published = self.publisher.publish(job, draft)
if (
published.result_id != draft.result_id
+75
View File
@@ -32,6 +32,7 @@ from typing import Final, Literal
from uuid import uuid4
from k1link.artifacts import utc_now_iso
from k1link.observatory.recorded_progress import RecordedProgress
OBSERVATORY_RECORDED_JOB_SCHEMA: Final = "missioncore.observatory-recorded-job/v1"
OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA: Final = "missioncore.observatory-recorded-job-request/v1"
@@ -159,6 +160,12 @@ CREATE TABLE IF NOT EXISTS observatory_recorded_jobs (
CREATE INDEX IF NOT EXISTS observatory_recorded_jobs_queue_order
ON observatory_recorded_jobs (state, priority_rank, created_at_utc, job_id);
CREATE TABLE IF NOT EXISTS observatory_recorded_progress (
job_id TEXT PRIMARY KEY REFERENCES observatory_recorded_jobs(job_id),
snapshot_json TEXT NOT NULL CHECK (length(snapshot_json) <= 2048),
received_at_utc TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS observatory_recorded_claim_receipts (
claim_request_id TEXT PRIMARY KEY,
request_sha256 TEXT NOT NULL,
@@ -1614,6 +1621,73 @@ class ObservatoryRecordedJobQueue:
)
return self._get_job(connection, job_id)
def report_progress(
self, job_id: str, *, claim_token: str, claimant_id: str,
progress: RecordedProgress,
) -> None:
"""Replace one small observation, fenced in the ownership transaction."""
_validate_pattern(job_id, _JOB_ID, "recorded job id")
_validate_pattern(claim_token, _TOKEN, "claim token")
payload = progress.model_dump_json()
with self._transaction() as connection:
job = self._get_job(connection, job_id)
now = self._timestamp()
self._require_active_claim(job, claim_token, now=now)
if (
job.claim_generation != progress.claim_generation
or job.active_claimant_id != claimant_id
):
raise ObservatoryRecordedQueueStaleClaimError("progress owner is stale")
if job.state != "running":
raise ObservatoryRecordedQueueConflictError("progress requires running execution")
row = connection.execute(
"SELECT snapshot_json FROM observatory_recorded_progress WHERE job_id = ?",
(job_id,),
).fetchone()
if row is not None:
previous = RecordedProgress.model_validate_json(row["snapshot_json"])
if previous.claim_generation == progress.claim_generation:
if previous == progress:
return
if not progress.follows(previous):
raise ObservatoryRecordedQueueConflictError("progress snapshot regressed")
connection.execute(
"INSERT INTO observatory_recorded_progress VALUES (?, ?, ?) "
"ON CONFLICT(job_id) DO UPDATE SET snapshot_json = excluded.snapshot_json, "
"received_at_utc = excluded.received_at_utc",
(job_id, payload, now),
)
def progress(self, job_id: str) -> dict[str, object]:
"""Read-only projection; never claims, renews, publishes or computes."""
_validate_pattern(job_id, _JOB_ID, "recorded job id")
with self._read_connection() as connection:
job = self._get_job(connection, job_id)
row = connection.execute(
"SELECT snapshot_json, received_at_utc FROM observatory_recorded_progress "
"WHERE job_id = ?", (job_id,),
).fetchone()
progress = None if row is None else RecordedProgress.model_validate_json(
row["snapshot_json"]
)
if progress is not None and progress.claim_generation != job.claim_generation:
progress = None
return {
"schema_version": "missioncore.observatory-recorded-progress-view/v1",
"job_id": job.job_id,
"source_session_id": job.source_session_id,
"setup_id": job.setup_id,
"definition_sha256": job.definition_sha256,
"claim_generation": job.claim_generation,
"state": job.state,
"received_at_utc": None if progress is None else row["received_at_utc"],
"age_seconds": None if progress is None else max(0.0, (
_parse_timestamp(self._timestamp(), "clock")
- _parse_timestamp(row["received_at_utc"], "progress receipt")
).total_seconds()),
"progress": None if progress is None else progress.model_dump(mode="json"),
}
def authorize_claim_access(
self,
job_id: str,
@@ -2816,6 +2890,7 @@ class ObservatoryRecordedJobQueue:
def _validate_schema(self, connection: sqlite3.Connection) -> None:
expected = {
"observatory_recorded_progress": 3,
"observatory_recorded_jobs": 50,
"observatory_recorded_claim_receipts": 6,
"observatory_recorded_claim_grants_v3": 6,
+166
View File
@@ -0,0 +1,166 @@
"""Bounded, attempt-scoped observation of recorded execution, never authority.
The execution thread owns counters; a separate bounded sender samples them.
Progress loss cannot cancel compute or renew a claim. No frame event journal,
estimated completion, filesystem paths, or model-dependent UI contract.
"""
from __future__ import annotations
import math
import threading
import time
from collections.abc import Callable, Iterator
from contextlib import contextmanager, suppress
from contextvars import ContextVar
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, model_validator
type ProgressPhase = Literal[
"source-transfer",
"source-preparation",
"computing",
"result-assembly",
"result-transfer",
]
type ProgressUnit = Literal["frames", "members", "steps"]
class RecordedProgress(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
schema_version: Literal["missioncore.observatory-recorded-progress/v1"] = (
"missioncore.observatory-recorded-progress/v1"
)
claim_generation: int = Field(ge=1, le=2**53 - 1)
sequence: int = Field(ge=1, le=2**53 - 1)
phase_index: int = Field(ge=0, le=2**53 - 1)
phase: ProgressPhase
unit: ProgressUnit
completed: int = Field(ge=0, le=2**53 - 1)
total: int | None = Field(default=None, ge=1, le=2**53 - 1)
elapsed_seconds: float = Field(ge=0, allow_inf_nan=False)
phase_elapsed_seconds: float = Field(ge=0, allow_inf_nan=False)
@model_validator(mode="after")
def validate_counts(self) -> RecordedProgress:
if self.total is not None and self.completed > self.total:
raise ValueError("completed exceeds the observed total")
if self.phase_elapsed_seconds > self.elapsed_seconds:
raise ValueError("phase duration exceeds execution duration")
return self
def follows(self, previous: RecordedProgress) -> bool:
if self.claim_generation != previous.claim_generation:
return False
if self.sequence <= previous.sequence or self.phase_index < previous.phase_index:
return False
if self.elapsed_seconds < previous.elapsed_seconds:
return False
if self.phase_index != previous.phase_index:
return True
return (
self.phase == previous.phase
and self.unit == previous.unit
and self.completed >= previous.completed
and (previous.total is None or self.total == previous.total)
and self.phase_elapsed_seconds >= previous.phase_elapsed_seconds
)
class RecordedProgressTracker:
def __init__(self, generation: int, *, clock: Callable[[], float] = time.monotonic):
self.generation = generation
self.clock = clock
self.started = clock()
self.phase_started = self.started
self.lock = threading.Lock()
self.phase: ProgressPhase = "source-transfer"
self.unit: ProgressUnit = "members"
self.completed = 0
self.total: int | None = None
self.phase_index = 0
self.sequence = 0
def update(
self,
phase: ProgressPhase,
completed: int = 0,
total: int | None = None,
unit: ProgressUnit = "frames",
) -> None:
with self.lock:
if phase != self.phase or unit != self.unit:
self.phase_index += 1
self.phase_started = self.clock()
elif completed < self.completed:
raise ValueError("progress counter moved backwards within one phase")
elif total is None:
total = self.total
self.phase, self.unit = phase, unit
self.completed, self.total = completed, total
def sample(self) -> RecordedProgress:
with self.lock:
now = self.clock()
self.sequence += 1
return RecordedProgress(
claim_generation=self.generation,
sequence=self.sequence,
phase_index=self.phase_index,
phase=self.phase,
unit=self.unit,
completed=self.completed,
total=self.total,
elapsed_seconds=max(0.0, now - self.started),
phase_elapsed_seconds=max(0.0, now - self.phase_started),
)
_CURRENT: ContextVar[RecordedProgressTracker | None] = ContextVar(
"observatory_recorded_progress",
default=None,
)
def report_recorded_progress(
phase: ProgressPhase,
completed: int = 0,
total: int | None = None,
unit: ProgressUnit = "frames",
) -> None:
tracker = _CURRENT.get()
if tracker is not None:
tracker.update(phase, completed, total, unit)
@contextmanager
def observe_recorded_execution(
generation: int,
send: Callable[[RecordedProgress], None],
*,
interval_seconds: float = 1.0,
) -> Iterator[RecordedProgressTracker]:
if not math.isfinite(interval_seconds) or not 0.01 <= interval_seconds <= 30:
raise ValueError("progress sampling interval is invalid")
tracker = RecordedProgressTracker(generation)
stop = threading.Event()
def pump() -> None:
while not stop.is_set():
# Progress is secondary observation, not lease or result authority.
with suppress(Exception):
send(tracker.sample())
stop.wait(interval_seconds)
token = _CURRENT.set(tracker)
thread = threading.Thread(target=pump, name="observatory-progress", daemon=True)
thread.start()
try:
yield tracker
finally:
_CURRENT.reset(token)
stop.set()
# The production sender reads no body and uses 2-second I/O timeouts.
thread.join(timeout=3.0)
+14 -1
View File
@@ -18,6 +18,7 @@ import json
import re
import threading
from collections.abc import Callable, Mapping
from contextlib import nullcontext
from dataclasses import dataclass
from datetime import datetime
from typing import Annotated, Final, Literal, Protocol
@@ -31,6 +32,7 @@ from k1link.observatory.recorded_jobs import (
OBSERVATORY_RECORDED_JOB_SCHEMA,
RecordedExecutorIdentity,
)
from k1link.observatory.recorded_progress import observe_recorded_execution
WORKER_006_CONTOUR_ID: Final = "worker-006"
MAX_EXECUTOR_FAILURE_MESSAGE_LENGTH: Final = 512
@@ -498,7 +500,18 @@ class ObservatoryWorkerAgent:
heartbeat.start()
try:
result = adapter.execute(active_job)
send_progress = getattr(self._transport, "report_progress", None)
observation = (
observe_recorded_execution(
active_job.claim_generation,
lambda snapshot: send_progress(
job_id=active_job.job_id, claim_token=claim.claim_token,
progress=snapshot,
),
) if callable(send_progress) else nullcontext()
)
with observation:
result = adapter.execute(active_job)
if not isinstance(result, ObservatoryWorkerExecutionResult):
raise TypeError("executor returned an unknown result contract")
except Exception as exc:
@@ -47,6 +47,7 @@ from k1link.observatory.portable_worker_runtime import (
PortableWorkerResultDraft,
PortableWorkerSourceStage,
)
from k1link.observatory.recorded_progress import RecordedProgress, report_recorded_progress
from k1link.observatory.source_admission import (
PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID,
PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE,
@@ -279,6 +280,23 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
},
)
def report_progress(
self, *, job_id: str, claim_token: str, progress: RecordedProgress,
) -> None:
context = self._require_cached_claim(job_id, claim_token)
if progress.claim_generation != context.claim_generation:
raise ObservatoryWorkerHttpError("progress generation changed")
# No response body is needed. Bounded I/O cannot stall the execution thread.
with self._client.stream(
"POST", self._job_path(job_id, "progress"),
json={
"schema_version": "missioncore.observatory-worker-progress-request/v1",
"claim_token": claim_token, "progress": progress.model_dump(mode="json"),
}, timeout=httpx.Timeout(2.0),
) as response:
if response.status_code != 204:
raise ObservatoryWorkerHttpError("progress observation was not accepted")
def succeed(
self,
*,
@@ -336,6 +354,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
headers=headers,
)
members = _source_members(manifest, job)
report_recorded_progress("source-transfer", 0, len(members), "members")
root = _secure_directory(
self._work_root
/ "sources"
@@ -368,10 +387,16 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
members=camera_members,
layout=layout,
)
completed_members = len(camera_members) if camera_epoch_ready else 0
report_recorded_progress("source-transfer", completed_members, len(members), "members")
for destination, member in destinations.items():
if camera_epoch_ready and member.kind in {"camera-init", "camera-segment"}:
continue
if _matches_file(destination, member.sha256, member.byte_length):
completed_members += 1
report_recorded_progress(
"source-transfer", completed_members, len(members), "members",
)
continue
self._download_member(
job_id=job.job_id,
@@ -379,6 +404,8 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
member=member,
destination=destination,
)
completed_members += 1
report_recorded_progress("source-transfer", completed_members, len(members), "members")
manifest_path = root / "materialization-manifest.json"
_write_local_exact(manifest_path, canonical_json(manifest))
return PortableWorkerSourceStage(
@@ -440,7 +467,8 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
or {member.role for member in upload_members} != set(artifacts)
):
raise ObservatoryWorkerHttpError("Worker result artifact roles are not unique")
for member in upload_members:
report_recorded_progress("result-transfer", 0, len(upload_members), "members")
for member_index, member in enumerate(upload_members):
artifact = artifacts.get(member.role)
expected_member_id = hashlib.sha256(
canonical_json(
@@ -461,6 +489,9 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
"result upload plan differs from the local manifest"
)
if member.uploaded:
report_recorded_progress(
"result-transfer", member_index + 1, len(upload_members), "members",
)
continue
relative = relative_artifact_path(artifact.relative_path)
source = _confined_local_member(draft.root, relative.parts)
@@ -489,6 +520,9 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
raise ObservatoryWorkerHttpError(
"result upload acknowledgement did not seal its member"
)
report_recorded_progress(
"result-transfer", member_index + 1, len(upload_members), "members",
)
receipt = self._required_json_request(
"POST",
self._job_path(
+11
View File
@@ -1151,6 +1151,17 @@ def build_observatory_router(
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
}
@router.get("/api/v1/observatory/runs/{job_id}/progress")
def get_observatory_progress(
job_id: str = ApiPath(pattern=r"^observatory-run-[a-f0-9]{32}$"),
) -> dict[str, object]:
try:
return recorded_job_queue.progress(job_id)
except ObservatoryRecordedQueueNotFoundError as exc:
raise HTTPException(status_code=404, detail="Расчёт не найден.") from exc
except (ObservatoryRecordedQueueError, ValueError) as exc:
raise HTTPException(status_code=503, detail="Прогресс недоступен.") from exc
@router.get("/api/v1/observatory/runs/{job_id}")
def get_observatory_recorded_run(
job_id: str = ApiPath(
+18
View File
@@ -51,6 +51,7 @@ from k1link.observatory.recorded_jobs import (
ObservatoryRecordedQueueStaleClaimError,
RecordedExecutorIdentity,
)
from k1link.observatory.recorded_progress import RecordedProgress
OBSERVATORY_WORKER_CAPABILITY_CLAIM_REQUEST_SCHEMA: Final = (
"missioncore.observatory-worker-claim-request/v2"
@@ -194,6 +195,12 @@ class ObservatoryWorkerRenewRequest(_StrictWorkerRequest):
heartbeat_sequence: int = Field(ge=1)
class ObservatoryWorkerProgressRequest(_StrictWorkerRequest):
schema_version: Literal["missioncore.observatory-worker-progress-request/v1"]
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
progress: RecordedProgress
class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest):
schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"]
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
@@ -333,6 +340,17 @@ def build_observatory_worker_router(
)
).as_dict()
@router.post("/recorded-jobs/{job_id}/progress", status_code=204)
def report_job_progress(
request: ObservatoryWorkerProgressRequest,
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
) -> Response:
_queue_call(lambda: queue.report_progress(
job_id, claim_token=request.claim_token,
claimant_id=authentication.contour_id, progress=request.progress,
))
return Response(status_code=204)
@router.post("/recorded-jobs/{job_id}/checkpoint")
def checkpoint_job(
request: ObservatoryWorkerCheckpointRequest,
+339
View File
@@ -0,0 +1,339 @@
from __future__ import annotations
import json
import sqlite3
import sys
import threading
from pathlib import Path
import pytest
from pydantic import ValidationError
from test_observatory_recorded_jobs import _queue, _running_job
from test_observatory_worker_api import WORKER_HEADERS, _claim_request, _services
from k1link.observatory.m49_timing_progress import M49TimingProgress
from k1link.observatory.recorded_jobs import (
ObservatoryRecordedQueueConflictError,
ObservatoryRecordedQueueStaleClaimError,
)
from k1link.observatory.recorded_progress import (
RecordedProgress,
RecordedProgressTracker,
observe_recorded_execution,
report_recorded_progress,
)
def _progress(**changes: object) -> RecordedProgress:
return RecordedProgress.model_validate(
{
"claim_generation": 1,
"sequence": 1,
"phase_index": 1,
"phase": "computing",
"unit": "frames",
"completed": 3,
"total": 10,
"elapsed_seconds": 3.0,
"phase_elapsed_seconds": 2.0,
**changes,
}
)
@pytest.mark.parametrize(
"change",
[
{"completed": True},
{"completed": 11},
{"total": 0},
{"phase": "done"},
{"elapsed_seconds": float("nan")},
{"phase_elapsed_seconds": 4.0},
{"sequence": 2**53},
{"path": "/secret"},
],
)
def test_progress_rejects_unbounded_or_invented_values(change: dict[str, object]) -> None:
with pytest.raises(ValidationError):
_progress(**change)
def test_progress_is_one_durable_snapshot_not_job_identity_or_lease(tmp_path: Path) -> None:
queue = _queue(tmp_path)
job, claim = _running_job(queue)
original = queue.get(job.job_id).as_dict()
assert queue.progress(job.job_id)["progress"] is None
for sequence in range(1, 101):
queue.report_progress(
job.job_id,
claim_token=claim.claim_token,
claimant_id="recorded-worker",
progress=_progress(sequence=sequence),
)
assert queue.get(job.job_id).as_dict() == original
reopened = _queue(tmp_path)
view = reopened.progress(job.job_id)
assert view["definition_sha256"] == job.definition_sha256
assert view["claim_generation"] == 1
assert view["age_seconds"] == 0.0
assert view["progress"]["sequence"] == 100
with sqlite3.connect(queue.database_path) as connection:
assert (
connection.execute("SELECT COUNT(*) FROM observatory_recorded_progress").fetchone()[0]
== 1
)
@pytest.mark.parametrize(
"change",
[
{"sequence": 1, "completed": 4},
{"sequence": 2, "completed": 2},
{"sequence": 2, "total": 12},
{"sequence": 2, "phase": "result-transfer"},
{"sequence": 2, "elapsed_seconds": 2.0},
],
)
def test_progress_retry_is_idempotent_and_regressions_rejected(tmp_path: Path, change) -> None:
queue = _queue(tmp_path)
job, claim = _running_job(queue)
def send(value):
queue.report_progress(
job.job_id, claim_token=claim.claim_token, claimant_id="recorded-worker", progress=value
)
send(_progress())
send(_progress())
with pytest.raises(ObservatoryRecordedQueueConflictError):
send(_progress(**change))
send(_progress(sequence=3, phase_index=2, phase="result-assembly", completed=0, total=None))
def test_progress_old_generation_cannot_write_or_project_as_current(tmp_path: Path) -> None:
queue = _queue(tmp_path)
job, claim = _running_job(queue)
with pytest.raises(ObservatoryRecordedQueueStaleClaimError):
queue.report_progress(
job.job_id,
claim_token=claim.claim_token,
claimant_id="recorded-worker",
progress=_progress(claim_generation=2),
)
with pytest.raises(ObservatoryRecordedQueueStaleClaimError):
queue.report_progress(
job.job_id,
claim_token=claim.claim_token,
claimant_id="another-worker",
progress=_progress(),
)
# Synthetic old attempt retained on disk is not the new attempt's progress.
with sqlite3.connect(queue.database_path) as connection:
connection.execute(
"INSERT INTO observatory_recorded_progress VALUES (?, ?, ?)",
(job.job_id, _progress(claim_generation=2).model_dump_json(), job.created_at_utc),
)
assert queue.progress(job.job_id)["progress"] is None
def test_progress_tracker_preserves_total_and_separates_phases() -> None:
tracker = RecordedProgressTracker(2, clock=lambda: 10.0)
tracker.update("computing", 0, 10)
first = tracker.sample()
tracker.update("computing", 2)
second = tracker.sample()
assert second.total == 10 and second.follows(first)
tracker.update("result-assembly", unit="steps")
third = tracker.sample()
assert third.completed == 0 and third.total is None and third.follows(second)
def test_m49_invoker_preserves_arguments_and_observes_real_output(tmp_path, monkeypatch):
from k1link.observatory import m49_portable_executor as executor
script = tmp_path / "synthetic.py"
script.write_text(
"import os, sys\n"
"from pathlib import Path\n"
"assert sys.argv[1] == 'schedule.tsv'\n"
"assert sys.argv[2] == 'outputs'\n"
"assert os.environ['LANG'] == 'C' and os.environ['TZ'] == 'UTC'\n"
"Path(sys.argv[3]).write_text('timeline_frame_index\\tsource_frame_index\\tother\\n'"
" + '0\\t0\\t0\\t1\\t0\\t12\\t10\\t2\\t1.2\\t1.3\\n')\n"
"print('synthetic runner completed')\n"
)
counters = []
monkeypatch.setattr(
executor, "report_recorded_progress", lambda phase, count: counters.append(count)
)
executor._invoke_exact_runner(
binary=Path(sys.executable),
sequence=script,
schedule=Path("schedule.tsv"),
output=Path("outputs"),
timing=tmp_path / "timing.tsv",
workspace=tmp_path,
timeout_seconds=5,
)
assert counters[-1] == 1
assert (tmp_path / "runner.stdout.log").read_text().strip() == "synthetic runner completed"
assert (tmp_path / "runner.stderr.log").read_bytes() == b""
def test_m49_invoker_timeout_reaps_only_its_child(tmp_path):
import os
from k1link.observatory.m49_portable_executor import (
M49PortableExecutorError,
_invoke_exact_runner,
)
script = tmp_path / "synthetic.py"
script.write_text(
"import os, time\nfrom pathlib import Path\n"
"Path('synthetic.pid').write_text(str(os.getpid()))\ntime.sleep(30)\n"
)
with pytest.raises(M49PortableExecutorError, match="invocation failed"):
_invoke_exact_runner(
binary=Path(sys.executable),
sequence=script,
schedule=tmp_path / "schedule.tsv",
output=tmp_path / "outputs",
timing=tmp_path / "timing.tsv",
workspace=tmp_path,
timeout_seconds=1,
)
pid = int((tmp_path / "synthetic.pid").read_text())
with pytest.raises(ProcessLookupError):
os.kill(pid, 0)
def test_progress_sender_failure_does_not_escape_or_leave_a_thread() -> None:
sampled = threading.Event()
def unavailable(snapshot):
assert isinstance(snapshot, RecordedProgress)
sampled.set()
raise RuntimeError("synthetic observer connection loss")
with observe_recorded_execution(1, unavailable, interval_seconds=0.01) as tracker:
report_recorded_progress("computing", 5, 8)
assert sampled.wait(1)
assert tracker.sample().completed == 5
assert not any(thread.name == "observatory-progress" for thread in threading.enumerate())
report_recorded_progress("computing", 1, 2) # no leaked execution context
assert tracker.sample().completed == 5
def test_worker_keeps_compute_and_lease_when_progress_transport_fails(tmp_path: Path) -> None:
from test_observatory_worker_agent import (
BlockingExecutor,
FakeTransport,
_enqueue,
_heartbeat_agent,
)
from test_observatory_worker_agent import (
_queue as worker_queue,
)
queue = worker_queue(tmp_path)
job_id = _enqueue(queue)
attempted = threading.Event()
entered, release = threading.Event(), threading.Event()
class ObserverFailureTransport(FakeTransport):
def report_progress(self, **_kwargs):
attempted.set()
raise RuntimeError("observer unavailable")
agent = _heartbeat_agent(
ObserverFailureTransport(queue),
BlockingExecutor(entered=entered, release=release),
)
reports = []
thread = threading.Thread(target=lambda: reports.append(agent.run_once()))
thread.start()
assert entered.wait(2) and attempted.wait(2)
release.set()
thread.join(2)
assert not thread.is_alive()
assert reports[0].state == "succeeded"
assert queue.get(job_id).state == "succeeded"
def test_http_progress_uses_active_claim_and_short_bodyless_request(tmp_path: Path) -> None:
import httpx
from test_observatory_worker_http_transport import (
BEARER_TOKEN,
CLAIM_TOKEN,
JOB_ID,
_cache_claim,
_claim_response,
)
from k1link.observatory.worker_http_transport import ObservatoryWorkerHttpGateway
requests = []
def handle(request):
if request.url.path.endswith("/claims"):
return _claim_response()
requests.append(request)
assert request.url.path.endswith("/progress")
assert request.extensions["timeout"]["read"] == 2.0
assert json.loads(request.content)["claim_token"] == CLAIM_TOKEN
return httpx.Response(204)
with ObservatoryWorkerHttpGateway(
base_url="http://127.0.0.1:8000",
bearer_token=BEARER_TOKEN,
work_root=tmp_path,
transport=httpx.MockTransport(handle),
) as gateway:
_cache_claim(gateway)
gateway.report_progress(job_id=JOB_ID, claim_token=CLAIM_TOKEN, progress=_progress())
assert len(requests) == 1
def test_tgs_progress_counts_only_complete_ordered_flushed_rows(tmp_path: Path) -> None:
path = tmp_path / "timing.tsv"
observer = M49TimingProgress(path)
assert observer.poll() == 0
path.write_bytes(
b"timeline_frame_index\tsource_frame_index\trest\n0\t0\t0\t1\t0\t1\t1\t0\t1\t1\n1\t1"
)
assert observer.poll() == 1
with path.open("ab") as stream:
stream.write(b"\t1\t1\t1\t1\t1\t0\t1\t1\n")
assert observer.poll() == 2
with path.open("ab") as stream:
stream.write(b"4\t4\t4\t1\t4\t1\t1\t0\t1\t1\n")
assert observer.poll() == 2 and observer.invalid
def test_progress_api_requires_auth_exact_claim_and_strict_payload(tmp_path: Path) -> None:
from test_observatory_worker_api import _enqueue
client, queue = _services(tmp_path)
job_id = _enqueue(queue)
response = client.post(
"/api/v1/worker/observatory/recorded-jobs/claims",
headers=WORKER_HEADERS,
json=_claim_request("progress-claim"),
)
claim = response.json()
token = claim["claim_token"]
queue.start(job_id, claim_token=token)
path = f"/api/v1/worker/observatory/recorded-jobs/{job_id}/progress"
body = {
"schema_version": "missioncore.observatory-worker-progress-request/v1",
"claim_token": token,
"progress": _progress().model_dump(mode="json"),
}
assert client.post(path, json=body).status_code == 401
assert (
client.post(path, headers=WORKER_HEADERS, json={**body, "command": "no"}).status_code == 422
)
assert client.post(path, headers=WORKER_HEADERS, json=body).status_code == 204
assert queue.progress(job_id)["progress"] == json.loads(_progress().model_dump_json())