feat(observatory): expose attempt-bound recorded progress
This commit is contained in:
@@ -26,6 +26,7 @@ export interface ObservatoryRecordedJob {
|
||||
readonly sourceSessionId: string;
|
||||
readonly setupId: string;
|
||||
readonly definitionSha256: string;
|
||||
readonly claimGeneration: number;
|
||||
readonly state: ObservatoryRecordedJobState;
|
||||
readonly restartFromZero: boolean;
|
||||
readonly resultId: string | null;
|
||||
@@ -229,6 +230,7 @@ function decodeJob(value: unknown): ObservatoryRecordedJob {
|
||||
sourceSessionId: text(source.session_id, "source.session_id"),
|
||||
setupId: text(setup.setup_id, "setup.setup_id"),
|
||||
definitionSha256: String(setup.definition_sha256),
|
||||
claimGeneration: nonNegativeInteger(row.claim_generation, "claim_generation"),
|
||||
state,
|
||||
restartFromZero: boolean(row.restart_from_zero, "restart_from_zero"),
|
||||
resultId: result === null ? null : text(result.result_id, "result.result_id"),
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import type { ObservatoryRecordedJob } from "./recordedJobs";
|
||||
|
||||
const PHASES = {
|
||||
"source-transfer": "Передаём входные данные",
|
||||
"source-preparation": "Подготавливаем вход",
|
||||
computing: "Обрабатываем запись",
|
||||
"result-assembly": "Собираем результат",
|
||||
"result-transfer": "Передаём результат",
|
||||
} as const;
|
||||
const UNITS = { frames: "кадров", members: "файлов", steps: "операций" } as const;
|
||||
type Phase = keyof typeof PHASES;
|
||||
type Unit = keyof typeof UNITS;
|
||||
|
||||
export interface RecordedProgressView {
|
||||
readonly jobId: string;
|
||||
readonly claimGeneration: number;
|
||||
readonly sequence: number;
|
||||
readonly phase: Phase;
|
||||
readonly unit: Unit;
|
||||
readonly completed: number;
|
||||
readonly total: number | null;
|
||||
readonly ageSeconds: number;
|
||||
}
|
||||
|
||||
export async function fetchRecordedProgress(
|
||||
job: ObservatoryRecordedJob,
|
||||
{ signal, fetcher = globalThis.fetch }: {
|
||||
signal?: AbortSignal;
|
||||
fetcher?: typeof globalThis.fetch;
|
||||
} = {},
|
||||
): Promise<RecordedProgressView | null> {
|
||||
const request = new AbortController();
|
||||
const abort = () => request.abort();
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
if (signal?.aborted) request.abort();
|
||||
const timer = globalThis.setTimeout(abort, 2_500);
|
||||
try {
|
||||
const response = await fetcher(
|
||||
`/api/v1/observatory/runs/${encodeURIComponent(job.jobId)}/progress`,
|
||||
{ signal: request.signal, headers: { Accept: "application/json" } },
|
||||
);
|
||||
if (!response.ok) throw new Error("Прогресс расчёта недоступен.");
|
||||
return decodeRecordedProgress(await response.json(), job);
|
||||
} finally {
|
||||
globalThis.clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", abort);
|
||||
}
|
||||
}
|
||||
|
||||
export function decodeRecordedProgress(
|
||||
value: unknown, job: ObservatoryRecordedJob,
|
||||
): RecordedProgressView | null {
|
||||
const row = object(value);
|
||||
keys(row, ["schema_version", "job_id", "source_session_id", "setup_id",
|
||||
"definition_sha256", "claim_generation", "state", "received_at_utc", "age_seconds", "progress"]);
|
||||
if (row.schema_version !== "missioncore.observatory-recorded-progress-view/v1"
|
||||
|| row.job_id !== job.jobId || row.source_session_id !== job.sourceSessionId
|
||||
|| row.setup_id !== job.setupId || row.definition_sha256 !== job.definitionSha256
|
||||
|| row.claim_generation !== job.claimGeneration) throw new Error("Прогресс другой попытки.");
|
||||
if (row.progress === null || row.state !== job.state) return null;
|
||||
const progress = object(row.progress);
|
||||
keys(progress, ["schema_version", "claim_generation", "sequence", "phase_index",
|
||||
"phase", "unit", "completed", "total", "elapsed_seconds", "phase_elapsed_seconds"]);
|
||||
if (progress.schema_version !== "missioncore.observatory-recorded-progress/v1"
|
||||
|| progress.claim_generation !== row.claim_generation
|
||||
|| !(typeof progress.phase === "string" && Object.hasOwn(PHASES, progress.phase))
|
||||
|| !(typeof progress.unit === "string" && Object.hasOwn(UNITS, progress.unit))
|
||||
|| typeof row.received_at_utc !== "string" || !Number.isFinite(Date.parse(row.received_at_utc))) {
|
||||
throw new Error("Некорректный прогресс.");
|
||||
}
|
||||
const completed = integer(progress.completed);
|
||||
const total = progress.total === null ? null : integer(progress.total, 1);
|
||||
const elapsed = finite(progress.elapsed_seconds);
|
||||
if ((total !== null && completed > total) || finite(progress.phase_elapsed_seconds) > elapsed) {
|
||||
throw new Error("Некорректные счётчики прогресса.");
|
||||
}
|
||||
integer(progress.phase_index);
|
||||
return {
|
||||
jobId: job.jobId, claimGeneration: integer(progress.claim_generation, 1),
|
||||
sequence: integer(progress.sequence, 1), phase: progress.phase as Phase,
|
||||
unit: progress.unit as Unit, completed, total, ageSeconds: finite(row.age_seconds),
|
||||
};
|
||||
}
|
||||
|
||||
export function recordedProgressLabel(
|
||||
job: ObservatoryRecordedJob | null, progress: RecordedProgressView | null,
|
||||
): string {
|
||||
if (job?.publication.state === "pending") return "Сохраняем результат";
|
||||
if (!job || job.state === "accepted" || job.state === "queued") return "Ожидаем расчёт";
|
||||
if (job.state === "paused" || job.state === "preemption-pending") return "Расчёт приостановлен";
|
||||
if (job.state === "reconciliation-required") return "Проверяем состояние расчёта";
|
||||
if (!progress || progress.jobId !== job.jobId
|
||||
|| progress.claimGeneration !== job.claimGeneration) return "Ожидаем данные расчёта";
|
||||
if (progress.ageSeconds > 15) return "Ожидаем обновление прогресса";
|
||||
const count = progress.total === null
|
||||
? (progress.completed > 0 ? ` · ${progress.completed.toLocaleString("ru-RU")} ${UNITS[progress.unit]}` : "")
|
||||
: ` · ${progress.completed.toLocaleString("ru-RU")} / ${progress.total.toLocaleString("ru-RU")} ${UNITS[progress.unit]}`;
|
||||
return PHASES[progress.phase] + count;
|
||||
}
|
||||
|
||||
function object(value: unknown): Record<string, unknown> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Прогресс: ожидался объект.");
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
function keys(row: Record<string, unknown>, expected: string[]): void {
|
||||
if (Object.keys(row).length !== expected.length || expected.some((key) => !Object.hasOwn(row, key))) {
|
||||
throw new Error("Прогресс: неизвестные поля.");
|
||||
}
|
||||
}
|
||||
function finite(value: unknown): number {
|
||||
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new Error("Прогресс: некорректное число.");
|
||||
return value;
|
||||
}
|
||||
function integer(value: unknown, minimum = 0): number {
|
||||
const number = finite(value);
|
||||
if (!Number.isSafeInteger(number) || number < minimum) throw new Error("Прогресс: некорректный счётчик.");
|
||||
return number;
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { fetchRecordedProgress, type RecordedProgressView } from "./recordedProgress";
|
||||
|
||||
import {
|
||||
fetchObservatoryRecordedJobs,
|
||||
@@ -21,6 +22,7 @@ interface JobSnapshot {
|
||||
readonly jobs: readonly ObservatoryRecordedJob[];
|
||||
readonly state: RecordedJobsState;
|
||||
readonly error: string | null;
|
||||
readonly progress?: RecordedProgressView | null;
|
||||
}
|
||||
const EMPTY_JOBS = [] as const;
|
||||
const OPEN_STATES = new Set([
|
||||
@@ -63,14 +65,19 @@ export function useObservatoryRecordedJobs(
|
||||
setSnapshot({
|
||||
selectionKey, jobs,
|
||||
state: jobs.length > 0 ? "refreshing" : "loading", error: null,
|
||||
progress: current?.progress ?? null,
|
||||
});
|
||||
void fetchObservatoryRecordedJobs(sourceSessionId, setupId, {
|
||||
definitionSha256, signal: request.signal,
|
||||
}).then((next) => {
|
||||
}).then(async (next) => {
|
||||
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
||||
const pending = next.find((job) => OPEN_STATES.has(job.state));
|
||||
const progress = pending
|
||||
? await fetchRecordedProgress(pending, { signal: request.signal }).catch(() => null)
|
||||
: null;
|
||||
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
||||
if (pending) observedJobId.current = pending.jobId;
|
||||
setSnapshot({ selectionKey, jobs: next, state: "ready", error: null });
|
||||
setSnapshot({ selectionKey, jobs: next, state: "ready", error: null, progress });
|
||||
}).catch((caught: unknown) => {
|
||||
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
||||
setSnapshot({
|
||||
@@ -95,7 +102,7 @@ export function useObservatoryRecordedJobs(
|
||||
&& latestJob.jobId === observedJobId.current;
|
||||
|
||||
useEffect(() => {
|
||||
if (state !== "ready" || (!activeJob && !publicationPending)) return;
|
||||
if ((state !== "ready" && state !== "error") || (!activeJob && !publicationPending)) return;
|
||||
const timer = globalThis.setTimeout(
|
||||
() => setRevision((value) => value + 1),
|
||||
POLL_INTERVAL_MS,
|
||||
@@ -185,6 +192,7 @@ export function useObservatoryRecordedJobs(
|
||||
|
||||
return {
|
||||
jobs, latestJob, activeJob, publicationPending, computationFailed,
|
||||
progress: current?.progress ?? null,
|
||||
state, error, refresh, submit, retryPublication,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
import { useObservatoryCatalog } from "../../core/observatory/useObservatoryCatalog";
|
||||
import { useObservatoryLaboratorySetups } from "../../core/observatory/useObservatoryLaboratorySetups";
|
||||
import { useObservatoryRecordedJobs } from "../../core/observatory/useObservatoryRecordedJobs";
|
||||
import { recordedProgressLabel } from "../../core/observatory/recordedProgress";
|
||||
import type { WorkspaceDefinition } from "../../productModel";
|
||||
|
||||
const EMPTY_OBSERVATORY_ITEMS = [] as const;
|
||||
@@ -466,7 +467,12 @@ export function ObservatoryWorkspace({
|
||||
</Button>
|
||||
<div className="observatory-catalog-bar__run" aria-busy={calculationPending}>
|
||||
{calculationPending ? (
|
||||
<ActivityIndicator size="compact" label="Ожидание результата расчёта" />
|
||||
<>
|
||||
<ActivityIndicator size="compact" />
|
||||
<span role="status">
|
||||
{recordedProgressLabel(presentedJob, recordedJobsController.progress)}
|
||||
</span>
|
||||
</>
|
||||
) : null}
|
||||
{showCalculate ? (
|
||||
<Button
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { after, before, test } from "node:test";
|
||||
import { createServer } from "vite";
|
||||
|
||||
let server;
|
||||
let decode;
|
||||
let label;
|
||||
let fetchProgress;
|
||||
const job = {
|
||||
jobId: "observatory-run-" + "1".repeat(32),
|
||||
sourceSessionId: "source-a", setupId: "m49-tgs",
|
||||
definitionSha256: "a".repeat(64), claimGeneration: 2, state: "running",
|
||||
publication: { state: "not-required" },
|
||||
};
|
||||
function view() {
|
||||
return {
|
||||
schema_version: "missioncore.observatory-recorded-progress-view/v1",
|
||||
job_id: job.jobId, source_session_id: job.sourceSessionId, setup_id: job.setupId,
|
||||
definition_sha256: job.definitionSha256, claim_generation: 2, state: "running",
|
||||
received_at_utc: "2026-09-03T12:00:00Z", age_seconds: 0.5,
|
||||
progress: {
|
||||
schema_version: "missioncore.observatory-recorded-progress/v1",
|
||||
claim_generation: 2, sequence: 4, phase_index: 2,
|
||||
phase: "computing", unit: "frames", completed: 3, total: 10,
|
||||
elapsed_seconds: 8, phase_elapsed_seconds: 5,
|
||||
},
|
||||
};
|
||||
}
|
||||
before(async () => {
|
||||
server = await createServer({ configFile: false, logLevel: "silent", server: { middlewareMode: true } });
|
||||
const module = await server.ssrLoadModule("/src/core/observatory/recordedProgress.ts");
|
||||
decode = module.decodeRecordedProgress;
|
||||
label = module.recordedProgressLabel;
|
||||
fetchProgress = module.fetchRecordedProgress;
|
||||
});
|
||||
after(async () => { await server?.close(); });
|
||||
|
||||
test("actual frame count is separate from completion and publication", () => {
|
||||
const progress = decode(view(), job);
|
||||
assert.equal(label(job, progress), "Обрабатываем запись · 3 / 10 кадров");
|
||||
assert.equal(label({ ...job, publication: { state: "pending" } }, progress), "Сохраняем результат");
|
||||
assert.doesNotMatch(label(job, progress), /%|завершён|FPS|ETA/);
|
||||
});
|
||||
test("unknown total never becomes a synthetic percentage", () => {
|
||||
const value = view();
|
||||
value.progress.total = null;
|
||||
assert.equal(label(job, decode(value, job)), "Обрабатываем запись · 3 кадров");
|
||||
value.progress.completed = 0;
|
||||
assert.equal(label(job, decode(value, job)), "Обрабатываем запись");
|
||||
});
|
||||
test("old source definition or attempt cannot flash as current progress", () => {
|
||||
for (const delta of [
|
||||
{ job_id: "other" }, { source_session_id: "other" }, { setup_id: "other" },
|
||||
{ definition_sha256: "b".repeat(64) }, { claim_generation: 1 },
|
||||
]) assert.throws(() => decode({ ...view(), ...delta }, job));
|
||||
assert.equal(decode({ ...view(), state: "succeeded" }, job), null);
|
||||
assert.equal(label({ ...job, claimGeneration: 3 }, decode(view(), job)), "Ожидаем данные расчёта");
|
||||
});
|
||||
test("invalid counters, invented phase and extra fields fail closed", () => {
|
||||
for (const delta of [
|
||||
{ completed: true }, { completed: 11 }, { completed: -1 }, { total: 0 },
|
||||
{ completed: Number.MAX_SAFE_INTEGER + 1 }, { phase: "done" }, { path: "/private" },
|
||||
{ phase_elapsed_seconds: 9 }, { claim_generation: 3 },
|
||||
]) assert.throws(() => decode({ ...view(), progress: { ...view().progress, ...delta } }, job));
|
||||
});
|
||||
test("missing and stale telemetry show no fabricated advancement", () => {
|
||||
assert.equal(decode({ ...view(), progress: null, age_seconds: null, received_at_utc: null }, job), null);
|
||||
assert.equal(label(job, null), "Ожидаем данные расчёта");
|
||||
assert.equal(label(job, decode({ ...view(), age_seconds: 30 }, job)), "Ожидаем обновление прогресса");
|
||||
});
|
||||
test("progress is a read-only request bound to the active job", async () => {
|
||||
const calls = [];
|
||||
const progress = await fetchProgress(job, { fetcher: async (path, init) => {
|
||||
calls.push([path, init.method ?? "GET"]);
|
||||
return new Response(JSON.stringify(view()));
|
||||
} });
|
||||
assert.deepEqual(calls, [[`/api/v1/observatory/runs/${job.jobId}/progress`, "GET"]]);
|
||||
assert.equal(progress.completed, 3);
|
||||
});
|
||||
@@ -47,6 +47,7 @@ from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerSourceStage,
|
||||
inspect_runtime_candidate,
|
||||
)
|
||||
from k1link.observatory.recorded_progress import report_recorded_progress
|
||||
from k1link.observatory.worker_agent import ObservatoryWorkerExecutorRegistration
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -467,7 +468,9 @@ class InstalledLabPackageProfileRunner:
|
||||
}
|
||||
),
|
||||
)
|
||||
for container in _topological_containers(self.package):
|
||||
containers = _topological_containers(self.package)
|
||||
report_recorded_progress("computing", 0, len(containers), "steps")
|
||||
for step_index, container in enumerate(containers):
|
||||
container_output_root = output_root
|
||||
if container.role == "step":
|
||||
container_output_root = steps_root / container.container_id
|
||||
@@ -483,6 +486,8 @@ class InstalledLabPackageProfileRunner:
|
||||
name_token=attempt_token,
|
||||
)
|
||||
)
|
||||
report_recorded_progress("computing", step_index + 1, len(containers), "steps")
|
||||
report_recorded_progress("result-assembly", unit="steps")
|
||||
return _read_result_draft(
|
||||
output_root,
|
||||
plan=plan,
|
||||
|
||||
@@ -19,7 +19,9 @@ import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, Protocol, cast
|
||||
@@ -36,6 +38,7 @@ from k1link.observatory.m49_portable_source import (
|
||||
materialize_m49_portable_source_from_worker_stage,
|
||||
validate_m49_portable_source_stage_binding,
|
||||
)
|
||||
from k1link.observatory.m49_timing_progress import M49TimingProgress
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
OBSERVATION_ONLY_AUTHORITY,
|
||||
canonical_json,
|
||||
@@ -53,6 +56,7 @@ from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerSourceMaterializer,
|
||||
PortableWorkerSourceStage,
|
||||
)
|
||||
from k1link.observatory.recorded_progress import report_recorded_progress
|
||||
from k1link.observatory.worker_agent import SealedObservatoryRecordedJob
|
||||
|
||||
M49_COMPILED_RUNNER_BUILD_SCHEMA: Final = "missioncore.m49-tgs-portable-compiled-runner-build/v1"
|
||||
@@ -257,6 +261,7 @@ class M49PortableProfileRunnerAdapter:
|
||||
try:
|
||||
output = workspace / "outputs"
|
||||
timing = workspace / "timing.tsv"
|
||||
report_recorded_progress("computing", 0, stage.timeline_frame_count)
|
||||
invoker = self.invoker or _invoke_exact_runner
|
||||
invoker(
|
||||
binary=self.installation.runner_binary_path,
|
||||
@@ -267,6 +272,7 @@ class M49PortableProfileRunnerAdapter:
|
||||
workspace=workspace,
|
||||
timeout_seconds=self.installation.timeout_seconds,
|
||||
)
|
||||
report_recorded_progress("result-assembly", unit="steps")
|
||||
package = assemble_m49_portable_result(
|
||||
source_stage=stage,
|
||||
runner_output_root=output,
|
||||
@@ -335,19 +341,32 @@ def _invoke_exact_runner(
|
||||
stderr = workspace / "runner.stderr.log"
|
||||
try:
|
||||
with stdout.open("xb") as stdout_stream, stderr.open("xb") as stderr_stream:
|
||||
completed = subprocess.run(
|
||||
started = time.monotonic()
|
||||
progress = M49TimingProgress(timing)
|
||||
process = subprocess.Popen(
|
||||
[str(binary), str(sequence), str(schedule), str(output), str(timing)],
|
||||
cwd=workspace,
|
||||
env={"LANG": "C", "LC_ALL": "C", "TZ": "UTC"},
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=stdout_stream,
|
||||
stderr=stderr_stream,
|
||||
check=False,
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
try:
|
||||
while process.poll() is None:
|
||||
report_recorded_progress("computing", progress.poll())
|
||||
remaining = timeout_seconds - (time.monotonic() - started)
|
||||
if remaining <= 0:
|
||||
raise subprocess.TimeoutExpired(str(binary), timeout_seconds)
|
||||
with suppress(subprocess.TimeoutExpired):
|
||||
process.wait(timeout=min(0.5, remaining))
|
||||
report_recorded_progress("computing", progress.poll())
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait()
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
raise M49PortableExecutorError("portable M4.9 runner invocation failed") from exc
|
||||
if completed.returncode != 0:
|
||||
if process.returncode != 0:
|
||||
raise M49PortableExecutorError("portable M4.9 runner rejected its exact source stage")
|
||||
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ import numpy.typing as npt
|
||||
|
||||
from k1link.compute.lidar_replay import LidarReplayPackV2, build_lidar_replay_pack_v2
|
||||
from k1link.observatory.portable_result_contract import canonical_json
|
||||
from k1link.observatory.recorded_progress import report_recorded_progress
|
||||
from k1link.observatory.source_admission import (
|
||||
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||
@@ -175,6 +176,7 @@ def materialize_m49_portable_source_from_worker_stage(
|
||||
or worker_stage.source_adapter_sha256 != job.source_adapter_sha256
|
||||
):
|
||||
raise M49PortableSourceError("Worker source stage belongs to another job")
|
||||
report_recorded_progress("source-preparation")
|
||||
root = _safe_directory(worker_stage.root, "Worker source stage")
|
||||
manifest_payload, manifest = _read_canonical_document(
|
||||
root / "materialization-manifest.json",
|
||||
@@ -487,6 +489,7 @@ def _materialize_stage(
|
||||
index_rows: list[dict[str, object]] = []
|
||||
available_slot = 0
|
||||
sequence_logical = hashlib.sha256()
|
||||
report_recorded_progress("source-preparation", 0, len(anchors))
|
||||
for anchor in anchors:
|
||||
point_index = int(
|
||||
np.searchsorted(point_times, anchor.session_seconds, side="right") - 1
|
||||
@@ -534,6 +537,9 @@ def _materialize_stage(
|
||||
f"{anchor.timeline_frame_index}\t{anchor.source_frame_index}"
|
||||
f"\t{anchor.session_seconds:.9f}\t-1\t0"
|
||||
)
|
||||
report_recorded_progress(
|
||||
"source-preparation", anchor.timeline_frame_index + 1, len(anchors),
|
||||
)
|
||||
continue
|
||||
|
||||
start_seconds = anchor.session_seconds - profile.history_seconds
|
||||
@@ -593,6 +599,9 @@ def _materialize_stage(
|
||||
f"\t{anchor.session_seconds:.9f}\t{available_slot}\t{native.shape[0]}"
|
||||
)
|
||||
available_slot += 1
|
||||
report_recorded_progress(
|
||||
"source-preparation", anchor.timeline_frame_index + 1, len(anchors),
|
||||
)
|
||||
|
||||
if available_slot < 1:
|
||||
raise M49PortableSourceError("portable K1 source has no admissible LiDAR frames")
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Incremental observation of flushed TGS timing rows, not result validation."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class M49TimingProgress:
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
self.offset = 0
|
||||
self.pending = b""
|
||||
self.header = False
|
||||
self.completed = 0
|
||||
self.invalid = False
|
||||
|
||||
def poll(self) -> int:
|
||||
if self.invalid:
|
||||
return self.completed
|
||||
try:
|
||||
with self.path.open("rb") as stream:
|
||||
stream.seek(self.offset)
|
||||
data = stream.read(64 * 1024)
|
||||
self.offset += len(data)
|
||||
except OSError:
|
||||
return self.completed
|
||||
lines = (self.pending + data).split(b"\n")
|
||||
self.pending = lines.pop()
|
||||
if len(self.pending) > 2048:
|
||||
self.invalid = True
|
||||
self.pending = b""
|
||||
return self.completed
|
||||
for line in lines:
|
||||
if not self.header:
|
||||
self.header = line.startswith(b"timeline_frame_index\tsource_frame_index\t")
|
||||
if not self.header:
|
||||
self.invalid = True
|
||||
break
|
||||
continue
|
||||
fields = line.split(b"\t")
|
||||
if len(fields) != 10 or fields[0] != str(self.completed).encode():
|
||||
self.invalid = True
|
||||
break
|
||||
self.completed += 1
|
||||
return self.completed
|
||||
@@ -27,6 +27,7 @@ from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinitionRegistry,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.recorded_progress import report_recorded_progress
|
||||
from k1link.observatory.worker_agent import (
|
||||
ObservatoryWorkerExecutionResult,
|
||||
ObservatoryWorkerExecutorIdentity,
|
||||
@@ -732,6 +733,7 @@ class PortableWorkerExecutorAdapter:
|
||||
job: SealedObservatoryRecordedJob,
|
||||
) -> ObservatoryWorkerExecutionResult:
|
||||
self._verify_job(job)
|
||||
report_recorded_progress("source-transfer", unit="members")
|
||||
source = self.source_materializer.materialize(job)
|
||||
if (
|
||||
source.source_bundle_sha256 != job.source_bundle_sha256
|
||||
@@ -764,11 +766,13 @@ class PortableWorkerExecutorAdapter:
|
||||
result_contract_sha256=self.candidate.result_contract_sha256,
|
||||
phases=tuple(phase.phase_id for phase in self.candidate.phases),
|
||||
)
|
||||
report_recorded_progress("computing", unit="steps")
|
||||
draft = self.runner.run(plan, source)
|
||||
if draft.result_contract_sha256 != self.candidate.result_contract_sha256:
|
||||
raise PortableWorkerRuntimeJobRejectedError(
|
||||
"runtime result uses another result contract"
|
||||
)
|
||||
report_recorded_progress("result-transfer", unit="members")
|
||||
published = self.publisher.publish(job, draft)
|
||||
if (
|
||||
published.result_id != draft.result_id
|
||||
|
||||
@@ -32,6 +32,7 @@ from typing import Final, Literal
|
||||
from uuid import uuid4
|
||||
|
||||
from k1link.artifacts import utc_now_iso
|
||||
from k1link.observatory.recorded_progress import RecordedProgress
|
||||
|
||||
OBSERVATORY_RECORDED_JOB_SCHEMA: Final = "missioncore.observatory-recorded-job/v1"
|
||||
OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA: Final = "missioncore.observatory-recorded-job-request/v1"
|
||||
@@ -159,6 +160,12 @@ CREATE TABLE IF NOT EXISTS observatory_recorded_jobs (
|
||||
CREATE INDEX IF NOT EXISTS observatory_recorded_jobs_queue_order
|
||||
ON observatory_recorded_jobs (state, priority_rank, created_at_utc, job_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS observatory_recorded_progress (
|
||||
job_id TEXT PRIMARY KEY REFERENCES observatory_recorded_jobs(job_id),
|
||||
snapshot_json TEXT NOT NULL CHECK (length(snapshot_json) <= 2048),
|
||||
received_at_utc TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS observatory_recorded_claim_receipts (
|
||||
claim_request_id TEXT PRIMARY KEY,
|
||||
request_sha256 TEXT NOT NULL,
|
||||
@@ -1614,6 +1621,73 @@ class ObservatoryRecordedJobQueue:
|
||||
)
|
||||
return self._get_job(connection, job_id)
|
||||
|
||||
def report_progress(
|
||||
self, job_id: str, *, claim_token: str, claimant_id: str,
|
||||
progress: RecordedProgress,
|
||||
) -> None:
|
||||
"""Replace one small observation, fenced in the ownership transaction."""
|
||||
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||
_validate_pattern(claim_token, _TOKEN, "claim token")
|
||||
payload = progress.model_dump_json()
|
||||
with self._transaction() as connection:
|
||||
job = self._get_job(connection, job_id)
|
||||
now = self._timestamp()
|
||||
self._require_active_claim(job, claim_token, now=now)
|
||||
if (
|
||||
job.claim_generation != progress.claim_generation
|
||||
or job.active_claimant_id != claimant_id
|
||||
):
|
||||
raise ObservatoryRecordedQueueStaleClaimError("progress owner is stale")
|
||||
if job.state != "running":
|
||||
raise ObservatoryRecordedQueueConflictError("progress requires running execution")
|
||||
row = connection.execute(
|
||||
"SELECT snapshot_json FROM observatory_recorded_progress WHERE job_id = ?",
|
||||
(job_id,),
|
||||
).fetchone()
|
||||
if row is not None:
|
||||
previous = RecordedProgress.model_validate_json(row["snapshot_json"])
|
||||
if previous.claim_generation == progress.claim_generation:
|
||||
if previous == progress:
|
||||
return
|
||||
if not progress.follows(previous):
|
||||
raise ObservatoryRecordedQueueConflictError("progress snapshot regressed")
|
||||
connection.execute(
|
||||
"INSERT INTO observatory_recorded_progress VALUES (?, ?, ?) "
|
||||
"ON CONFLICT(job_id) DO UPDATE SET snapshot_json = excluded.snapshot_json, "
|
||||
"received_at_utc = excluded.received_at_utc",
|
||||
(job_id, payload, now),
|
||||
)
|
||||
|
||||
def progress(self, job_id: str) -> dict[str, object]:
|
||||
"""Read-only projection; never claims, renews, publishes or computes."""
|
||||
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||
with self._read_connection() as connection:
|
||||
job = self._get_job(connection, job_id)
|
||||
row = connection.execute(
|
||||
"SELECT snapshot_json, received_at_utc FROM observatory_recorded_progress "
|
||||
"WHERE job_id = ?", (job_id,),
|
||||
).fetchone()
|
||||
progress = None if row is None else RecordedProgress.model_validate_json(
|
||||
row["snapshot_json"]
|
||||
)
|
||||
if progress is not None and progress.claim_generation != job.claim_generation:
|
||||
progress = None
|
||||
return {
|
||||
"schema_version": "missioncore.observatory-recorded-progress-view/v1",
|
||||
"job_id": job.job_id,
|
||||
"source_session_id": job.source_session_id,
|
||||
"setup_id": job.setup_id,
|
||||
"definition_sha256": job.definition_sha256,
|
||||
"claim_generation": job.claim_generation,
|
||||
"state": job.state,
|
||||
"received_at_utc": None if progress is None else row["received_at_utc"],
|
||||
"age_seconds": None if progress is None else max(0.0, (
|
||||
_parse_timestamp(self._timestamp(), "clock")
|
||||
- _parse_timestamp(row["received_at_utc"], "progress receipt")
|
||||
).total_seconds()),
|
||||
"progress": None if progress is None else progress.model_dump(mode="json"),
|
||||
}
|
||||
|
||||
def authorize_claim_access(
|
||||
self,
|
||||
job_id: str,
|
||||
@@ -2816,6 +2890,7 @@ class ObservatoryRecordedJobQueue:
|
||||
|
||||
def _validate_schema(self, connection: sqlite3.Connection) -> None:
|
||||
expected = {
|
||||
"observatory_recorded_progress": 3,
|
||||
"observatory_recorded_jobs": 50,
|
||||
"observatory_recorded_claim_receipts": 6,
|
||||
"observatory_recorded_claim_grants_v3": 6,
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Bounded, attempt-scoped observation of recorded execution, never authority.
|
||||
|
||||
The execution thread owns counters; a separate bounded sender samples them.
|
||||
Progress loss cannot cancel compute or renew a claim. No frame event journal,
|
||||
estimated completion, filesystem paths, or model-dependent UI contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager, suppress
|
||||
from contextvars import ContextVar
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
type ProgressPhase = Literal[
|
||||
"source-transfer",
|
||||
"source-preparation",
|
||||
"computing",
|
||||
"result-assembly",
|
||||
"result-transfer",
|
||||
]
|
||||
type ProgressUnit = Literal["frames", "members", "steps"]
|
||||
|
||||
|
||||
class RecordedProgress(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
|
||||
|
||||
schema_version: Literal["missioncore.observatory-recorded-progress/v1"] = (
|
||||
"missioncore.observatory-recorded-progress/v1"
|
||||
)
|
||||
claim_generation: int = Field(ge=1, le=2**53 - 1)
|
||||
sequence: int = Field(ge=1, le=2**53 - 1)
|
||||
phase_index: int = Field(ge=0, le=2**53 - 1)
|
||||
phase: ProgressPhase
|
||||
unit: ProgressUnit
|
||||
completed: int = Field(ge=0, le=2**53 - 1)
|
||||
total: int | None = Field(default=None, ge=1, le=2**53 - 1)
|
||||
elapsed_seconds: float = Field(ge=0, allow_inf_nan=False)
|
||||
phase_elapsed_seconds: float = Field(ge=0, allow_inf_nan=False)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_counts(self) -> RecordedProgress:
|
||||
if self.total is not None and self.completed > self.total:
|
||||
raise ValueError("completed exceeds the observed total")
|
||||
if self.phase_elapsed_seconds > self.elapsed_seconds:
|
||||
raise ValueError("phase duration exceeds execution duration")
|
||||
return self
|
||||
|
||||
def follows(self, previous: RecordedProgress) -> bool:
|
||||
if self.claim_generation != previous.claim_generation:
|
||||
return False
|
||||
if self.sequence <= previous.sequence or self.phase_index < previous.phase_index:
|
||||
return False
|
||||
if self.elapsed_seconds < previous.elapsed_seconds:
|
||||
return False
|
||||
if self.phase_index != previous.phase_index:
|
||||
return True
|
||||
return (
|
||||
self.phase == previous.phase
|
||||
and self.unit == previous.unit
|
||||
and self.completed >= previous.completed
|
||||
and (previous.total is None or self.total == previous.total)
|
||||
and self.phase_elapsed_seconds >= previous.phase_elapsed_seconds
|
||||
)
|
||||
|
||||
|
||||
class RecordedProgressTracker:
|
||||
def __init__(self, generation: int, *, clock: Callable[[], float] = time.monotonic):
|
||||
self.generation = generation
|
||||
self.clock = clock
|
||||
self.started = clock()
|
||||
self.phase_started = self.started
|
||||
self.lock = threading.Lock()
|
||||
self.phase: ProgressPhase = "source-transfer"
|
||||
self.unit: ProgressUnit = "members"
|
||||
self.completed = 0
|
||||
self.total: int | None = None
|
||||
self.phase_index = 0
|
||||
self.sequence = 0
|
||||
|
||||
def update(
|
||||
self,
|
||||
phase: ProgressPhase,
|
||||
completed: int = 0,
|
||||
total: int | None = None,
|
||||
unit: ProgressUnit = "frames",
|
||||
) -> None:
|
||||
with self.lock:
|
||||
if phase != self.phase or unit != self.unit:
|
||||
self.phase_index += 1
|
||||
self.phase_started = self.clock()
|
||||
elif completed < self.completed:
|
||||
raise ValueError("progress counter moved backwards within one phase")
|
||||
elif total is None:
|
||||
total = self.total
|
||||
self.phase, self.unit = phase, unit
|
||||
self.completed, self.total = completed, total
|
||||
|
||||
def sample(self) -> RecordedProgress:
|
||||
with self.lock:
|
||||
now = self.clock()
|
||||
self.sequence += 1
|
||||
return RecordedProgress(
|
||||
claim_generation=self.generation,
|
||||
sequence=self.sequence,
|
||||
phase_index=self.phase_index,
|
||||
phase=self.phase,
|
||||
unit=self.unit,
|
||||
completed=self.completed,
|
||||
total=self.total,
|
||||
elapsed_seconds=max(0.0, now - self.started),
|
||||
phase_elapsed_seconds=max(0.0, now - self.phase_started),
|
||||
)
|
||||
|
||||
|
||||
_CURRENT: ContextVar[RecordedProgressTracker | None] = ContextVar(
|
||||
"observatory_recorded_progress",
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
def report_recorded_progress(
|
||||
phase: ProgressPhase,
|
||||
completed: int = 0,
|
||||
total: int | None = None,
|
||||
unit: ProgressUnit = "frames",
|
||||
) -> None:
|
||||
tracker = _CURRENT.get()
|
||||
if tracker is not None:
|
||||
tracker.update(phase, completed, total, unit)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def observe_recorded_execution(
|
||||
generation: int,
|
||||
send: Callable[[RecordedProgress], None],
|
||||
*,
|
||||
interval_seconds: float = 1.0,
|
||||
) -> Iterator[RecordedProgressTracker]:
|
||||
if not math.isfinite(interval_seconds) or not 0.01 <= interval_seconds <= 30:
|
||||
raise ValueError("progress sampling interval is invalid")
|
||||
tracker = RecordedProgressTracker(generation)
|
||||
stop = threading.Event()
|
||||
|
||||
def pump() -> None:
|
||||
while not stop.is_set():
|
||||
# Progress is secondary observation, not lease or result authority.
|
||||
with suppress(Exception):
|
||||
send(tracker.sample())
|
||||
stop.wait(interval_seconds)
|
||||
|
||||
token = _CURRENT.set(tracker)
|
||||
thread = threading.Thread(target=pump, name="observatory-progress", daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield tracker
|
||||
finally:
|
||||
_CURRENT.reset(token)
|
||||
stop.set()
|
||||
# The production sender reads no body and uses 2-second I/O timeouts.
|
||||
thread.join(timeout=3.0)
|
||||
@@ -18,6 +18,7 @@ import json
|
||||
import re
|
||||
import threading
|
||||
from collections.abc import Callable, Mapping
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Final, Literal, Protocol
|
||||
@@ -31,6 +32,7 @@ from k1link.observatory.recorded_jobs import (
|
||||
OBSERVATORY_RECORDED_JOB_SCHEMA,
|
||||
RecordedExecutorIdentity,
|
||||
)
|
||||
from k1link.observatory.recorded_progress import observe_recorded_execution
|
||||
|
||||
WORKER_006_CONTOUR_ID: Final = "worker-006"
|
||||
MAX_EXECUTOR_FAILURE_MESSAGE_LENGTH: Final = 512
|
||||
@@ -498,7 +500,18 @@ class ObservatoryWorkerAgent:
|
||||
heartbeat.start()
|
||||
|
||||
try:
|
||||
result = adapter.execute(active_job)
|
||||
send_progress = getattr(self._transport, "report_progress", None)
|
||||
observation = (
|
||||
observe_recorded_execution(
|
||||
active_job.claim_generation,
|
||||
lambda snapshot: send_progress(
|
||||
job_id=active_job.job_id, claim_token=claim.claim_token,
|
||||
progress=snapshot,
|
||||
),
|
||||
) if callable(send_progress) else nullcontext()
|
||||
)
|
||||
with observation:
|
||||
result = adapter.execute(active_job)
|
||||
if not isinstance(result, ObservatoryWorkerExecutionResult):
|
||||
raise TypeError("executor returned an unknown result contract")
|
||||
except Exception as exc:
|
||||
|
||||
@@ -47,6 +47,7 @@ from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerResultDraft,
|
||||
PortableWorkerSourceStage,
|
||||
)
|
||||
from k1link.observatory.recorded_progress import RecordedProgress, report_recorded_progress
|
||||
from k1link.observatory.source_admission import (
|
||||
PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID,
|
||||
PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE,
|
||||
@@ -279,6 +280,23 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
},
|
||||
)
|
||||
|
||||
def report_progress(
|
||||
self, *, job_id: str, claim_token: str, progress: RecordedProgress,
|
||||
) -> None:
|
||||
context = self._require_cached_claim(job_id, claim_token)
|
||||
if progress.claim_generation != context.claim_generation:
|
||||
raise ObservatoryWorkerHttpError("progress generation changed")
|
||||
# No response body is needed. Bounded I/O cannot stall the execution thread.
|
||||
with self._client.stream(
|
||||
"POST", self._job_path(job_id, "progress"),
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-worker-progress-request/v1",
|
||||
"claim_token": claim_token, "progress": progress.model_dump(mode="json"),
|
||||
}, timeout=httpx.Timeout(2.0),
|
||||
) as response:
|
||||
if response.status_code != 204:
|
||||
raise ObservatoryWorkerHttpError("progress observation was not accepted")
|
||||
|
||||
def succeed(
|
||||
self,
|
||||
*,
|
||||
@@ -336,6 +354,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
headers=headers,
|
||||
)
|
||||
members = _source_members(manifest, job)
|
||||
report_recorded_progress("source-transfer", 0, len(members), "members")
|
||||
root = _secure_directory(
|
||||
self._work_root
|
||||
/ "sources"
|
||||
@@ -368,10 +387,16 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
members=camera_members,
|
||||
layout=layout,
|
||||
)
|
||||
completed_members = len(camera_members) if camera_epoch_ready else 0
|
||||
report_recorded_progress("source-transfer", completed_members, len(members), "members")
|
||||
for destination, member in destinations.items():
|
||||
if camera_epoch_ready and member.kind in {"camera-init", "camera-segment"}:
|
||||
continue
|
||||
if _matches_file(destination, member.sha256, member.byte_length):
|
||||
completed_members += 1
|
||||
report_recorded_progress(
|
||||
"source-transfer", completed_members, len(members), "members",
|
||||
)
|
||||
continue
|
||||
self._download_member(
|
||||
job_id=job.job_id,
|
||||
@@ -379,6 +404,8 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
member=member,
|
||||
destination=destination,
|
||||
)
|
||||
completed_members += 1
|
||||
report_recorded_progress("source-transfer", completed_members, len(members), "members")
|
||||
manifest_path = root / "materialization-manifest.json"
|
||||
_write_local_exact(manifest_path, canonical_json(manifest))
|
||||
return PortableWorkerSourceStage(
|
||||
@@ -440,7 +467,8 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
or {member.role for member in upload_members} != set(artifacts)
|
||||
):
|
||||
raise ObservatoryWorkerHttpError("Worker result artifact roles are not unique")
|
||||
for member in upload_members:
|
||||
report_recorded_progress("result-transfer", 0, len(upload_members), "members")
|
||||
for member_index, member in enumerate(upload_members):
|
||||
artifact = artifacts.get(member.role)
|
||||
expected_member_id = hashlib.sha256(
|
||||
canonical_json(
|
||||
@@ -461,6 +489,9 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
"result upload plan differs from the local manifest"
|
||||
)
|
||||
if member.uploaded:
|
||||
report_recorded_progress(
|
||||
"result-transfer", member_index + 1, len(upload_members), "members",
|
||||
)
|
||||
continue
|
||||
relative = relative_artifact_path(artifact.relative_path)
|
||||
source = _confined_local_member(draft.root, relative.parts)
|
||||
@@ -489,6 +520,9 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"result upload acknowledgement did not seal its member"
|
||||
)
|
||||
report_recorded_progress(
|
||||
"result-transfer", member_index + 1, len(upload_members), "members",
|
||||
)
|
||||
receipt = self._required_json_request(
|
||||
"POST",
|
||||
self._job_path(
|
||||
|
||||
@@ -1151,6 +1151,17 @@ def build_observatory_router(
|
||||
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
|
||||
@router.get("/api/v1/observatory/runs/{job_id}/progress")
|
||||
def get_observatory_progress(
|
||||
job_id: str = ApiPath(pattern=r"^observatory-run-[a-f0-9]{32}$"),
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
return recorded_job_queue.progress(job_id)
|
||||
except ObservatoryRecordedQueueNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail="Расчёт не найден.") from exc
|
||||
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||
raise HTTPException(status_code=503, detail="Прогресс недоступен.") from exc
|
||||
|
||||
@router.get("/api/v1/observatory/runs/{job_id}")
|
||||
def get_observatory_recorded_run(
|
||||
job_id: str = ApiPath(
|
||||
|
||||
@@ -51,6 +51,7 @@ from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedQueueStaleClaimError,
|
||||
RecordedExecutorIdentity,
|
||||
)
|
||||
from k1link.observatory.recorded_progress import RecordedProgress
|
||||
|
||||
OBSERVATORY_WORKER_CAPABILITY_CLAIM_REQUEST_SCHEMA: Final = (
|
||||
"missioncore.observatory-worker-claim-request/v2"
|
||||
@@ -194,6 +195,12 @@ class ObservatoryWorkerRenewRequest(_StrictWorkerRequest):
|
||||
heartbeat_sequence: int = Field(ge=1)
|
||||
|
||||
|
||||
class ObservatoryWorkerProgressRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-progress-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
progress: RecordedProgress
|
||||
|
||||
|
||||
class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
@@ -333,6 +340,17 @@ def build_observatory_worker_router(
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/progress", status_code=204)
|
||||
def report_job_progress(
|
||||
request: ObservatoryWorkerProgressRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> Response:
|
||||
_queue_call(lambda: queue.report_progress(
|
||||
job_id, claim_token=request.claim_token,
|
||||
claimant_id=authentication.contour_id, progress=request.progress,
|
||||
))
|
||||
return Response(status_code=204)
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/checkpoint")
|
||||
def checkpoint_job(
|
||||
request: ObservatoryWorkerCheckpointRequest,
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from test_observatory_recorded_jobs import _queue, _running_job
|
||||
from test_observatory_worker_api import WORKER_HEADERS, _claim_request, _services
|
||||
|
||||
from k1link.observatory.m49_timing_progress import M49TimingProgress
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedQueueConflictError,
|
||||
ObservatoryRecordedQueueStaleClaimError,
|
||||
)
|
||||
from k1link.observatory.recorded_progress import (
|
||||
RecordedProgress,
|
||||
RecordedProgressTracker,
|
||||
observe_recorded_execution,
|
||||
report_recorded_progress,
|
||||
)
|
||||
|
||||
|
||||
def _progress(**changes: object) -> RecordedProgress:
|
||||
return RecordedProgress.model_validate(
|
||||
{
|
||||
"claim_generation": 1,
|
||||
"sequence": 1,
|
||||
"phase_index": 1,
|
||||
"phase": "computing",
|
||||
"unit": "frames",
|
||||
"completed": 3,
|
||||
"total": 10,
|
||||
"elapsed_seconds": 3.0,
|
||||
"phase_elapsed_seconds": 2.0,
|
||||
**changes,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
{"completed": True},
|
||||
{"completed": 11},
|
||||
{"total": 0},
|
||||
{"phase": "done"},
|
||||
{"elapsed_seconds": float("nan")},
|
||||
{"phase_elapsed_seconds": 4.0},
|
||||
{"sequence": 2**53},
|
||||
{"path": "/secret"},
|
||||
],
|
||||
)
|
||||
def test_progress_rejects_unbounded_or_invented_values(change: dict[str, object]) -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
_progress(**change)
|
||||
|
||||
|
||||
def test_progress_is_one_durable_snapshot_not_job_identity_or_lease(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job, claim = _running_job(queue)
|
||||
original = queue.get(job.job_id).as_dict()
|
||||
assert queue.progress(job.job_id)["progress"] is None
|
||||
for sequence in range(1, 101):
|
||||
queue.report_progress(
|
||||
job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
claimant_id="recorded-worker",
|
||||
progress=_progress(sequence=sequence),
|
||||
)
|
||||
assert queue.get(job.job_id).as_dict() == original
|
||||
reopened = _queue(tmp_path)
|
||||
view = reopened.progress(job.job_id)
|
||||
assert view["definition_sha256"] == job.definition_sha256
|
||||
assert view["claim_generation"] == 1
|
||||
assert view["age_seconds"] == 0.0
|
||||
assert view["progress"]["sequence"] == 100
|
||||
with sqlite3.connect(queue.database_path) as connection:
|
||||
assert (
|
||||
connection.execute("SELECT COUNT(*) FROM observatory_recorded_progress").fetchone()[0]
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
{"sequence": 1, "completed": 4},
|
||||
{"sequence": 2, "completed": 2},
|
||||
{"sequence": 2, "total": 12},
|
||||
{"sequence": 2, "phase": "result-transfer"},
|
||||
{"sequence": 2, "elapsed_seconds": 2.0},
|
||||
],
|
||||
)
|
||||
def test_progress_retry_is_idempotent_and_regressions_rejected(tmp_path: Path, change) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job, claim = _running_job(queue)
|
||||
|
||||
def send(value):
|
||||
queue.report_progress(
|
||||
job.job_id, claim_token=claim.claim_token, claimant_id="recorded-worker", progress=value
|
||||
)
|
||||
|
||||
send(_progress())
|
||||
send(_progress())
|
||||
with pytest.raises(ObservatoryRecordedQueueConflictError):
|
||||
send(_progress(**change))
|
||||
send(_progress(sequence=3, phase_index=2, phase="result-assembly", completed=0, total=None))
|
||||
|
||||
|
||||
def test_progress_old_generation_cannot_write_or_project_as_current(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job, claim = _running_job(queue)
|
||||
with pytest.raises(ObservatoryRecordedQueueStaleClaimError):
|
||||
queue.report_progress(
|
||||
job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
claimant_id="recorded-worker",
|
||||
progress=_progress(claim_generation=2),
|
||||
)
|
||||
with pytest.raises(ObservatoryRecordedQueueStaleClaimError):
|
||||
queue.report_progress(
|
||||
job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
claimant_id="another-worker",
|
||||
progress=_progress(),
|
||||
)
|
||||
# Synthetic old attempt retained on disk is not the new attempt's progress.
|
||||
with sqlite3.connect(queue.database_path) as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO observatory_recorded_progress VALUES (?, ?, ?)",
|
||||
(job.job_id, _progress(claim_generation=2).model_dump_json(), job.created_at_utc),
|
||||
)
|
||||
assert queue.progress(job.job_id)["progress"] is None
|
||||
|
||||
|
||||
def test_progress_tracker_preserves_total_and_separates_phases() -> None:
|
||||
tracker = RecordedProgressTracker(2, clock=lambda: 10.0)
|
||||
tracker.update("computing", 0, 10)
|
||||
first = tracker.sample()
|
||||
tracker.update("computing", 2)
|
||||
second = tracker.sample()
|
||||
assert second.total == 10 and second.follows(first)
|
||||
tracker.update("result-assembly", unit="steps")
|
||||
third = tracker.sample()
|
||||
assert third.completed == 0 and third.total is None and third.follows(second)
|
||||
|
||||
|
||||
def test_m49_invoker_preserves_arguments_and_observes_real_output(tmp_path, monkeypatch):
|
||||
from k1link.observatory import m49_portable_executor as executor
|
||||
|
||||
script = tmp_path / "synthetic.py"
|
||||
script.write_text(
|
||||
"import os, sys\n"
|
||||
"from pathlib import Path\n"
|
||||
"assert sys.argv[1] == 'schedule.tsv'\n"
|
||||
"assert sys.argv[2] == 'outputs'\n"
|
||||
"assert os.environ['LANG'] == 'C' and os.environ['TZ'] == 'UTC'\n"
|
||||
"Path(sys.argv[3]).write_text('timeline_frame_index\\tsource_frame_index\\tother\\n'"
|
||||
" + '0\\t0\\t0\\t1\\t0\\t12\\t10\\t2\\t1.2\\t1.3\\n')\n"
|
||||
"print('synthetic runner completed')\n"
|
||||
)
|
||||
counters = []
|
||||
monkeypatch.setattr(
|
||||
executor, "report_recorded_progress", lambda phase, count: counters.append(count)
|
||||
)
|
||||
executor._invoke_exact_runner(
|
||||
binary=Path(sys.executable),
|
||||
sequence=script,
|
||||
schedule=Path("schedule.tsv"),
|
||||
output=Path("outputs"),
|
||||
timing=tmp_path / "timing.tsv",
|
||||
workspace=tmp_path,
|
||||
timeout_seconds=5,
|
||||
)
|
||||
assert counters[-1] == 1
|
||||
assert (tmp_path / "runner.stdout.log").read_text().strip() == "synthetic runner completed"
|
||||
assert (tmp_path / "runner.stderr.log").read_bytes() == b""
|
||||
|
||||
|
||||
def test_m49_invoker_timeout_reaps_only_its_child(tmp_path):
|
||||
import os
|
||||
|
||||
from k1link.observatory.m49_portable_executor import (
|
||||
M49PortableExecutorError,
|
||||
_invoke_exact_runner,
|
||||
)
|
||||
|
||||
script = tmp_path / "synthetic.py"
|
||||
script.write_text(
|
||||
"import os, time\nfrom pathlib import Path\n"
|
||||
"Path('synthetic.pid').write_text(str(os.getpid()))\ntime.sleep(30)\n"
|
||||
)
|
||||
with pytest.raises(M49PortableExecutorError, match="invocation failed"):
|
||||
_invoke_exact_runner(
|
||||
binary=Path(sys.executable),
|
||||
sequence=script,
|
||||
schedule=tmp_path / "schedule.tsv",
|
||||
output=tmp_path / "outputs",
|
||||
timing=tmp_path / "timing.tsv",
|
||||
workspace=tmp_path,
|
||||
timeout_seconds=1,
|
||||
)
|
||||
pid = int((tmp_path / "synthetic.pid").read_text())
|
||||
with pytest.raises(ProcessLookupError):
|
||||
os.kill(pid, 0)
|
||||
|
||||
|
||||
def test_progress_sender_failure_does_not_escape_or_leave_a_thread() -> None:
|
||||
sampled = threading.Event()
|
||||
|
||||
def unavailable(snapshot):
|
||||
assert isinstance(snapshot, RecordedProgress)
|
||||
sampled.set()
|
||||
raise RuntimeError("synthetic observer connection loss")
|
||||
|
||||
with observe_recorded_execution(1, unavailable, interval_seconds=0.01) as tracker:
|
||||
report_recorded_progress("computing", 5, 8)
|
||||
assert sampled.wait(1)
|
||||
assert tracker.sample().completed == 5
|
||||
assert not any(thread.name == "observatory-progress" for thread in threading.enumerate())
|
||||
report_recorded_progress("computing", 1, 2) # no leaked execution context
|
||||
assert tracker.sample().completed == 5
|
||||
|
||||
|
||||
def test_worker_keeps_compute_and_lease_when_progress_transport_fails(tmp_path: Path) -> None:
|
||||
from test_observatory_worker_agent import (
|
||||
BlockingExecutor,
|
||||
FakeTransport,
|
||||
_enqueue,
|
||||
_heartbeat_agent,
|
||||
)
|
||||
from test_observatory_worker_agent import (
|
||||
_queue as worker_queue,
|
||||
)
|
||||
|
||||
queue = worker_queue(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
attempted = threading.Event()
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
|
||||
class ObserverFailureTransport(FakeTransport):
|
||||
def report_progress(self, **_kwargs):
|
||||
attempted.set()
|
||||
raise RuntimeError("observer unavailable")
|
||||
|
||||
agent = _heartbeat_agent(
|
||||
ObserverFailureTransport(queue),
|
||||
BlockingExecutor(entered=entered, release=release),
|
||||
)
|
||||
reports = []
|
||||
thread = threading.Thread(target=lambda: reports.append(agent.run_once()))
|
||||
thread.start()
|
||||
assert entered.wait(2) and attempted.wait(2)
|
||||
release.set()
|
||||
thread.join(2)
|
||||
assert not thread.is_alive()
|
||||
assert reports[0].state == "succeeded"
|
||||
assert queue.get(job_id).state == "succeeded"
|
||||
|
||||
|
||||
def test_http_progress_uses_active_claim_and_short_bodyless_request(tmp_path: Path) -> None:
|
||||
import httpx
|
||||
from test_observatory_worker_http_transport import (
|
||||
BEARER_TOKEN,
|
||||
CLAIM_TOKEN,
|
||||
JOB_ID,
|
||||
_cache_claim,
|
||||
_claim_response,
|
||||
)
|
||||
|
||||
from k1link.observatory.worker_http_transport import ObservatoryWorkerHttpGateway
|
||||
|
||||
requests = []
|
||||
|
||||
def handle(request):
|
||||
if request.url.path.endswith("/claims"):
|
||||
return _claim_response()
|
||||
requests.append(request)
|
||||
assert request.url.path.endswith("/progress")
|
||||
assert request.extensions["timeout"]["read"] == 2.0
|
||||
assert json.loads(request.content)["claim_token"] == CLAIM_TOKEN
|
||||
return httpx.Response(204)
|
||||
|
||||
with ObservatoryWorkerHttpGateway(
|
||||
base_url="http://127.0.0.1:8000",
|
||||
bearer_token=BEARER_TOKEN,
|
||||
work_root=tmp_path,
|
||||
transport=httpx.MockTransport(handle),
|
||||
) as gateway:
|
||||
_cache_claim(gateway)
|
||||
gateway.report_progress(job_id=JOB_ID, claim_token=CLAIM_TOKEN, progress=_progress())
|
||||
assert len(requests) == 1
|
||||
|
||||
|
||||
def test_tgs_progress_counts_only_complete_ordered_flushed_rows(tmp_path: Path) -> None:
|
||||
path = tmp_path / "timing.tsv"
|
||||
observer = M49TimingProgress(path)
|
||||
assert observer.poll() == 0
|
||||
path.write_bytes(
|
||||
b"timeline_frame_index\tsource_frame_index\trest\n0\t0\t0\t1\t0\t1\t1\t0\t1\t1\n1\t1"
|
||||
)
|
||||
assert observer.poll() == 1
|
||||
with path.open("ab") as stream:
|
||||
stream.write(b"\t1\t1\t1\t1\t1\t0\t1\t1\n")
|
||||
assert observer.poll() == 2
|
||||
with path.open("ab") as stream:
|
||||
stream.write(b"4\t4\t4\t1\t4\t1\t1\t0\t1\t1\n")
|
||||
assert observer.poll() == 2 and observer.invalid
|
||||
|
||||
|
||||
def test_progress_api_requires_auth_exact_claim_and_strict_payload(tmp_path: Path) -> None:
|
||||
from test_observatory_worker_api import _enqueue
|
||||
|
||||
client, queue = _services(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
response = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
headers=WORKER_HEADERS,
|
||||
json=_claim_request("progress-claim"),
|
||||
)
|
||||
claim = response.json()
|
||||
token = claim["claim_token"]
|
||||
queue.start(job_id, claim_token=token)
|
||||
path = f"/api/v1/worker/observatory/recorded-jobs/{job_id}/progress"
|
||||
body = {
|
||||
"schema_version": "missioncore.observatory-worker-progress-request/v1",
|
||||
"claim_token": token,
|
||||
"progress": _progress().model_dump(mode="json"),
|
||||
}
|
||||
assert client.post(path, json=body).status_code == 401
|
||||
assert (
|
||||
client.post(path, headers=WORKER_HEADERS, json={**body, "command": "no"}).status_code == 422
|
||||
)
|
||||
assert client.post(path, headers=WORKER_HEADERS, json=body).status_code == 204
|
||||
assert queue.progress(job_id)["progress"] == json.loads(_progress().model_dump_json())
|
||||
Reference in New Issue
Block a user