feat(observatory): expose attempt-bound recorded progress
This commit is contained in:
@@ -26,6 +26,7 @@ export interface ObservatoryRecordedJob {
|
|||||||
readonly sourceSessionId: string;
|
readonly sourceSessionId: string;
|
||||||
readonly setupId: string;
|
readonly setupId: string;
|
||||||
readonly definitionSha256: string;
|
readonly definitionSha256: string;
|
||||||
|
readonly claimGeneration: number;
|
||||||
readonly state: ObservatoryRecordedJobState;
|
readonly state: ObservatoryRecordedJobState;
|
||||||
readonly restartFromZero: boolean;
|
readonly restartFromZero: boolean;
|
||||||
readonly resultId: string | null;
|
readonly resultId: string | null;
|
||||||
@@ -229,6 +230,7 @@ function decodeJob(value: unknown): ObservatoryRecordedJob {
|
|||||||
sourceSessionId: text(source.session_id, "source.session_id"),
|
sourceSessionId: text(source.session_id, "source.session_id"),
|
||||||
setupId: text(setup.setup_id, "setup.setup_id"),
|
setupId: text(setup.setup_id, "setup.setup_id"),
|
||||||
definitionSha256: String(setup.definition_sha256),
|
definitionSha256: String(setup.definition_sha256),
|
||||||
|
claimGeneration: nonNegativeInteger(row.claim_generation, "claim_generation"),
|
||||||
state,
|
state,
|
||||||
restartFromZero: boolean(row.restart_from_zero, "restart_from_zero"),
|
restartFromZero: boolean(row.restart_from_zero, "restart_from_zero"),
|
||||||
resultId: result === null ? null : text(result.result_id, "result.result_id"),
|
resultId: result === null ? null : text(result.result_id, "result.result_id"),
|
||||||
|
|||||||
@@ -0,0 +1,118 @@
|
|||||||
|
import type { ObservatoryRecordedJob } from "./recordedJobs";
|
||||||
|
|
||||||
|
const PHASES = {
|
||||||
|
"source-transfer": "Передаём входные данные",
|
||||||
|
"source-preparation": "Подготавливаем вход",
|
||||||
|
computing: "Обрабатываем запись",
|
||||||
|
"result-assembly": "Собираем результат",
|
||||||
|
"result-transfer": "Передаём результат",
|
||||||
|
} as const;
|
||||||
|
const UNITS = { frames: "кадров", members: "файлов", steps: "операций" } as const;
|
||||||
|
type Phase = keyof typeof PHASES;
|
||||||
|
type Unit = keyof typeof UNITS;
|
||||||
|
|
||||||
|
export interface RecordedProgressView {
|
||||||
|
readonly jobId: string;
|
||||||
|
readonly claimGeneration: number;
|
||||||
|
readonly sequence: number;
|
||||||
|
readonly phase: Phase;
|
||||||
|
readonly unit: Unit;
|
||||||
|
readonly completed: number;
|
||||||
|
readonly total: number | null;
|
||||||
|
readonly ageSeconds: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchRecordedProgress(
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
{ signal, fetcher = globalThis.fetch }: {
|
||||||
|
signal?: AbortSignal;
|
||||||
|
fetcher?: typeof globalThis.fetch;
|
||||||
|
} = {},
|
||||||
|
): Promise<RecordedProgressView | null> {
|
||||||
|
const request = new AbortController();
|
||||||
|
const abort = () => request.abort();
|
||||||
|
signal?.addEventListener("abort", abort, { once: true });
|
||||||
|
if (signal?.aborted) request.abort();
|
||||||
|
const timer = globalThis.setTimeout(abort, 2_500);
|
||||||
|
try {
|
||||||
|
const response = await fetcher(
|
||||||
|
`/api/v1/observatory/runs/${encodeURIComponent(job.jobId)}/progress`,
|
||||||
|
{ signal: request.signal, headers: { Accept: "application/json" } },
|
||||||
|
);
|
||||||
|
if (!response.ok) throw new Error("Прогресс расчёта недоступен.");
|
||||||
|
return decodeRecordedProgress(await response.json(), job);
|
||||||
|
} finally {
|
||||||
|
globalThis.clearTimeout(timer);
|
||||||
|
signal?.removeEventListener("abort", abort);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function decodeRecordedProgress(
|
||||||
|
value: unknown, job: ObservatoryRecordedJob,
|
||||||
|
): RecordedProgressView | null {
|
||||||
|
const row = object(value);
|
||||||
|
keys(row, ["schema_version", "job_id", "source_session_id", "setup_id",
|
||||||
|
"definition_sha256", "claim_generation", "state", "received_at_utc", "age_seconds", "progress"]);
|
||||||
|
if (row.schema_version !== "missioncore.observatory-recorded-progress-view/v1"
|
||||||
|
|| row.job_id !== job.jobId || row.source_session_id !== job.sourceSessionId
|
||||||
|
|| row.setup_id !== job.setupId || row.definition_sha256 !== job.definitionSha256
|
||||||
|
|| row.claim_generation !== job.claimGeneration) throw new Error("Прогресс другой попытки.");
|
||||||
|
if (row.progress === null || row.state !== job.state) return null;
|
||||||
|
const progress = object(row.progress);
|
||||||
|
keys(progress, ["schema_version", "claim_generation", "sequence", "phase_index",
|
||||||
|
"phase", "unit", "completed", "total", "elapsed_seconds", "phase_elapsed_seconds"]);
|
||||||
|
if (progress.schema_version !== "missioncore.observatory-recorded-progress/v1"
|
||||||
|
|| progress.claim_generation !== row.claim_generation
|
||||||
|
|| !(typeof progress.phase === "string" && Object.hasOwn(PHASES, progress.phase))
|
||||||
|
|| !(typeof progress.unit === "string" && Object.hasOwn(UNITS, progress.unit))
|
||||||
|
|| typeof row.received_at_utc !== "string" || !Number.isFinite(Date.parse(row.received_at_utc))) {
|
||||||
|
throw new Error("Некорректный прогресс.");
|
||||||
|
}
|
||||||
|
const completed = integer(progress.completed);
|
||||||
|
const total = progress.total === null ? null : integer(progress.total, 1);
|
||||||
|
const elapsed = finite(progress.elapsed_seconds);
|
||||||
|
if ((total !== null && completed > total) || finite(progress.phase_elapsed_seconds) > elapsed) {
|
||||||
|
throw new Error("Некорректные счётчики прогресса.");
|
||||||
|
}
|
||||||
|
integer(progress.phase_index);
|
||||||
|
return {
|
||||||
|
jobId: job.jobId, claimGeneration: integer(progress.claim_generation, 1),
|
||||||
|
sequence: integer(progress.sequence, 1), phase: progress.phase as Phase,
|
||||||
|
unit: progress.unit as Unit, completed, total, ageSeconds: finite(row.age_seconds),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function recordedProgressLabel(
|
||||||
|
job: ObservatoryRecordedJob | null, progress: RecordedProgressView | null,
|
||||||
|
): string {
|
||||||
|
if (job?.publication.state === "pending") return "Сохраняем результат";
|
||||||
|
if (!job || job.state === "accepted" || job.state === "queued") return "Ожидаем расчёт";
|
||||||
|
if (job.state === "paused" || job.state === "preemption-pending") return "Расчёт приостановлен";
|
||||||
|
if (job.state === "reconciliation-required") return "Проверяем состояние расчёта";
|
||||||
|
if (!progress || progress.jobId !== job.jobId
|
||||||
|
|| progress.claimGeneration !== job.claimGeneration) return "Ожидаем данные расчёта";
|
||||||
|
if (progress.ageSeconds > 15) return "Ожидаем обновление прогресса";
|
||||||
|
const count = progress.total === null
|
||||||
|
? (progress.completed > 0 ? ` · ${progress.completed.toLocaleString("ru-RU")} ${UNITS[progress.unit]}` : "")
|
||||||
|
: ` · ${progress.completed.toLocaleString("ru-RU")} / ${progress.total.toLocaleString("ru-RU")} ${UNITS[progress.unit]}`;
|
||||||
|
return PHASES[progress.phase] + count;
|
||||||
|
}
|
||||||
|
|
||||||
|
function object(value: unknown): Record<string, unknown> {
|
||||||
|
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Прогресс: ожидался объект.");
|
||||||
|
return value as Record<string, unknown>;
|
||||||
|
}
|
||||||
|
function keys(row: Record<string, unknown>, expected: string[]): void {
|
||||||
|
if (Object.keys(row).length !== expected.length || expected.some((key) => !Object.hasOwn(row, key))) {
|
||||||
|
throw new Error("Прогресс: неизвестные поля.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
function finite(value: unknown): number {
|
||||||
|
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new Error("Прогресс: некорректное число.");
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
function integer(value: unknown, minimum = 0): number {
|
||||||
|
const number = finite(value);
|
||||||
|
if (!Number.isSafeInteger(number) || number < minimum) throw new Error("Прогресс: некорректный счётчик.");
|
||||||
|
return number;
|
||||||
|
}
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import { useCallback, useEffect, useRef, useState } from "react";
|
import { useCallback, useEffect, useRef, useState } from "react";
|
||||||
|
import { fetchRecordedProgress, type RecordedProgressView } from "./recordedProgress";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
fetchObservatoryRecordedJobs,
|
fetchObservatoryRecordedJobs,
|
||||||
@@ -21,6 +22,7 @@ interface JobSnapshot {
|
|||||||
readonly jobs: readonly ObservatoryRecordedJob[];
|
readonly jobs: readonly ObservatoryRecordedJob[];
|
||||||
readonly state: RecordedJobsState;
|
readonly state: RecordedJobsState;
|
||||||
readonly error: string | null;
|
readonly error: string | null;
|
||||||
|
readonly progress?: RecordedProgressView | null;
|
||||||
}
|
}
|
||||||
const EMPTY_JOBS = [] as const;
|
const EMPTY_JOBS = [] as const;
|
||||||
const OPEN_STATES = new Set([
|
const OPEN_STATES = new Set([
|
||||||
@@ -63,14 +65,19 @@ export function useObservatoryRecordedJobs(
|
|||||||
setSnapshot({
|
setSnapshot({
|
||||||
selectionKey, jobs,
|
selectionKey, jobs,
|
||||||
state: jobs.length > 0 ? "refreshing" : "loading", error: null,
|
state: jobs.length > 0 ? "refreshing" : "loading", error: null,
|
||||||
|
progress: current?.progress ?? null,
|
||||||
});
|
});
|
||||||
void fetchObservatoryRecordedJobs(sourceSessionId, setupId, {
|
void fetchObservatoryRecordedJobs(sourceSessionId, setupId, {
|
||||||
definitionSha256, signal: request.signal,
|
definitionSha256, signal: request.signal,
|
||||||
}).then((next) => {
|
}).then(async (next) => {
|
||||||
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
||||||
const pending = next.find((job) => OPEN_STATES.has(job.state));
|
const pending = next.find((job) => OPEN_STATES.has(job.state));
|
||||||
|
const progress = pending
|
||||||
|
? await fetchRecordedProgress(pending, { signal: request.signal }).catch(() => null)
|
||||||
|
: null;
|
||||||
|
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
||||||
if (pending) observedJobId.current = pending.jobId;
|
if (pending) observedJobId.current = pending.jobId;
|
||||||
setSnapshot({ selectionKey, jobs: next, state: "ready", error: null });
|
setSnapshot({ selectionKey, jobs: next, state: "ready", error: null, progress });
|
||||||
}).catch((caught: unknown) => {
|
}).catch((caught: unknown) => {
|
||||||
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
||||||
setSnapshot({
|
setSnapshot({
|
||||||
@@ -95,7 +102,7 @@ export function useObservatoryRecordedJobs(
|
|||||||
&& latestJob.jobId === observedJobId.current;
|
&& latestJob.jobId === observedJobId.current;
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (state !== "ready" || (!activeJob && !publicationPending)) return;
|
if ((state !== "ready" && state !== "error") || (!activeJob && !publicationPending)) return;
|
||||||
const timer = globalThis.setTimeout(
|
const timer = globalThis.setTimeout(
|
||||||
() => setRevision((value) => value + 1),
|
() => setRevision((value) => value + 1),
|
||||||
POLL_INTERVAL_MS,
|
POLL_INTERVAL_MS,
|
||||||
@@ -185,6 +192,7 @@ export function useObservatoryRecordedJobs(
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
jobs, latestJob, activeJob, publicationPending, computationFailed,
|
jobs, latestJob, activeJob, publicationPending, computationFailed,
|
||||||
|
progress: current?.progress ?? null,
|
||||||
state, error, refresh, submit, retryPublication,
|
state, error, refresh, submit, retryPublication,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ import {
|
|||||||
import { useObservatoryCatalog } from "../../core/observatory/useObservatoryCatalog";
|
import { useObservatoryCatalog } from "../../core/observatory/useObservatoryCatalog";
|
||||||
import { useObservatoryLaboratorySetups } from "../../core/observatory/useObservatoryLaboratorySetups";
|
import { useObservatoryLaboratorySetups } from "../../core/observatory/useObservatoryLaboratorySetups";
|
||||||
import { useObservatoryRecordedJobs } from "../../core/observatory/useObservatoryRecordedJobs";
|
import { useObservatoryRecordedJobs } from "../../core/observatory/useObservatoryRecordedJobs";
|
||||||
|
import { recordedProgressLabel } from "../../core/observatory/recordedProgress";
|
||||||
import type { WorkspaceDefinition } from "../../productModel";
|
import type { WorkspaceDefinition } from "../../productModel";
|
||||||
|
|
||||||
const EMPTY_OBSERVATORY_ITEMS = [] as const;
|
const EMPTY_OBSERVATORY_ITEMS = [] as const;
|
||||||
@@ -466,7 +467,12 @@ export function ObservatoryWorkspace({
|
|||||||
</Button>
|
</Button>
|
||||||
<div className="observatory-catalog-bar__run" aria-busy={calculationPending}>
|
<div className="observatory-catalog-bar__run" aria-busy={calculationPending}>
|
||||||
{calculationPending ? (
|
{calculationPending ? (
|
||||||
<ActivityIndicator size="compact" label="Ожидание результата расчёта" />
|
<>
|
||||||
|
<ActivityIndicator size="compact" />
|
||||||
|
<span role="status">
|
||||||
|
{recordedProgressLabel(presentedJob, recordedJobsController.progress)}
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
{showCalculate ? (
|
{showCalculate ? (
|
||||||
<Button
|
<Button
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { after, before, test } from "node:test";
|
||||||
|
import { createServer } from "vite";
|
||||||
|
|
||||||
|
let server;
|
||||||
|
let decode;
|
||||||
|
let label;
|
||||||
|
let fetchProgress;
|
||||||
|
const job = {
|
||||||
|
jobId: "observatory-run-" + "1".repeat(32),
|
||||||
|
sourceSessionId: "source-a", setupId: "m49-tgs",
|
||||||
|
definitionSha256: "a".repeat(64), claimGeneration: 2, state: "running",
|
||||||
|
publication: { state: "not-required" },
|
||||||
|
};
|
||||||
|
function view() {
|
||||||
|
return {
|
||||||
|
schema_version: "missioncore.observatory-recorded-progress-view/v1",
|
||||||
|
job_id: job.jobId, source_session_id: job.sourceSessionId, setup_id: job.setupId,
|
||||||
|
definition_sha256: job.definitionSha256, claim_generation: 2, state: "running",
|
||||||
|
received_at_utc: "2026-09-03T12:00:00Z", age_seconds: 0.5,
|
||||||
|
progress: {
|
||||||
|
schema_version: "missioncore.observatory-recorded-progress/v1",
|
||||||
|
claim_generation: 2, sequence: 4, phase_index: 2,
|
||||||
|
phase: "computing", unit: "frames", completed: 3, total: 10,
|
||||||
|
elapsed_seconds: 8, phase_elapsed_seconds: 5,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
before(async () => {
|
||||||
|
server = await createServer({ configFile: false, logLevel: "silent", server: { middlewareMode: true } });
|
||||||
|
const module = await server.ssrLoadModule("/src/core/observatory/recordedProgress.ts");
|
||||||
|
decode = module.decodeRecordedProgress;
|
||||||
|
label = module.recordedProgressLabel;
|
||||||
|
fetchProgress = module.fetchRecordedProgress;
|
||||||
|
});
|
||||||
|
after(async () => { await server?.close(); });
|
||||||
|
|
||||||
|
test("actual frame count is separate from completion and publication", () => {
|
||||||
|
const progress = decode(view(), job);
|
||||||
|
assert.equal(label(job, progress), "Обрабатываем запись · 3 / 10 кадров");
|
||||||
|
assert.equal(label({ ...job, publication: { state: "pending" } }, progress), "Сохраняем результат");
|
||||||
|
assert.doesNotMatch(label(job, progress), /%|завершён|FPS|ETA/);
|
||||||
|
});
|
||||||
|
test("unknown total never becomes a synthetic percentage", () => {
|
||||||
|
const value = view();
|
||||||
|
value.progress.total = null;
|
||||||
|
assert.equal(label(job, decode(value, job)), "Обрабатываем запись · 3 кадров");
|
||||||
|
value.progress.completed = 0;
|
||||||
|
assert.equal(label(job, decode(value, job)), "Обрабатываем запись");
|
||||||
|
});
|
||||||
|
test("old source definition or attempt cannot flash as current progress", () => {
|
||||||
|
for (const delta of [
|
||||||
|
{ job_id: "other" }, { source_session_id: "other" }, { setup_id: "other" },
|
||||||
|
{ definition_sha256: "b".repeat(64) }, { claim_generation: 1 },
|
||||||
|
]) assert.throws(() => decode({ ...view(), ...delta }, job));
|
||||||
|
assert.equal(decode({ ...view(), state: "succeeded" }, job), null);
|
||||||
|
assert.equal(label({ ...job, claimGeneration: 3 }, decode(view(), job)), "Ожидаем данные расчёта");
|
||||||
|
});
|
||||||
|
test("invalid counters, invented phase and extra fields fail closed", () => {
|
||||||
|
for (const delta of [
|
||||||
|
{ completed: true }, { completed: 11 }, { completed: -1 }, { total: 0 },
|
||||||
|
{ completed: Number.MAX_SAFE_INTEGER + 1 }, { phase: "done" }, { path: "/private" },
|
||||||
|
{ phase_elapsed_seconds: 9 }, { claim_generation: 3 },
|
||||||
|
]) assert.throws(() => decode({ ...view(), progress: { ...view().progress, ...delta } }, job));
|
||||||
|
});
|
||||||
|
test("missing and stale telemetry show no fabricated advancement", () => {
|
||||||
|
assert.equal(decode({ ...view(), progress: null, age_seconds: null, received_at_utc: null }, job), null);
|
||||||
|
assert.equal(label(job, null), "Ожидаем данные расчёта");
|
||||||
|
assert.equal(label(job, decode({ ...view(), age_seconds: 30 }, job)), "Ожидаем обновление прогресса");
|
||||||
|
});
|
||||||
|
test("progress is a read-only request bound to the active job", async () => {
|
||||||
|
const calls = [];
|
||||||
|
const progress = await fetchProgress(job, { fetcher: async (path, init) => {
|
||||||
|
calls.push([path, init.method ?? "GET"]);
|
||||||
|
return new Response(JSON.stringify(view()));
|
||||||
|
} });
|
||||||
|
assert.deepEqual(calls, [[`/api/v1/observatory/runs/${job.jobId}/progress`, "GET"]]);
|
||||||
|
assert.equal(progress.completed, 3);
|
||||||
|
});
|
||||||
@@ -47,6 +47,7 @@ from k1link.observatory.portable_worker_runtime import (
|
|||||||
PortableWorkerSourceStage,
|
PortableWorkerSourceStage,
|
||||||
inspect_runtime_candidate,
|
inspect_runtime_candidate,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.recorded_progress import report_recorded_progress
|
||||||
from k1link.observatory.worker_agent import ObservatoryWorkerExecutorRegistration
|
from k1link.observatory.worker_agent import ObservatoryWorkerExecutorRegistration
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -467,7 +468,9 @@ class InstalledLabPackageProfileRunner:
|
|||||||
}
|
}
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
for container in _topological_containers(self.package):
|
containers = _topological_containers(self.package)
|
||||||
|
report_recorded_progress("computing", 0, len(containers), "steps")
|
||||||
|
for step_index, container in enumerate(containers):
|
||||||
container_output_root = output_root
|
container_output_root = output_root
|
||||||
if container.role == "step":
|
if container.role == "step":
|
||||||
container_output_root = steps_root / container.container_id
|
container_output_root = steps_root / container.container_id
|
||||||
@@ -483,6 +486,8 @@ class InstalledLabPackageProfileRunner:
|
|||||||
name_token=attempt_token,
|
name_token=attempt_token,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
report_recorded_progress("computing", step_index + 1, len(containers), "steps")
|
||||||
|
report_recorded_progress("result-assembly", unit="steps")
|
||||||
return _read_result_draft(
|
return _read_result_draft(
|
||||||
output_root,
|
output_root,
|
||||||
plan=plan,
|
plan=plan,
|
||||||
|
|||||||
@@ -19,7 +19,9 @@ import shutil
|
|||||||
import stat
|
import stat
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import time
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
|
from contextlib import suppress
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Final, Protocol, cast
|
from typing import Final, Protocol, cast
|
||||||
@@ -36,6 +38,7 @@ from k1link.observatory.m49_portable_source import (
|
|||||||
materialize_m49_portable_source_from_worker_stage,
|
materialize_m49_portable_source_from_worker_stage,
|
||||||
validate_m49_portable_source_stage_binding,
|
validate_m49_portable_source_stage_binding,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.m49_timing_progress import M49TimingProgress
|
||||||
from k1link.observatory.portable_result_contract import (
|
from k1link.observatory.portable_result_contract import (
|
||||||
OBSERVATION_ONLY_AUTHORITY,
|
OBSERVATION_ONLY_AUTHORITY,
|
||||||
canonical_json,
|
canonical_json,
|
||||||
@@ -53,6 +56,7 @@ from k1link.observatory.portable_worker_runtime import (
|
|||||||
PortableWorkerSourceMaterializer,
|
PortableWorkerSourceMaterializer,
|
||||||
PortableWorkerSourceStage,
|
PortableWorkerSourceStage,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.recorded_progress import report_recorded_progress
|
||||||
from k1link.observatory.worker_agent import SealedObservatoryRecordedJob
|
from k1link.observatory.worker_agent import SealedObservatoryRecordedJob
|
||||||
|
|
||||||
M49_COMPILED_RUNNER_BUILD_SCHEMA: Final = "missioncore.m49-tgs-portable-compiled-runner-build/v1"
|
M49_COMPILED_RUNNER_BUILD_SCHEMA: Final = "missioncore.m49-tgs-portable-compiled-runner-build/v1"
|
||||||
@@ -257,6 +261,7 @@ class M49PortableProfileRunnerAdapter:
|
|||||||
try:
|
try:
|
||||||
output = workspace / "outputs"
|
output = workspace / "outputs"
|
||||||
timing = workspace / "timing.tsv"
|
timing = workspace / "timing.tsv"
|
||||||
|
report_recorded_progress("computing", 0, stage.timeline_frame_count)
|
||||||
invoker = self.invoker or _invoke_exact_runner
|
invoker = self.invoker or _invoke_exact_runner
|
||||||
invoker(
|
invoker(
|
||||||
binary=self.installation.runner_binary_path,
|
binary=self.installation.runner_binary_path,
|
||||||
@@ -267,6 +272,7 @@ class M49PortableProfileRunnerAdapter:
|
|||||||
workspace=workspace,
|
workspace=workspace,
|
||||||
timeout_seconds=self.installation.timeout_seconds,
|
timeout_seconds=self.installation.timeout_seconds,
|
||||||
)
|
)
|
||||||
|
report_recorded_progress("result-assembly", unit="steps")
|
||||||
package = assemble_m49_portable_result(
|
package = assemble_m49_portable_result(
|
||||||
source_stage=stage,
|
source_stage=stage,
|
||||||
runner_output_root=output,
|
runner_output_root=output,
|
||||||
@@ -335,19 +341,32 @@ def _invoke_exact_runner(
|
|||||||
stderr = workspace / "runner.stderr.log"
|
stderr = workspace / "runner.stderr.log"
|
||||||
try:
|
try:
|
||||||
with stdout.open("xb") as stdout_stream, stderr.open("xb") as stderr_stream:
|
with stdout.open("xb") as stdout_stream, stderr.open("xb") as stderr_stream:
|
||||||
completed = subprocess.run(
|
started = time.monotonic()
|
||||||
|
progress = M49TimingProgress(timing)
|
||||||
|
process = subprocess.Popen(
|
||||||
[str(binary), str(sequence), str(schedule), str(output), str(timing)],
|
[str(binary), str(sequence), str(schedule), str(output), str(timing)],
|
||||||
cwd=workspace,
|
cwd=workspace,
|
||||||
env={"LANG": "C", "LC_ALL": "C", "TZ": "UTC"},
|
env={"LANG": "C", "LC_ALL": "C", "TZ": "UTC"},
|
||||||
stdin=subprocess.DEVNULL,
|
stdin=subprocess.DEVNULL,
|
||||||
stdout=stdout_stream,
|
stdout=stdout_stream,
|
||||||
stderr=stderr_stream,
|
stderr=stderr_stream,
|
||||||
check=False,
|
|
||||||
timeout=timeout_seconds,
|
|
||||||
)
|
)
|
||||||
|
try:
|
||||||
|
while process.poll() is None:
|
||||||
|
report_recorded_progress("computing", progress.poll())
|
||||||
|
remaining = timeout_seconds - (time.monotonic() - started)
|
||||||
|
if remaining <= 0:
|
||||||
|
raise subprocess.TimeoutExpired(str(binary), timeout_seconds)
|
||||||
|
with suppress(subprocess.TimeoutExpired):
|
||||||
|
process.wait(timeout=min(0.5, remaining))
|
||||||
|
report_recorded_progress("computing", progress.poll())
|
||||||
|
finally:
|
||||||
|
if process.poll() is None:
|
||||||
|
process.kill()
|
||||||
|
process.wait()
|
||||||
except (OSError, subprocess.SubprocessError) as exc:
|
except (OSError, subprocess.SubprocessError) as exc:
|
||||||
raise M49PortableExecutorError("portable M4.9 runner invocation failed") from exc
|
raise M49PortableExecutorError("portable M4.9 runner invocation failed") from exc
|
||||||
if completed.returncode != 0:
|
if process.returncode != 0:
|
||||||
raise M49PortableExecutorError("portable M4.9 runner rejected its exact source stage")
|
raise M49PortableExecutorError("portable M4.9 runner rejected its exact source stage")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ import numpy.typing as npt
|
|||||||
|
|
||||||
from k1link.compute.lidar_replay import LidarReplayPackV2, build_lidar_replay_pack_v2
|
from k1link.compute.lidar_replay import LidarReplayPackV2, build_lidar_replay_pack_v2
|
||||||
from k1link.observatory.portable_result_contract import canonical_json
|
from k1link.observatory.portable_result_contract import canonical_json
|
||||||
|
from k1link.observatory.recorded_progress import report_recorded_progress
|
||||||
from k1link.observatory.source_admission import (
|
from k1link.observatory.source_admission import (
|
||||||
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||||
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||||
@@ -175,6 +176,7 @@ def materialize_m49_portable_source_from_worker_stage(
|
|||||||
or worker_stage.source_adapter_sha256 != job.source_adapter_sha256
|
or worker_stage.source_adapter_sha256 != job.source_adapter_sha256
|
||||||
):
|
):
|
||||||
raise M49PortableSourceError("Worker source stage belongs to another job")
|
raise M49PortableSourceError("Worker source stage belongs to another job")
|
||||||
|
report_recorded_progress("source-preparation")
|
||||||
root = _safe_directory(worker_stage.root, "Worker source stage")
|
root = _safe_directory(worker_stage.root, "Worker source stage")
|
||||||
manifest_payload, manifest = _read_canonical_document(
|
manifest_payload, manifest = _read_canonical_document(
|
||||||
root / "materialization-manifest.json",
|
root / "materialization-manifest.json",
|
||||||
@@ -487,6 +489,7 @@ def _materialize_stage(
|
|||||||
index_rows: list[dict[str, object]] = []
|
index_rows: list[dict[str, object]] = []
|
||||||
available_slot = 0
|
available_slot = 0
|
||||||
sequence_logical = hashlib.sha256()
|
sequence_logical = hashlib.sha256()
|
||||||
|
report_recorded_progress("source-preparation", 0, len(anchors))
|
||||||
for anchor in anchors:
|
for anchor in anchors:
|
||||||
point_index = int(
|
point_index = int(
|
||||||
np.searchsorted(point_times, anchor.session_seconds, side="right") - 1
|
np.searchsorted(point_times, anchor.session_seconds, side="right") - 1
|
||||||
@@ -534,6 +537,9 @@ def _materialize_stage(
|
|||||||
f"{anchor.timeline_frame_index}\t{anchor.source_frame_index}"
|
f"{anchor.timeline_frame_index}\t{anchor.source_frame_index}"
|
||||||
f"\t{anchor.session_seconds:.9f}\t-1\t0"
|
f"\t{anchor.session_seconds:.9f}\t-1\t0"
|
||||||
)
|
)
|
||||||
|
report_recorded_progress(
|
||||||
|
"source-preparation", anchor.timeline_frame_index + 1, len(anchors),
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
start_seconds = anchor.session_seconds - profile.history_seconds
|
start_seconds = anchor.session_seconds - profile.history_seconds
|
||||||
@@ -593,6 +599,9 @@ def _materialize_stage(
|
|||||||
f"\t{anchor.session_seconds:.9f}\t{available_slot}\t{native.shape[0]}"
|
f"\t{anchor.session_seconds:.9f}\t{available_slot}\t{native.shape[0]}"
|
||||||
)
|
)
|
||||||
available_slot += 1
|
available_slot += 1
|
||||||
|
report_recorded_progress(
|
||||||
|
"source-preparation", anchor.timeline_frame_index + 1, len(anchors),
|
||||||
|
)
|
||||||
|
|
||||||
if available_slot < 1:
|
if available_slot < 1:
|
||||||
raise M49PortableSourceError("portable K1 source has no admissible LiDAR frames")
|
raise M49PortableSourceError("portable K1 source has no admissible LiDAR frames")
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
"""Incremental observation of flushed TGS timing rows, not result validation."""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
class M49TimingProgress:
|
||||||
|
def __init__(self, path: Path) -> None:
|
||||||
|
self.path = path
|
||||||
|
self.offset = 0
|
||||||
|
self.pending = b""
|
||||||
|
self.header = False
|
||||||
|
self.completed = 0
|
||||||
|
self.invalid = False
|
||||||
|
|
||||||
|
def poll(self) -> int:
|
||||||
|
if self.invalid:
|
||||||
|
return self.completed
|
||||||
|
try:
|
||||||
|
with self.path.open("rb") as stream:
|
||||||
|
stream.seek(self.offset)
|
||||||
|
data = stream.read(64 * 1024)
|
||||||
|
self.offset += len(data)
|
||||||
|
except OSError:
|
||||||
|
return self.completed
|
||||||
|
lines = (self.pending + data).split(b"\n")
|
||||||
|
self.pending = lines.pop()
|
||||||
|
if len(self.pending) > 2048:
|
||||||
|
self.invalid = True
|
||||||
|
self.pending = b""
|
||||||
|
return self.completed
|
||||||
|
for line in lines:
|
||||||
|
if not self.header:
|
||||||
|
self.header = line.startswith(b"timeline_frame_index\tsource_frame_index\t")
|
||||||
|
if not self.header:
|
||||||
|
self.invalid = True
|
||||||
|
break
|
||||||
|
continue
|
||||||
|
fields = line.split(b"\t")
|
||||||
|
if len(fields) != 10 or fields[0] != str(self.completed).encode():
|
||||||
|
self.invalid = True
|
||||||
|
break
|
||||||
|
self.completed += 1
|
||||||
|
return self.completed
|
||||||
@@ -27,6 +27,7 @@ from k1link.observatory.portable_run_definitions import (
|
|||||||
PortableRunDefinitionRegistry,
|
PortableRunDefinitionRegistry,
|
||||||
canonical_sha256,
|
canonical_sha256,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.recorded_progress import report_recorded_progress
|
||||||
from k1link.observatory.worker_agent import (
|
from k1link.observatory.worker_agent import (
|
||||||
ObservatoryWorkerExecutionResult,
|
ObservatoryWorkerExecutionResult,
|
||||||
ObservatoryWorkerExecutorIdentity,
|
ObservatoryWorkerExecutorIdentity,
|
||||||
@@ -732,6 +733,7 @@ class PortableWorkerExecutorAdapter:
|
|||||||
job: SealedObservatoryRecordedJob,
|
job: SealedObservatoryRecordedJob,
|
||||||
) -> ObservatoryWorkerExecutionResult:
|
) -> ObservatoryWorkerExecutionResult:
|
||||||
self._verify_job(job)
|
self._verify_job(job)
|
||||||
|
report_recorded_progress("source-transfer", unit="members")
|
||||||
source = self.source_materializer.materialize(job)
|
source = self.source_materializer.materialize(job)
|
||||||
if (
|
if (
|
||||||
source.source_bundle_sha256 != job.source_bundle_sha256
|
source.source_bundle_sha256 != job.source_bundle_sha256
|
||||||
@@ -764,11 +766,13 @@ class PortableWorkerExecutorAdapter:
|
|||||||
result_contract_sha256=self.candidate.result_contract_sha256,
|
result_contract_sha256=self.candidate.result_contract_sha256,
|
||||||
phases=tuple(phase.phase_id for phase in self.candidate.phases),
|
phases=tuple(phase.phase_id for phase in self.candidate.phases),
|
||||||
)
|
)
|
||||||
|
report_recorded_progress("computing", unit="steps")
|
||||||
draft = self.runner.run(plan, source)
|
draft = self.runner.run(plan, source)
|
||||||
if draft.result_contract_sha256 != self.candidate.result_contract_sha256:
|
if draft.result_contract_sha256 != self.candidate.result_contract_sha256:
|
||||||
raise PortableWorkerRuntimeJobRejectedError(
|
raise PortableWorkerRuntimeJobRejectedError(
|
||||||
"runtime result uses another result contract"
|
"runtime result uses another result contract"
|
||||||
)
|
)
|
||||||
|
report_recorded_progress("result-transfer", unit="members")
|
||||||
published = self.publisher.publish(job, draft)
|
published = self.publisher.publish(job, draft)
|
||||||
if (
|
if (
|
||||||
published.result_id != draft.result_id
|
published.result_id != draft.result_id
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ from typing import Final, Literal
|
|||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from k1link.artifacts import utc_now_iso
|
from k1link.artifacts import utc_now_iso
|
||||||
|
from k1link.observatory.recorded_progress import RecordedProgress
|
||||||
|
|
||||||
OBSERVATORY_RECORDED_JOB_SCHEMA: Final = "missioncore.observatory-recorded-job/v1"
|
OBSERVATORY_RECORDED_JOB_SCHEMA: Final = "missioncore.observatory-recorded-job/v1"
|
||||||
OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA: Final = "missioncore.observatory-recorded-job-request/v1"
|
OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA: Final = "missioncore.observatory-recorded-job-request/v1"
|
||||||
@@ -159,6 +160,12 @@ CREATE TABLE IF NOT EXISTS observatory_recorded_jobs (
|
|||||||
CREATE INDEX IF NOT EXISTS observatory_recorded_jobs_queue_order
|
CREATE INDEX IF NOT EXISTS observatory_recorded_jobs_queue_order
|
||||||
ON observatory_recorded_jobs (state, priority_rank, created_at_utc, job_id);
|
ON observatory_recorded_jobs (state, priority_rank, created_at_utc, job_id);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS observatory_recorded_progress (
|
||||||
|
job_id TEXT PRIMARY KEY REFERENCES observatory_recorded_jobs(job_id),
|
||||||
|
snapshot_json TEXT NOT NULL CHECK (length(snapshot_json) <= 2048),
|
||||||
|
received_at_utc TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS observatory_recorded_claim_receipts (
|
CREATE TABLE IF NOT EXISTS observatory_recorded_claim_receipts (
|
||||||
claim_request_id TEXT PRIMARY KEY,
|
claim_request_id TEXT PRIMARY KEY,
|
||||||
request_sha256 TEXT NOT NULL,
|
request_sha256 TEXT NOT NULL,
|
||||||
@@ -1614,6 +1621,73 @@ class ObservatoryRecordedJobQueue:
|
|||||||
)
|
)
|
||||||
return self._get_job(connection, job_id)
|
return self._get_job(connection, job_id)
|
||||||
|
|
||||||
|
def report_progress(
|
||||||
|
self, job_id: str, *, claim_token: str, claimant_id: str,
|
||||||
|
progress: RecordedProgress,
|
||||||
|
) -> None:
|
||||||
|
"""Replace one small observation, fenced in the ownership transaction."""
|
||||||
|
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||||
|
_validate_pattern(claim_token, _TOKEN, "claim token")
|
||||||
|
payload = progress.model_dump_json()
|
||||||
|
with self._transaction() as connection:
|
||||||
|
job = self._get_job(connection, job_id)
|
||||||
|
now = self._timestamp()
|
||||||
|
self._require_active_claim(job, claim_token, now=now)
|
||||||
|
if (
|
||||||
|
job.claim_generation != progress.claim_generation
|
||||||
|
or job.active_claimant_id != claimant_id
|
||||||
|
):
|
||||||
|
raise ObservatoryRecordedQueueStaleClaimError("progress owner is stale")
|
||||||
|
if job.state != "running":
|
||||||
|
raise ObservatoryRecordedQueueConflictError("progress requires running execution")
|
||||||
|
row = connection.execute(
|
||||||
|
"SELECT snapshot_json FROM observatory_recorded_progress WHERE job_id = ?",
|
||||||
|
(job_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row is not None:
|
||||||
|
previous = RecordedProgress.model_validate_json(row["snapshot_json"])
|
||||||
|
if previous.claim_generation == progress.claim_generation:
|
||||||
|
if previous == progress:
|
||||||
|
return
|
||||||
|
if not progress.follows(previous):
|
||||||
|
raise ObservatoryRecordedQueueConflictError("progress snapshot regressed")
|
||||||
|
connection.execute(
|
||||||
|
"INSERT INTO observatory_recorded_progress VALUES (?, ?, ?) "
|
||||||
|
"ON CONFLICT(job_id) DO UPDATE SET snapshot_json = excluded.snapshot_json, "
|
||||||
|
"received_at_utc = excluded.received_at_utc",
|
||||||
|
(job_id, payload, now),
|
||||||
|
)
|
||||||
|
|
||||||
|
def progress(self, job_id: str) -> dict[str, object]:
|
||||||
|
"""Read-only projection; never claims, renews, publishes or computes."""
|
||||||
|
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||||
|
with self._read_connection() as connection:
|
||||||
|
job = self._get_job(connection, job_id)
|
||||||
|
row = connection.execute(
|
||||||
|
"SELECT snapshot_json, received_at_utc FROM observatory_recorded_progress "
|
||||||
|
"WHERE job_id = ?", (job_id,),
|
||||||
|
).fetchone()
|
||||||
|
progress = None if row is None else RecordedProgress.model_validate_json(
|
||||||
|
row["snapshot_json"]
|
||||||
|
)
|
||||||
|
if progress is not None and progress.claim_generation != job.claim_generation:
|
||||||
|
progress = None
|
||||||
|
return {
|
||||||
|
"schema_version": "missioncore.observatory-recorded-progress-view/v1",
|
||||||
|
"job_id": job.job_id,
|
||||||
|
"source_session_id": job.source_session_id,
|
||||||
|
"setup_id": job.setup_id,
|
||||||
|
"definition_sha256": job.definition_sha256,
|
||||||
|
"claim_generation": job.claim_generation,
|
||||||
|
"state": job.state,
|
||||||
|
"received_at_utc": None if progress is None else row["received_at_utc"],
|
||||||
|
"age_seconds": None if progress is None else max(0.0, (
|
||||||
|
_parse_timestamp(self._timestamp(), "clock")
|
||||||
|
- _parse_timestamp(row["received_at_utc"], "progress receipt")
|
||||||
|
).total_seconds()),
|
||||||
|
"progress": None if progress is None else progress.model_dump(mode="json"),
|
||||||
|
}
|
||||||
|
|
||||||
def authorize_claim_access(
|
def authorize_claim_access(
|
||||||
self,
|
self,
|
||||||
job_id: str,
|
job_id: str,
|
||||||
@@ -2816,6 +2890,7 @@ class ObservatoryRecordedJobQueue:
|
|||||||
|
|
||||||
def _validate_schema(self, connection: sqlite3.Connection) -> None:
|
def _validate_schema(self, connection: sqlite3.Connection) -> None:
|
||||||
expected = {
|
expected = {
|
||||||
|
"observatory_recorded_progress": 3,
|
||||||
"observatory_recorded_jobs": 50,
|
"observatory_recorded_jobs": 50,
|
||||||
"observatory_recorded_claim_receipts": 6,
|
"observatory_recorded_claim_receipts": 6,
|
||||||
"observatory_recorded_claim_grants_v3": 6,
|
"observatory_recorded_claim_grants_v3": 6,
|
||||||
|
|||||||
@@ -0,0 +1,166 @@
|
|||||||
|
"""Bounded, attempt-scoped observation of recorded execution, never authority.
|
||||||
|
|
||||||
|
The execution thread owns counters; a separate bounded sender samples them.
|
||||||
|
Progress loss cannot cancel compute or renew a claim. No frame event journal,
|
||||||
|
estimated completion, filesystem paths, or model-dependent UI contract.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import math
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from collections.abc import Callable, Iterator
|
||||||
|
from contextlib import contextmanager, suppress
|
||||||
|
from contextvars import ContextVar
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||||
|
|
||||||
|
type ProgressPhase = Literal[
|
||||||
|
"source-transfer",
|
||||||
|
"source-preparation",
|
||||||
|
"computing",
|
||||||
|
"result-assembly",
|
||||||
|
"result-transfer",
|
||||||
|
]
|
||||||
|
type ProgressUnit = Literal["frames", "members", "steps"]
|
||||||
|
|
||||||
|
|
||||||
|
class RecordedProgress(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
|
||||||
|
|
||||||
|
schema_version: Literal["missioncore.observatory-recorded-progress/v1"] = (
|
||||||
|
"missioncore.observatory-recorded-progress/v1"
|
||||||
|
)
|
||||||
|
claim_generation: int = Field(ge=1, le=2**53 - 1)
|
||||||
|
sequence: int = Field(ge=1, le=2**53 - 1)
|
||||||
|
phase_index: int = Field(ge=0, le=2**53 - 1)
|
||||||
|
phase: ProgressPhase
|
||||||
|
unit: ProgressUnit
|
||||||
|
completed: int = Field(ge=0, le=2**53 - 1)
|
||||||
|
total: int | None = Field(default=None, ge=1, le=2**53 - 1)
|
||||||
|
elapsed_seconds: float = Field(ge=0, allow_inf_nan=False)
|
||||||
|
phase_elapsed_seconds: float = Field(ge=0, allow_inf_nan=False)
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_counts(self) -> RecordedProgress:
|
||||||
|
if self.total is not None and self.completed > self.total:
|
||||||
|
raise ValueError("completed exceeds the observed total")
|
||||||
|
if self.phase_elapsed_seconds > self.elapsed_seconds:
|
||||||
|
raise ValueError("phase duration exceeds execution duration")
|
||||||
|
return self
|
||||||
|
|
||||||
|
def follows(self, previous: RecordedProgress) -> bool:
|
||||||
|
if self.claim_generation != previous.claim_generation:
|
||||||
|
return False
|
||||||
|
if self.sequence <= previous.sequence or self.phase_index < previous.phase_index:
|
||||||
|
return False
|
||||||
|
if self.elapsed_seconds < previous.elapsed_seconds:
|
||||||
|
return False
|
||||||
|
if self.phase_index != previous.phase_index:
|
||||||
|
return True
|
||||||
|
return (
|
||||||
|
self.phase == previous.phase
|
||||||
|
and self.unit == previous.unit
|
||||||
|
and self.completed >= previous.completed
|
||||||
|
and (previous.total is None or self.total == previous.total)
|
||||||
|
and self.phase_elapsed_seconds >= previous.phase_elapsed_seconds
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class RecordedProgressTracker:
|
||||||
|
def __init__(self, generation: int, *, clock: Callable[[], float] = time.monotonic):
|
||||||
|
self.generation = generation
|
||||||
|
self.clock = clock
|
||||||
|
self.started = clock()
|
||||||
|
self.phase_started = self.started
|
||||||
|
self.lock = threading.Lock()
|
||||||
|
self.phase: ProgressPhase = "source-transfer"
|
||||||
|
self.unit: ProgressUnit = "members"
|
||||||
|
self.completed = 0
|
||||||
|
self.total: int | None = None
|
||||||
|
self.phase_index = 0
|
||||||
|
self.sequence = 0
|
||||||
|
|
||||||
|
def update(
|
||||||
|
self,
|
||||||
|
phase: ProgressPhase,
|
||||||
|
completed: int = 0,
|
||||||
|
total: int | None = None,
|
||||||
|
unit: ProgressUnit = "frames",
|
||||||
|
) -> None:
|
||||||
|
with self.lock:
|
||||||
|
if phase != self.phase or unit != self.unit:
|
||||||
|
self.phase_index += 1
|
||||||
|
self.phase_started = self.clock()
|
||||||
|
elif completed < self.completed:
|
||||||
|
raise ValueError("progress counter moved backwards within one phase")
|
||||||
|
elif total is None:
|
||||||
|
total = self.total
|
||||||
|
self.phase, self.unit = phase, unit
|
||||||
|
self.completed, self.total = completed, total
|
||||||
|
|
||||||
|
def sample(self) -> RecordedProgress:
|
||||||
|
with self.lock:
|
||||||
|
now = self.clock()
|
||||||
|
self.sequence += 1
|
||||||
|
return RecordedProgress(
|
||||||
|
claim_generation=self.generation,
|
||||||
|
sequence=self.sequence,
|
||||||
|
phase_index=self.phase_index,
|
||||||
|
phase=self.phase,
|
||||||
|
unit=self.unit,
|
||||||
|
completed=self.completed,
|
||||||
|
total=self.total,
|
||||||
|
elapsed_seconds=max(0.0, now - self.started),
|
||||||
|
phase_elapsed_seconds=max(0.0, now - self.phase_started),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_CURRENT: ContextVar[RecordedProgressTracker | None] = ContextVar(
|
||||||
|
"observatory_recorded_progress",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def report_recorded_progress(
|
||||||
|
phase: ProgressPhase,
|
||||||
|
completed: int = 0,
|
||||||
|
total: int | None = None,
|
||||||
|
unit: ProgressUnit = "frames",
|
||||||
|
) -> None:
|
||||||
|
tracker = _CURRENT.get()
|
||||||
|
if tracker is not None:
|
||||||
|
tracker.update(phase, completed, total, unit)
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def observe_recorded_execution(
|
||||||
|
generation: int,
|
||||||
|
send: Callable[[RecordedProgress], None],
|
||||||
|
*,
|
||||||
|
interval_seconds: float = 1.0,
|
||||||
|
) -> Iterator[RecordedProgressTracker]:
|
||||||
|
if not math.isfinite(interval_seconds) or not 0.01 <= interval_seconds <= 30:
|
||||||
|
raise ValueError("progress sampling interval is invalid")
|
||||||
|
tracker = RecordedProgressTracker(generation)
|
||||||
|
stop = threading.Event()
|
||||||
|
|
||||||
|
def pump() -> None:
|
||||||
|
while not stop.is_set():
|
||||||
|
# Progress is secondary observation, not lease or result authority.
|
||||||
|
with suppress(Exception):
|
||||||
|
send(tracker.sample())
|
||||||
|
stop.wait(interval_seconds)
|
||||||
|
|
||||||
|
token = _CURRENT.set(tracker)
|
||||||
|
thread = threading.Thread(target=pump, name="observatory-progress", daemon=True)
|
||||||
|
thread.start()
|
||||||
|
try:
|
||||||
|
yield tracker
|
||||||
|
finally:
|
||||||
|
_CURRENT.reset(token)
|
||||||
|
stop.set()
|
||||||
|
# The production sender reads no body and uses 2-second I/O timeouts.
|
||||||
|
thread.join(timeout=3.0)
|
||||||
@@ -18,6 +18,7 @@ import json
|
|||||||
import re
|
import re
|
||||||
import threading
|
import threading
|
||||||
from collections.abc import Callable, Mapping
|
from collections.abc import Callable, Mapping
|
||||||
|
from contextlib import nullcontext
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Annotated, Final, Literal, Protocol
|
from typing import Annotated, Final, Literal, Protocol
|
||||||
@@ -31,6 +32,7 @@ from k1link.observatory.recorded_jobs import (
|
|||||||
OBSERVATORY_RECORDED_JOB_SCHEMA,
|
OBSERVATORY_RECORDED_JOB_SCHEMA,
|
||||||
RecordedExecutorIdentity,
|
RecordedExecutorIdentity,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.recorded_progress import observe_recorded_execution
|
||||||
|
|
||||||
WORKER_006_CONTOUR_ID: Final = "worker-006"
|
WORKER_006_CONTOUR_ID: Final = "worker-006"
|
||||||
MAX_EXECUTOR_FAILURE_MESSAGE_LENGTH: Final = 512
|
MAX_EXECUTOR_FAILURE_MESSAGE_LENGTH: Final = 512
|
||||||
@@ -498,7 +500,18 @@ class ObservatoryWorkerAgent:
|
|||||||
heartbeat.start()
|
heartbeat.start()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = adapter.execute(active_job)
|
send_progress = getattr(self._transport, "report_progress", None)
|
||||||
|
observation = (
|
||||||
|
observe_recorded_execution(
|
||||||
|
active_job.claim_generation,
|
||||||
|
lambda snapshot: send_progress(
|
||||||
|
job_id=active_job.job_id, claim_token=claim.claim_token,
|
||||||
|
progress=snapshot,
|
||||||
|
),
|
||||||
|
) if callable(send_progress) else nullcontext()
|
||||||
|
)
|
||||||
|
with observation:
|
||||||
|
result = adapter.execute(active_job)
|
||||||
if not isinstance(result, ObservatoryWorkerExecutionResult):
|
if not isinstance(result, ObservatoryWorkerExecutionResult):
|
||||||
raise TypeError("executor returned an unknown result contract")
|
raise TypeError("executor returned an unknown result contract")
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ from k1link.observatory.portable_worker_runtime import (
|
|||||||
PortableWorkerResultDraft,
|
PortableWorkerResultDraft,
|
||||||
PortableWorkerSourceStage,
|
PortableWorkerSourceStage,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.recorded_progress import RecordedProgress, report_recorded_progress
|
||||||
from k1link.observatory.source_admission import (
|
from k1link.observatory.source_admission import (
|
||||||
PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID,
|
PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID,
|
||||||
PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE,
|
PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE,
|
||||||
@@ -279,6 +280,23 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def report_progress(
|
||||||
|
self, *, job_id: str, claim_token: str, progress: RecordedProgress,
|
||||||
|
) -> None:
|
||||||
|
context = self._require_cached_claim(job_id, claim_token)
|
||||||
|
if progress.claim_generation != context.claim_generation:
|
||||||
|
raise ObservatoryWorkerHttpError("progress generation changed")
|
||||||
|
# No response body is needed. Bounded I/O cannot stall the execution thread.
|
||||||
|
with self._client.stream(
|
||||||
|
"POST", self._job_path(job_id, "progress"),
|
||||||
|
json={
|
||||||
|
"schema_version": "missioncore.observatory-worker-progress-request/v1",
|
||||||
|
"claim_token": claim_token, "progress": progress.model_dump(mode="json"),
|
||||||
|
}, timeout=httpx.Timeout(2.0),
|
||||||
|
) as response:
|
||||||
|
if response.status_code != 204:
|
||||||
|
raise ObservatoryWorkerHttpError("progress observation was not accepted")
|
||||||
|
|
||||||
def succeed(
|
def succeed(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -336,6 +354,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
|||||||
headers=headers,
|
headers=headers,
|
||||||
)
|
)
|
||||||
members = _source_members(manifest, job)
|
members = _source_members(manifest, job)
|
||||||
|
report_recorded_progress("source-transfer", 0, len(members), "members")
|
||||||
root = _secure_directory(
|
root = _secure_directory(
|
||||||
self._work_root
|
self._work_root
|
||||||
/ "sources"
|
/ "sources"
|
||||||
@@ -368,10 +387,16 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
|||||||
members=camera_members,
|
members=camera_members,
|
||||||
layout=layout,
|
layout=layout,
|
||||||
)
|
)
|
||||||
|
completed_members = len(camera_members) if camera_epoch_ready else 0
|
||||||
|
report_recorded_progress("source-transfer", completed_members, len(members), "members")
|
||||||
for destination, member in destinations.items():
|
for destination, member in destinations.items():
|
||||||
if camera_epoch_ready and member.kind in {"camera-init", "camera-segment"}:
|
if camera_epoch_ready and member.kind in {"camera-init", "camera-segment"}:
|
||||||
continue
|
continue
|
||||||
if _matches_file(destination, member.sha256, member.byte_length):
|
if _matches_file(destination, member.sha256, member.byte_length):
|
||||||
|
completed_members += 1
|
||||||
|
report_recorded_progress(
|
||||||
|
"source-transfer", completed_members, len(members), "members",
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
self._download_member(
|
self._download_member(
|
||||||
job_id=job.job_id,
|
job_id=job.job_id,
|
||||||
@@ -379,6 +404,8 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
|||||||
member=member,
|
member=member,
|
||||||
destination=destination,
|
destination=destination,
|
||||||
)
|
)
|
||||||
|
completed_members += 1
|
||||||
|
report_recorded_progress("source-transfer", completed_members, len(members), "members")
|
||||||
manifest_path = root / "materialization-manifest.json"
|
manifest_path = root / "materialization-manifest.json"
|
||||||
_write_local_exact(manifest_path, canonical_json(manifest))
|
_write_local_exact(manifest_path, canonical_json(manifest))
|
||||||
return PortableWorkerSourceStage(
|
return PortableWorkerSourceStage(
|
||||||
@@ -440,7 +467,8 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
|||||||
or {member.role for member in upload_members} != set(artifacts)
|
or {member.role for member in upload_members} != set(artifacts)
|
||||||
):
|
):
|
||||||
raise ObservatoryWorkerHttpError("Worker result artifact roles are not unique")
|
raise ObservatoryWorkerHttpError("Worker result artifact roles are not unique")
|
||||||
for member in upload_members:
|
report_recorded_progress("result-transfer", 0, len(upload_members), "members")
|
||||||
|
for member_index, member in enumerate(upload_members):
|
||||||
artifact = artifacts.get(member.role)
|
artifact = artifacts.get(member.role)
|
||||||
expected_member_id = hashlib.sha256(
|
expected_member_id = hashlib.sha256(
|
||||||
canonical_json(
|
canonical_json(
|
||||||
@@ -461,6 +489,9 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
|||||||
"result upload plan differs from the local manifest"
|
"result upload plan differs from the local manifest"
|
||||||
)
|
)
|
||||||
if member.uploaded:
|
if member.uploaded:
|
||||||
|
report_recorded_progress(
|
||||||
|
"result-transfer", member_index + 1, len(upload_members), "members",
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
relative = relative_artifact_path(artifact.relative_path)
|
relative = relative_artifact_path(artifact.relative_path)
|
||||||
source = _confined_local_member(draft.root, relative.parts)
|
source = _confined_local_member(draft.root, relative.parts)
|
||||||
@@ -489,6 +520,9 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
|||||||
raise ObservatoryWorkerHttpError(
|
raise ObservatoryWorkerHttpError(
|
||||||
"result upload acknowledgement did not seal its member"
|
"result upload acknowledgement did not seal its member"
|
||||||
)
|
)
|
||||||
|
report_recorded_progress(
|
||||||
|
"result-transfer", member_index + 1, len(upload_members), "members",
|
||||||
|
)
|
||||||
receipt = self._required_json_request(
|
receipt = self._required_json_request(
|
||||||
"POST",
|
"POST",
|
||||||
self._job_path(
|
self._job_path(
|
||||||
|
|||||||
@@ -1151,6 +1151,17 @@ def build_observatory_router(
|
|||||||
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@router.get("/api/v1/observatory/runs/{job_id}/progress")
|
||||||
|
def get_observatory_progress(
|
||||||
|
job_id: str = ApiPath(pattern=r"^observatory-run-[a-f0-9]{32}$"),
|
||||||
|
) -> dict[str, object]:
|
||||||
|
try:
|
||||||
|
return recorded_job_queue.progress(job_id)
|
||||||
|
except ObservatoryRecordedQueueNotFoundError as exc:
|
||||||
|
raise HTTPException(status_code=404, detail="Расчёт не найден.") from exc
|
||||||
|
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||||
|
raise HTTPException(status_code=503, detail="Прогресс недоступен.") from exc
|
||||||
|
|
||||||
@router.get("/api/v1/observatory/runs/{job_id}")
|
@router.get("/api/v1/observatory/runs/{job_id}")
|
||||||
def get_observatory_recorded_run(
|
def get_observatory_recorded_run(
|
||||||
job_id: str = ApiPath(
|
job_id: str = ApiPath(
|
||||||
|
|||||||
@@ -51,6 +51,7 @@ from k1link.observatory.recorded_jobs import (
|
|||||||
ObservatoryRecordedQueueStaleClaimError,
|
ObservatoryRecordedQueueStaleClaimError,
|
||||||
RecordedExecutorIdentity,
|
RecordedExecutorIdentity,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.recorded_progress import RecordedProgress
|
||||||
|
|
||||||
OBSERVATORY_WORKER_CAPABILITY_CLAIM_REQUEST_SCHEMA: Final = (
|
OBSERVATORY_WORKER_CAPABILITY_CLAIM_REQUEST_SCHEMA: Final = (
|
||||||
"missioncore.observatory-worker-claim-request/v2"
|
"missioncore.observatory-worker-claim-request/v2"
|
||||||
@@ -194,6 +195,12 @@ class ObservatoryWorkerRenewRequest(_StrictWorkerRequest):
|
|||||||
heartbeat_sequence: int = Field(ge=1)
|
heartbeat_sequence: int = Field(ge=1)
|
||||||
|
|
||||||
|
|
||||||
|
class ObservatoryWorkerProgressRequest(_StrictWorkerRequest):
|
||||||
|
schema_version: Literal["missioncore.observatory-worker-progress-request/v1"]
|
||||||
|
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||||
|
progress: RecordedProgress
|
||||||
|
|
||||||
|
|
||||||
class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest):
|
class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest):
|
||||||
schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"]
|
schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"]
|
||||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||||
@@ -333,6 +340,17 @@ def build_observatory_worker_router(
|
|||||||
)
|
)
|
||||||
).as_dict()
|
).as_dict()
|
||||||
|
|
||||||
|
@router.post("/recorded-jobs/{job_id}/progress", status_code=204)
|
||||||
|
def report_job_progress(
|
||||||
|
request: ObservatoryWorkerProgressRequest,
|
||||||
|
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||||
|
) -> Response:
|
||||||
|
_queue_call(lambda: queue.report_progress(
|
||||||
|
job_id, claim_token=request.claim_token,
|
||||||
|
claimant_id=authentication.contour_id, progress=request.progress,
|
||||||
|
))
|
||||||
|
return Response(status_code=204)
|
||||||
|
|
||||||
@router.post("/recorded-jobs/{job_id}/checkpoint")
|
@router.post("/recorded-jobs/{job_id}/checkpoint")
|
||||||
def checkpoint_job(
|
def checkpoint_job(
|
||||||
request: ObservatoryWorkerCheckpointRequest,
|
request: ObservatoryWorkerCheckpointRequest,
|
||||||
|
|||||||
@@ -0,0 +1,339 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from pydantic import ValidationError
|
||||||
|
from test_observatory_recorded_jobs import _queue, _running_job
|
||||||
|
from test_observatory_worker_api import WORKER_HEADERS, _claim_request, _services
|
||||||
|
|
||||||
|
from k1link.observatory.m49_timing_progress import M49TimingProgress
|
||||||
|
from k1link.observatory.recorded_jobs import (
|
||||||
|
ObservatoryRecordedQueueConflictError,
|
||||||
|
ObservatoryRecordedQueueStaleClaimError,
|
||||||
|
)
|
||||||
|
from k1link.observatory.recorded_progress import (
|
||||||
|
RecordedProgress,
|
||||||
|
RecordedProgressTracker,
|
||||||
|
observe_recorded_execution,
|
||||||
|
report_recorded_progress,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _progress(**changes: object) -> RecordedProgress:
|
||||||
|
return RecordedProgress.model_validate(
|
||||||
|
{
|
||||||
|
"claim_generation": 1,
|
||||||
|
"sequence": 1,
|
||||||
|
"phase_index": 1,
|
||||||
|
"phase": "computing",
|
||||||
|
"unit": "frames",
|
||||||
|
"completed": 3,
|
||||||
|
"total": 10,
|
||||||
|
"elapsed_seconds": 3.0,
|
||||||
|
"phase_elapsed_seconds": 2.0,
|
||||||
|
**changes,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"change",
|
||||||
|
[
|
||||||
|
{"completed": True},
|
||||||
|
{"completed": 11},
|
||||||
|
{"total": 0},
|
||||||
|
{"phase": "done"},
|
||||||
|
{"elapsed_seconds": float("nan")},
|
||||||
|
{"phase_elapsed_seconds": 4.0},
|
||||||
|
{"sequence": 2**53},
|
||||||
|
{"path": "/secret"},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_progress_rejects_unbounded_or_invented_values(change: dict[str, object]) -> None:
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
_progress(**change)
|
||||||
|
|
||||||
|
|
||||||
|
def test_progress_is_one_durable_snapshot_not_job_identity_or_lease(tmp_path: Path) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
job, claim = _running_job(queue)
|
||||||
|
original = queue.get(job.job_id).as_dict()
|
||||||
|
assert queue.progress(job.job_id)["progress"] is None
|
||||||
|
for sequence in range(1, 101):
|
||||||
|
queue.report_progress(
|
||||||
|
job.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
claimant_id="recorded-worker",
|
||||||
|
progress=_progress(sequence=sequence),
|
||||||
|
)
|
||||||
|
assert queue.get(job.job_id).as_dict() == original
|
||||||
|
reopened = _queue(tmp_path)
|
||||||
|
view = reopened.progress(job.job_id)
|
||||||
|
assert view["definition_sha256"] == job.definition_sha256
|
||||||
|
assert view["claim_generation"] == 1
|
||||||
|
assert view["age_seconds"] == 0.0
|
||||||
|
assert view["progress"]["sequence"] == 100
|
||||||
|
with sqlite3.connect(queue.database_path) as connection:
|
||||||
|
assert (
|
||||||
|
connection.execute("SELECT COUNT(*) FROM observatory_recorded_progress").fetchone()[0]
|
||||||
|
== 1
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"change",
|
||||||
|
[
|
||||||
|
{"sequence": 1, "completed": 4},
|
||||||
|
{"sequence": 2, "completed": 2},
|
||||||
|
{"sequence": 2, "total": 12},
|
||||||
|
{"sequence": 2, "phase": "result-transfer"},
|
||||||
|
{"sequence": 2, "elapsed_seconds": 2.0},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_progress_retry_is_idempotent_and_regressions_rejected(tmp_path: Path, change) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
job, claim = _running_job(queue)
|
||||||
|
|
||||||
|
def send(value):
|
||||||
|
queue.report_progress(
|
||||||
|
job.job_id, claim_token=claim.claim_token, claimant_id="recorded-worker", progress=value
|
||||||
|
)
|
||||||
|
|
||||||
|
send(_progress())
|
||||||
|
send(_progress())
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueConflictError):
|
||||||
|
send(_progress(**change))
|
||||||
|
send(_progress(sequence=3, phase_index=2, phase="result-assembly", completed=0, total=None))
|
||||||
|
|
||||||
|
|
||||||
|
def test_progress_old_generation_cannot_write_or_project_as_current(tmp_path: Path) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
job, claim = _running_job(queue)
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueStaleClaimError):
|
||||||
|
queue.report_progress(
|
||||||
|
job.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
claimant_id="recorded-worker",
|
||||||
|
progress=_progress(claim_generation=2),
|
||||||
|
)
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueStaleClaimError):
|
||||||
|
queue.report_progress(
|
||||||
|
job.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
claimant_id="another-worker",
|
||||||
|
progress=_progress(),
|
||||||
|
)
|
||||||
|
# Synthetic old attempt retained on disk is not the new attempt's progress.
|
||||||
|
with sqlite3.connect(queue.database_path) as connection:
|
||||||
|
connection.execute(
|
||||||
|
"INSERT INTO observatory_recorded_progress VALUES (?, ?, ?)",
|
||||||
|
(job.job_id, _progress(claim_generation=2).model_dump_json(), job.created_at_utc),
|
||||||
|
)
|
||||||
|
assert queue.progress(job.job_id)["progress"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_progress_tracker_preserves_total_and_separates_phases() -> None:
|
||||||
|
tracker = RecordedProgressTracker(2, clock=lambda: 10.0)
|
||||||
|
tracker.update("computing", 0, 10)
|
||||||
|
first = tracker.sample()
|
||||||
|
tracker.update("computing", 2)
|
||||||
|
second = tracker.sample()
|
||||||
|
assert second.total == 10 and second.follows(first)
|
||||||
|
tracker.update("result-assembly", unit="steps")
|
||||||
|
third = tracker.sample()
|
||||||
|
assert third.completed == 0 and third.total is None and third.follows(second)
|
||||||
|
|
||||||
|
|
||||||
|
def test_m49_invoker_preserves_arguments_and_observes_real_output(tmp_path, monkeypatch):
|
||||||
|
from k1link.observatory import m49_portable_executor as executor
|
||||||
|
|
||||||
|
script = tmp_path / "synthetic.py"
|
||||||
|
script.write_text(
|
||||||
|
"import os, sys\n"
|
||||||
|
"from pathlib import Path\n"
|
||||||
|
"assert sys.argv[1] == 'schedule.tsv'\n"
|
||||||
|
"assert sys.argv[2] == 'outputs'\n"
|
||||||
|
"assert os.environ['LANG'] == 'C' and os.environ['TZ'] == 'UTC'\n"
|
||||||
|
"Path(sys.argv[3]).write_text('timeline_frame_index\\tsource_frame_index\\tother\\n'"
|
||||||
|
" + '0\\t0\\t0\\t1\\t0\\t12\\t10\\t2\\t1.2\\t1.3\\n')\n"
|
||||||
|
"print('synthetic runner completed')\n"
|
||||||
|
)
|
||||||
|
counters = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
executor, "report_recorded_progress", lambda phase, count: counters.append(count)
|
||||||
|
)
|
||||||
|
executor._invoke_exact_runner(
|
||||||
|
binary=Path(sys.executable),
|
||||||
|
sequence=script,
|
||||||
|
schedule=Path("schedule.tsv"),
|
||||||
|
output=Path("outputs"),
|
||||||
|
timing=tmp_path / "timing.tsv",
|
||||||
|
workspace=tmp_path,
|
||||||
|
timeout_seconds=5,
|
||||||
|
)
|
||||||
|
assert counters[-1] == 1
|
||||||
|
assert (tmp_path / "runner.stdout.log").read_text().strip() == "synthetic runner completed"
|
||||||
|
assert (tmp_path / "runner.stderr.log").read_bytes() == b""
|
||||||
|
|
||||||
|
|
||||||
|
def test_m49_invoker_timeout_reaps_only_its_child(tmp_path):
|
||||||
|
import os
|
||||||
|
|
||||||
|
from k1link.observatory.m49_portable_executor import (
|
||||||
|
M49PortableExecutorError,
|
||||||
|
_invoke_exact_runner,
|
||||||
|
)
|
||||||
|
|
||||||
|
script = tmp_path / "synthetic.py"
|
||||||
|
script.write_text(
|
||||||
|
"import os, time\nfrom pathlib import Path\n"
|
||||||
|
"Path('synthetic.pid').write_text(str(os.getpid()))\ntime.sleep(30)\n"
|
||||||
|
)
|
||||||
|
with pytest.raises(M49PortableExecutorError, match="invocation failed"):
|
||||||
|
_invoke_exact_runner(
|
||||||
|
binary=Path(sys.executable),
|
||||||
|
sequence=script,
|
||||||
|
schedule=tmp_path / "schedule.tsv",
|
||||||
|
output=tmp_path / "outputs",
|
||||||
|
timing=tmp_path / "timing.tsv",
|
||||||
|
workspace=tmp_path,
|
||||||
|
timeout_seconds=1,
|
||||||
|
)
|
||||||
|
pid = int((tmp_path / "synthetic.pid").read_text())
|
||||||
|
with pytest.raises(ProcessLookupError):
|
||||||
|
os.kill(pid, 0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_progress_sender_failure_does_not_escape_or_leave_a_thread() -> None:
|
||||||
|
sampled = threading.Event()
|
||||||
|
|
||||||
|
def unavailable(snapshot):
|
||||||
|
assert isinstance(snapshot, RecordedProgress)
|
||||||
|
sampled.set()
|
||||||
|
raise RuntimeError("synthetic observer connection loss")
|
||||||
|
|
||||||
|
with observe_recorded_execution(1, unavailable, interval_seconds=0.01) as tracker:
|
||||||
|
report_recorded_progress("computing", 5, 8)
|
||||||
|
assert sampled.wait(1)
|
||||||
|
assert tracker.sample().completed == 5
|
||||||
|
assert not any(thread.name == "observatory-progress" for thread in threading.enumerate())
|
||||||
|
report_recorded_progress("computing", 1, 2) # no leaked execution context
|
||||||
|
assert tracker.sample().completed == 5
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_keeps_compute_and_lease_when_progress_transport_fails(tmp_path: Path) -> None:
|
||||||
|
from test_observatory_worker_agent import (
|
||||||
|
BlockingExecutor,
|
||||||
|
FakeTransport,
|
||||||
|
_enqueue,
|
||||||
|
_heartbeat_agent,
|
||||||
|
)
|
||||||
|
from test_observatory_worker_agent import (
|
||||||
|
_queue as worker_queue,
|
||||||
|
)
|
||||||
|
|
||||||
|
queue = worker_queue(tmp_path)
|
||||||
|
job_id = _enqueue(queue)
|
||||||
|
attempted = threading.Event()
|
||||||
|
entered, release = threading.Event(), threading.Event()
|
||||||
|
|
||||||
|
class ObserverFailureTransport(FakeTransport):
|
||||||
|
def report_progress(self, **_kwargs):
|
||||||
|
attempted.set()
|
||||||
|
raise RuntimeError("observer unavailable")
|
||||||
|
|
||||||
|
agent = _heartbeat_agent(
|
||||||
|
ObserverFailureTransport(queue),
|
||||||
|
BlockingExecutor(entered=entered, release=release),
|
||||||
|
)
|
||||||
|
reports = []
|
||||||
|
thread = threading.Thread(target=lambda: reports.append(agent.run_once()))
|
||||||
|
thread.start()
|
||||||
|
assert entered.wait(2) and attempted.wait(2)
|
||||||
|
release.set()
|
||||||
|
thread.join(2)
|
||||||
|
assert not thread.is_alive()
|
||||||
|
assert reports[0].state == "succeeded"
|
||||||
|
assert queue.get(job_id).state == "succeeded"
|
||||||
|
|
||||||
|
|
||||||
|
def test_http_progress_uses_active_claim_and_short_bodyless_request(tmp_path: Path) -> None:
|
||||||
|
import httpx
|
||||||
|
from test_observatory_worker_http_transport import (
|
||||||
|
BEARER_TOKEN,
|
||||||
|
CLAIM_TOKEN,
|
||||||
|
JOB_ID,
|
||||||
|
_cache_claim,
|
||||||
|
_claim_response,
|
||||||
|
)
|
||||||
|
|
||||||
|
from k1link.observatory.worker_http_transport import ObservatoryWorkerHttpGateway
|
||||||
|
|
||||||
|
requests = []
|
||||||
|
|
||||||
|
def handle(request):
|
||||||
|
if request.url.path.endswith("/claims"):
|
||||||
|
return _claim_response()
|
||||||
|
requests.append(request)
|
||||||
|
assert request.url.path.endswith("/progress")
|
||||||
|
assert request.extensions["timeout"]["read"] == 2.0
|
||||||
|
assert json.loads(request.content)["claim_token"] == CLAIM_TOKEN
|
||||||
|
return httpx.Response(204)
|
||||||
|
|
||||||
|
with ObservatoryWorkerHttpGateway(
|
||||||
|
base_url="http://127.0.0.1:8000",
|
||||||
|
bearer_token=BEARER_TOKEN,
|
||||||
|
work_root=tmp_path,
|
||||||
|
transport=httpx.MockTransport(handle),
|
||||||
|
) as gateway:
|
||||||
|
_cache_claim(gateway)
|
||||||
|
gateway.report_progress(job_id=JOB_ID, claim_token=CLAIM_TOKEN, progress=_progress())
|
||||||
|
assert len(requests) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_tgs_progress_counts_only_complete_ordered_flushed_rows(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "timing.tsv"
|
||||||
|
observer = M49TimingProgress(path)
|
||||||
|
assert observer.poll() == 0
|
||||||
|
path.write_bytes(
|
||||||
|
b"timeline_frame_index\tsource_frame_index\trest\n0\t0\t0\t1\t0\t1\t1\t0\t1\t1\n1\t1"
|
||||||
|
)
|
||||||
|
assert observer.poll() == 1
|
||||||
|
with path.open("ab") as stream:
|
||||||
|
stream.write(b"\t1\t1\t1\t1\t1\t0\t1\t1\n")
|
||||||
|
assert observer.poll() == 2
|
||||||
|
with path.open("ab") as stream:
|
||||||
|
stream.write(b"4\t4\t4\t1\t4\t1\t1\t0\t1\t1\n")
|
||||||
|
assert observer.poll() == 2 and observer.invalid
|
||||||
|
|
||||||
|
|
||||||
|
def test_progress_api_requires_auth_exact_claim_and_strict_payload(tmp_path: Path) -> None:
|
||||||
|
from test_observatory_worker_api import _enqueue
|
||||||
|
|
||||||
|
client, queue = _services(tmp_path)
|
||||||
|
job_id = _enqueue(queue)
|
||||||
|
response = client.post(
|
||||||
|
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||||
|
headers=WORKER_HEADERS,
|
||||||
|
json=_claim_request("progress-claim"),
|
||||||
|
)
|
||||||
|
claim = response.json()
|
||||||
|
token = claim["claim_token"]
|
||||||
|
queue.start(job_id, claim_token=token)
|
||||||
|
path = f"/api/v1/worker/observatory/recorded-jobs/{job_id}/progress"
|
||||||
|
body = {
|
||||||
|
"schema_version": "missioncore.observatory-worker-progress-request/v1",
|
||||||
|
"claim_token": token,
|
||||||
|
"progress": _progress().model_dump(mode="json"),
|
||||||
|
}
|
||||||
|
assert client.post(path, json=body).status_code == 401
|
||||||
|
assert (
|
||||||
|
client.post(path, headers=WORKER_HEADERS, json={**body, "command": "no"}).status_code == 422
|
||||||
|
)
|
||||||
|
assert client.post(path, headers=WORKER_HEADERS, json=body).status_code == 204
|
||||||
|
assert queue.progress(job_id)["progress"] == json.loads(_progress().model_dump_json())
|
||||||
Reference in New Issue
Block a user