feat(observatory): add durable recorded compute queue
This commit is contained in:
DCCONSTRUCTIONS
@@ -61,8 +61,8 @@ export interface ObservatoryLaboratoryRunPreflight {
|
||||
readonly sourceSessionId: string;
|
||||
readonly setupId: string;
|
||||
readonly definitionSha256: string | null;
|
||||
readonly outcome: "existing" | "blocked";
|
||||
readonly submissionAllowed: false;
|
||||
readonly outcome: "existing" | "queueable" | "blocked";
|
||||
readonly submissionAllowed: boolean;
|
||||
readonly checks: readonly {
|
||||
readonly checkId: string;
|
||||
readonly outcome: "pass" | "fail" | "not-applicable";
|
||||
@@ -287,8 +287,15 @@ function decodePreflight(value: unknown): ObservatoryLaboratoryRunPreflight {
|
||||
sourceSessionId: text(row.source_session_id, "preflight source_session_id"),
|
||||
setupId: text(row.setup_id, "preflight setup_id"),
|
||||
definitionSha256: digest,
|
||||
outcome: oneOf(row.outcome, ["existing", "blocked"] as const, "preflight outcome"),
|
||||
submissionAllowed: exact(row.submission_allowed, false, "preflight submission_allowed"),
|
||||
outcome: oneOf(
|
||||
row.outcome,
|
||||
["existing", "queueable", "blocked"] as const,
|
||||
"preflight outcome",
|
||||
),
|
||||
submissionAllowed: boolean(
|
||||
row.submission_allowed,
|
||||
"preflight submission_allowed",
|
||||
),
|
||||
checks: array(row.checks, "preflight checks").map((item) => {
|
||||
const check = record(item, "preflight check");
|
||||
exactKeys(check, ["check_id", "message", "outcome", "reason_code"], "preflight check");
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
const SUBMIT_SCHEMA = "missioncore.observatory-recorded-run-submit/v1";
|
||||
const JOB_SCHEMA = "missioncore.observatory-recorded-job/v1";
|
||||
const JOB_LIST_SCHEMA = "missioncore.observatory-recorded-job-list/v1";
|
||||
const SHA256 = /^[a-f0-9]{64}$/;
|
||||
|
||||
export type ObservatoryRecordedJobState =
|
||||
| "accepted"
|
||||
| "queued"
|
||||
| "claimed"
|
||||
| "running"
|
||||
| "paused"
|
||||
| "preemption-pending"
|
||||
| "succeeded"
|
||||
| "failed"
|
||||
| "reconciliation-required";
|
||||
|
||||
export interface ObservatoryRecordedJob {
|
||||
readonly jobId: string;
|
||||
readonly idempotencyKey: string;
|
||||
readonly sourceSessionId: string;
|
||||
readonly setupId: string;
|
||||
readonly definitionSha256: string;
|
||||
readonly state: ObservatoryRecordedJobState;
|
||||
readonly restartFromZero: boolean;
|
||||
readonly resultId: string | null;
|
||||
readonly terminalMessage: string | null;
|
||||
readonly createdAtUtc: string;
|
||||
readonly updatedAtUtc: string;
|
||||
}
|
||||
|
||||
export type ObservatoryRecordedJobFetch = (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>;
|
||||
|
||||
export class ObservatoryRecordedJobContractError extends Error {
|
||||
readonly status: number | null;
|
||||
|
||||
constructor(message: string, status: number | null = null) {
|
||||
super(message);
|
||||
this.name = "ObservatoryRecordedJobContractError";
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchObservatoryRecordedJobs(
|
||||
sourceSessionId: string,
|
||||
setupId: string,
|
||||
{
|
||||
signal,
|
||||
fetcher = globalThis.fetch,
|
||||
}: {
|
||||
signal?: AbortSignal;
|
||||
fetcher?: ObservatoryRecordedJobFetch;
|
||||
} = {},
|
||||
): Promise<readonly ObservatoryRecordedJob[]> {
|
||||
const query = new URLSearchParams({
|
||||
source_session_id: sourceSessionId,
|
||||
setup_id: setupId,
|
||||
limit: "20",
|
||||
});
|
||||
const response = await request(fetcher, `/api/v1/observatory/runs?${query}`, {
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json" },
|
||||
signal,
|
||||
});
|
||||
const body = await responseBody(response);
|
||||
if (!response.ok) throw apiError(body, response.status);
|
||||
const row = record(body, "список расчётов");
|
||||
exactKeys(row, ["authority", "items", "schema_version"], "список расчётов");
|
||||
exact(row.schema_version, JOB_LIST_SCHEMA, "schema_version списка расчётов");
|
||||
observationAuthority(row.authority);
|
||||
return array(row.items, "items").map(decodeJob).filter((job) => (
|
||||
job.sourceSessionId === sourceSessionId && job.setupId === setupId
|
||||
));
|
||||
}
|
||||
|
||||
export async function submitObservatoryRecordedJob(
|
||||
sourceSessionId: string,
|
||||
setupId: string,
|
||||
idempotencyKey: string,
|
||||
{
|
||||
signal,
|
||||
fetcher = globalThis.fetch,
|
||||
}: {
|
||||
signal?: AbortSignal;
|
||||
fetcher?: ObservatoryRecordedJobFetch;
|
||||
} = {},
|
||||
): Promise<ObservatoryRecordedJob> {
|
||||
const response = await request(fetcher, "/api/v1/observatory/runs", {
|
||||
method: "POST",
|
||||
headers: { Accept: "application/json", "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
schema_version: SUBMIT_SCHEMA,
|
||||
idempotency_key: idempotencyKey,
|
||||
source_session_id: sourceSessionId,
|
||||
setup_id: setupId,
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
const body = await responseBody(response);
|
||||
if (!response.ok) throw apiError(body, response.status);
|
||||
const job = decodeJob(body);
|
||||
if (job.sourceSessionId !== sourceSessionId || job.setupId !== setupId) {
|
||||
throw new ObservatoryRecordedJobContractError(
|
||||
"Расчёт относится к другой сессии или сетапу.",
|
||||
);
|
||||
}
|
||||
return job;
|
||||
}
|
||||
|
||||
function decodeJob(value: unknown): ObservatoryRecordedJob {
|
||||
const row = record(value, "расчёт");
|
||||
exactKeys(row, [
|
||||
"authority", "checkpoint_policy", "claim_generation", "created_at_utc", "executor",
|
||||
"idempotency_key", "identity_sha256", "job_id", "preemption_receipt_sha256",
|
||||
"preemption_requested", "priority", "request_sha256", "restart_from_zero", "result",
|
||||
"schema_version", "setup", "source", "state", "submission_receipt_sha256", "terminal",
|
||||
"updated_at_utc",
|
||||
], "расчёт");
|
||||
exact(row.schema_version, JOB_SCHEMA, "schema_version расчёта");
|
||||
observationAuthority(row.authority);
|
||||
digest(row.request_sha256, "request_sha256");
|
||||
digest(row.identity_sha256, "identity_sha256");
|
||||
digest(row.submission_receipt_sha256, "submission_receipt_sha256");
|
||||
const source = record(row.source, "source");
|
||||
exactKeys(source, [
|
||||
"adapter", "bundle_sha256", "capability_manifest_sha256", "catalog_sha256", "session_id",
|
||||
], "source");
|
||||
digest(source.catalog_sha256, "source.catalog_sha256");
|
||||
digest(source.bundle_sha256, "source.bundle_sha256");
|
||||
digest(source.capability_manifest_sha256, "source.capability_manifest_sha256");
|
||||
const setup = record(row.setup, "setup");
|
||||
exactKeys(setup, ["definition_id", "definition_sha256", "definition_version", "setup_id"], "setup");
|
||||
digest(setup.definition_sha256, "setup.definition_sha256");
|
||||
const state = oneOf(row.state, [
|
||||
"accepted", "queued", "claimed", "running", "paused", "preemption-pending",
|
||||
"succeeded", "failed", "reconciliation-required",
|
||||
] as const, "state");
|
||||
const result = row.result === null ? null : record(row.result, "result");
|
||||
if (result !== null) exactKeys(result, ["result_id", "sha256"], "result");
|
||||
const terminal = row.terminal === null ? null : record(row.terminal, "terminal");
|
||||
if (terminal !== null) exactKeys(terminal, ["code", "message"], "terminal");
|
||||
return {
|
||||
jobId: text(row.job_id, "job_id"),
|
||||
idempotencyKey: text(row.idempotency_key, "idempotency_key"),
|
||||
sourceSessionId: text(source.session_id, "source.session_id"),
|
||||
setupId: text(setup.setup_id, "setup.setup_id"),
|
||||
definitionSha256: String(setup.definition_sha256),
|
||||
state,
|
||||
restartFromZero: boolean(row.restart_from_zero, "restart_from_zero"),
|
||||
resultId: result === null ? null : text(result.result_id, "result.result_id"),
|
||||
terminalMessage: terminal === null || terminal.message === null
|
||||
? null
|
||||
: text(terminal.message, "terminal.message"),
|
||||
createdAtUtc: text(row.created_at_utc, "created_at_utc"),
|
||||
updatedAtUtc: text(row.updated_at_utc, "updated_at_utc"),
|
||||
};
|
||||
}
|
||||
|
||||
function observationAuthority(value: unknown): void {
|
||||
const row = record(value, "authority");
|
||||
const keys = ["commands_enabled", "actuation_allowed", "navigation_or_safety_accepted", "production_accepted"];
|
||||
exactKeys(row, keys, "authority");
|
||||
keys.forEach((key) => exact(row[key], false, `authority.${key}`));
|
||||
}
|
||||
|
||||
async function request(fetcher: ObservatoryRecordedJobFetch, input: string, init: RequestInit): Promise<Response> {
|
||||
try {
|
||||
return await fetcher(input, init);
|
||||
} catch (error) {
|
||||
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
||||
throw new ObservatoryRecordedJobContractError("Очередь расчётов недоступна.");
|
||||
}
|
||||
}
|
||||
|
||||
async function responseBody(response: Response): Promise<unknown> {
|
||||
const body = await response.text();
|
||||
if (!body) return undefined;
|
||||
try { return JSON.parse(body) as unknown; } catch { return body; }
|
||||
}
|
||||
|
||||
function apiError(body: unknown, status: number): ObservatoryRecordedJobContractError {
|
||||
const detail = body && typeof body === "object" && !Array.isArray(body)
|
||||
? (body as Record<string, unknown>).detail
|
||||
: null;
|
||||
return new ObservatoryRecordedJobContractError(
|
||||
typeof detail === "string" && detail.trim() ? detail : `Observatory API вернул HTTP ${status}.`,
|
||||
status,
|
||||
);
|
||||
}
|
||||
|
||||
function record(value: unknown, label: string): Record<string, unknown> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
throw new ObservatoryRecordedJobContractError(`${label}: ожидался объект.`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function array(value: unknown, label: string): unknown[] {
|
||||
if (!Array.isArray(value)) throw new ObservatoryRecordedJobContractError(`${label}: ожидался массив.`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function text(value: unknown, label: string): string {
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
throw new ObservatoryRecordedJobContractError(`${label}: ожидался текст.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function digest(value: unknown, label: string): string {
|
||||
const valueText = text(value, label);
|
||||
if (!SHA256.test(valueText)) throw new ObservatoryRecordedJobContractError(`${label}: некорректный digest.`);
|
||||
return valueText;
|
||||
}
|
||||
|
||||
function boolean(value: unknown, label: string): boolean {
|
||||
if (typeof value !== "boolean") throw new ObservatoryRecordedJobContractError(`${label}: ожидался boolean.`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function exact<T>(value: unknown, expected: T, label: string): T {
|
||||
if (value !== expected) throw new ObservatoryRecordedJobContractError(`${label}: значение изменилось.`);
|
||||
return expected;
|
||||
}
|
||||
|
||||
function oneOf<const T extends readonly string[]>(value: unknown, allowed: T, label: string): T[number] {
|
||||
if (typeof value !== "string" || !allowed.includes(value)) {
|
||||
throw new ObservatoryRecordedJobContractError(`${label}: значение не поддерживается.`);
|
||||
}
|
||||
return value as T[number];
|
||||
}
|
||||
|
||||
function exactKeys(value: Record<string, unknown>, expected: readonly string[], label: string): void {
|
||||
const actual = Object.keys(value).sort();
|
||||
const sortedExpected = [...expected].sort();
|
||||
if (actual.length !== sortedExpected.length || actual.some((key, index) => key !== sortedExpected[index])) {
|
||||
throw new ObservatoryRecordedJobContractError(`${label}: обнаружены неизвестные поля.`);
|
||||
}
|
||||
}
|
||||
@@ -111,6 +111,11 @@ export function useObservatoryLaboratorySetups(sourceSessionId: string) {
|
||||
}
|
||||
}, [preflight.kind, selectedSetup, sourceSessionId]);
|
||||
|
||||
useEffect(() => {
|
||||
if (state !== "ready" || !selectedSetup || preflight.kind !== "idle") return;
|
||||
void check();
|
||||
}, [check, preflight.kind, selectedSetup, state]);
|
||||
|
||||
return {
|
||||
catalog: activeCatalog,
|
||||
state,
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
|
||||
import {
|
||||
fetchObservatoryRecordedJobs,
|
||||
submitObservatoryRecordedJob,
|
||||
type ObservatoryRecordedJob,
|
||||
} from "./recordedJobs";
|
||||
|
||||
type RecordedJobsState = "idle" | "loading" | "ready" | "refreshing" | "submitting" | "error";
|
||||
|
||||
const OPEN_STATES = new Set([
|
||||
"accepted", "queued", "claimed", "running", "paused", "preemption-pending",
|
||||
"reconciliation-required",
|
||||
]);
|
||||
|
||||
export function useObservatoryRecordedJobs(sourceSessionId: string, setupId: string) {
|
||||
const [jobs, setJobs] = useState<readonly ObservatoryRecordedJob[]>([]);
|
||||
const [state, setState] = useState<RecordedJobsState>("idle");
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [revision, setRevision] = useState(0);
|
||||
const requestRef = useRef<AbortController | null>(null);
|
||||
const requestSequence = useRef(0);
|
||||
const idempotencyKeys = useRef(new Map<string, string>());
|
||||
const selectionKey = `${sourceSessionId}\u0000${setupId}`;
|
||||
|
||||
useEffect(() => {
|
||||
requestRef.current?.abort();
|
||||
requestRef.current = null;
|
||||
if (!sourceSessionId || !setupId) {
|
||||
setJobs([]);
|
||||
setState("idle");
|
||||
setError(null);
|
||||
return;
|
||||
}
|
||||
const sequence = ++requestSequence.current;
|
||||
const request = new AbortController();
|
||||
requestRef.current = request;
|
||||
setState((current) => jobs.length > 0 && current !== "idle" ? "refreshing" : "loading");
|
||||
setError(null);
|
||||
void fetchObservatoryRecordedJobs(sourceSessionId, setupId, { signal: request.signal })
|
||||
.then((next) => {
|
||||
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
||||
setJobs(next);
|
||||
setState("ready");
|
||||
})
|
||||
.catch((caught: unknown) => {
|
||||
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
||||
setState("error");
|
||||
setError(caught instanceof Error && caught.message.trim()
|
||||
? caught.message
|
||||
: "Очередь расчётов недоступна.");
|
||||
});
|
||||
return () => request.abort();
|
||||
}, [revision, selectionKey]);
|
||||
|
||||
useEffect(() => () => requestRef.current?.abort(), []);
|
||||
|
||||
const latestJob = jobs[0] ?? null;
|
||||
const activeJob = useMemo(
|
||||
() => jobs.find((job) => OPEN_STATES.has(job.state)) ?? null,
|
||||
[jobs],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!latestJob || activeJob) return;
|
||||
idempotencyKeys.current.delete(selectionKey);
|
||||
}, [activeJob, latestJob, selectionKey]);
|
||||
|
||||
const refresh = useCallback(() => setRevision((value) => value + 1), []);
|
||||
|
||||
const submit = useCallback(async (): Promise<ObservatoryRecordedJob | null> => {
|
||||
if (!sourceSessionId || !setupId || state === "submitting") return null;
|
||||
if (activeJob) return activeJob;
|
||||
requestRef.current?.abort();
|
||||
const request = new AbortController();
|
||||
requestRef.current = request;
|
||||
const key = idempotencyKeys.current.get(selectionKey)
|
||||
?? createIdempotencyKey();
|
||||
idempotencyKeys.current.set(selectionKey, key);
|
||||
setState("submitting");
|
||||
setError(null);
|
||||
try {
|
||||
const job = await submitObservatoryRecordedJob(sourceSessionId, setupId, key, {
|
||||
signal: request.signal,
|
||||
});
|
||||
if (request.signal.aborted || requestRef.current !== request) return null;
|
||||
setJobs((current) => [job, ...current.filter((candidate) => candidate.jobId !== job.jobId)]);
|
||||
setState("ready");
|
||||
return job;
|
||||
} catch (caught) {
|
||||
if (request.signal.aborted || requestRef.current !== request) return null;
|
||||
setState("error");
|
||||
setError(caught instanceof Error && caught.message.trim()
|
||||
? caught.message
|
||||
: "Не удалось поставить расчёт в очередь.");
|
||||
return null;
|
||||
} finally {
|
||||
if (requestRef.current === request) requestRef.current = null;
|
||||
}
|
||||
}, [activeJob, selectionKey, setupId, sourceSessionId, state]);
|
||||
|
||||
return { jobs, latestJob, activeJob, state, error, refresh, submit };
|
||||
}
|
||||
|
||||
function createIdempotencyKey(): string {
|
||||
const entropy = typeof globalThis.crypto?.randomUUID === "function"
|
||||
? globalThis.crypto.randomUUID()
|
||||
: `${Date.now().toString(36)}-${Math.random().toString(16).slice(2)}`;
|
||||
return `observatory-ui:${entropy}`;
|
||||
}
|
||||
|
||||
export type ObservatoryRecordedJobsController = ReturnType<typeof useObservatoryRecordedJobs>;
|
||||
@@ -20,8 +20,6 @@
|
||||
}
|
||||
|
||||
.observatory-lead,
|
||||
.observatory-catalog-bar,
|
||||
.observatory-catalog-bar__controls,
|
||||
.observatory-notice {
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
@@ -35,14 +33,12 @@
|
||||
.observatory-lead h2,
|
||||
.observatory-catalog-bar h3,
|
||||
.observatory-session-summary h3,
|
||||
.observatory-setup-detail h3,
|
||||
.observatory-evidence h3 {
|
||||
margin: 0.3rem 0 0;
|
||||
}
|
||||
|
||||
.observatory-catalog-bar h3,
|
||||
.observatory-session-summary h3,
|
||||
.observatory-setup-detail h3,
|
||||
.observatory-evidence h3 {
|
||||
font-size: var(--nodedc-font-size-lg);
|
||||
letter-spacing: -0.02em;
|
||||
@@ -50,7 +46,6 @@
|
||||
}
|
||||
|
||||
.observatory-lead p,
|
||||
.observatory-catalog-bar p,
|
||||
.observatory-state p,
|
||||
.observatory-evidence-empty p {
|
||||
max-width: 52rem;
|
||||
@@ -67,17 +62,37 @@
|
||||
}
|
||||
|
||||
.observatory-catalog-bar__copy {
|
||||
flex: 1 1 auto;
|
||||
display: grid;
|
||||
flex: 0 0 auto;
|
||||
align-content: center;
|
||||
gap: 0.1rem;
|
||||
}
|
||||
|
||||
.observatory-catalog-bar h3 {
|
||||
margin-top: 0.1rem;
|
||||
}
|
||||
|
||||
.observatory-catalog-bar__controls {
|
||||
flex: 2 1 48rem;
|
||||
min-width: 0;
|
||||
flex: 1 1 auto;
|
||||
flex-wrap: nowrap;
|
||||
justify-content: flex-end;
|
||||
}
|
||||
|
||||
.observatory-catalog-bar__controls .nodedc-select-anchor {
|
||||
flex: 1 1 18rem;
|
||||
min-width: min(22rem, 100%);
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.observatory-catalog-bar__controls > .nodedc-button {
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.observatory-catalog-bar__run {
|
||||
display: flex;
|
||||
flex: 0 0 auto;
|
||||
align-items: center;
|
||||
gap: 0.55rem;
|
||||
}
|
||||
|
||||
.observatory-state {
|
||||
@@ -105,9 +120,9 @@
|
||||
|
||||
.observatory-session-summary {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(14rem, 1.1fr) minmax(22rem, 1.6fr) minmax(14rem, auto);
|
||||
grid-template-columns: minmax(11rem, 1.05fr) minmax(20rem, 1.6fr) minmax(11rem, auto);
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.observatory-session-summary__identity {
|
||||
@@ -131,7 +146,7 @@
|
||||
.observatory-session-summary__facts {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(6.5rem, 1fr));
|
||||
gap: 1rem;
|
||||
gap: 0.75rem;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
@@ -159,7 +174,7 @@
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
justify-items: end;
|
||||
gap: 0.6rem;
|
||||
gap: 0.4rem;
|
||||
}
|
||||
|
||||
.observatory-modalities {
|
||||
@@ -171,135 +186,6 @@
|
||||
color: var(--nodedc-text-muted);
|
||||
}
|
||||
|
||||
.observatory-setup-detail {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
gap: 0.85rem 1rem;
|
||||
container-name: observatory-setup;
|
||||
container-type: inline-size;
|
||||
}
|
||||
|
||||
.observatory-setup-detail__identity {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.observatory-setup-detail__identity h3 {
|
||||
margin: 0.15rem 0 0;
|
||||
}
|
||||
|
||||
.observatory-setup-detail__identity p {
|
||||
max-width: 68rem;
|
||||
margin: 0.3rem 0 0;
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: var(--nodedc-font-size-sm);
|
||||
line-height: 1.35;
|
||||
}
|
||||
|
||||
.observatory-setup-detail__badges,
|
||||
.observatory-setup-detail__actions,
|
||||
.observatory-setup-detail__reason {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.55rem;
|
||||
}
|
||||
|
||||
.observatory-setup-detail__badges {
|
||||
align-self: start;
|
||||
justify-content: flex-end;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.observatory-setup-detail__facts {
|
||||
display: grid;
|
||||
grid-column: 1 / -1;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: 0.75rem;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.observatory-setup-detail__facts > div {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 0.2rem;
|
||||
padding: 0.65rem 0.75rem;
|
||||
border-radius: var(--nodedc-radius-control-compact);
|
||||
background: var(--nodedc-glass-control-bg);
|
||||
}
|
||||
|
||||
.observatory-setup-detail__facts dt {
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: var(--nodedc-font-size-xs);
|
||||
}
|
||||
|
||||
.observatory-setup-detail__facts dd {
|
||||
min-width: 0;
|
||||
margin: 0;
|
||||
overflow-wrap: anywhere;
|
||||
color: var(--nodedc-text-secondary);
|
||||
font-size: var(--nodedc-font-size-sm);
|
||||
line-height: 1.35;
|
||||
}
|
||||
|
||||
.observatory-setup-results {
|
||||
display: grid;
|
||||
grid-column: 1 / -1;
|
||||
gap: 0.35rem;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.observatory-setup-results li {
|
||||
display: grid;
|
||||
grid-template-columns: auto minmax(0, 1fr) auto;
|
||||
align-items: center;
|
||||
gap: 0.65rem;
|
||||
min-height: 3.15rem;
|
||||
padding: 0.5rem 0.7rem;
|
||||
border-radius: var(--nodedc-radius-control-compact);
|
||||
background: var(--nodedc-glass-control-bg);
|
||||
color: var(--nodedc-text-secondary);
|
||||
}
|
||||
|
||||
.observatory-setup-results span {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 0.1rem;
|
||||
}
|
||||
|
||||
.observatory-setup-results strong,
|
||||
.observatory-setup-results small {
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.observatory-setup-results small {
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: var(--nodedc-font-size-sm);
|
||||
line-height: 1.35;
|
||||
}
|
||||
|
||||
.observatory-setup-detail__footer {
|
||||
display: flex;
|
||||
grid-column: 1 / -1;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 0.8rem;
|
||||
padding-top: 0.1rem;
|
||||
}
|
||||
|
||||
.observatory-setup-detail__reason {
|
||||
min-width: 0;
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: var(--nodedc-font-size-sm);
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.observatory-setup-detail__actions {
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.observatory-evidence {
|
||||
display: grid;
|
||||
gap: 0.65rem;
|
||||
@@ -454,7 +340,7 @@
|
||||
font-size: var(--nodedc-font-size-sm);
|
||||
}
|
||||
|
||||
@container observatory-session (max-width: 56rem) {
|
||||
@container observatory-session (max-width: 46rem) {
|
||||
.observatory-session-summary {
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
align-items: start;
|
||||
@@ -467,29 +353,6 @@
|
||||
}
|
||||
}
|
||||
|
||||
@container observatory-setup (max-width: 48rem) {
|
||||
.observatory-setup-detail {
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
}
|
||||
|
||||
.observatory-setup-detail__badges {
|
||||
justify-content: flex-start;
|
||||
}
|
||||
|
||||
.observatory-setup-detail__facts {
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
}
|
||||
|
||||
.observatory-setup-detail__footer {
|
||||
align-items: stretch;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.observatory-setup-detail__actions {
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
}
|
||||
|
||||
@container observatory-session (max-width: 48rem) {
|
||||
.observatory-evidence-card {
|
||||
grid-template-columns: auto minmax(0, 1fr) auto;
|
||||
@@ -535,4 +398,8 @@
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.observatory-catalog-bar__run {
|
||||
justify-content: flex-end;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
import {
|
||||
ActivityIndicator,
|
||||
Button,
|
||||
GlassSurface,
|
||||
Icon,
|
||||
StatusBadge,
|
||||
} from "@nodedc/ui-react";
|
||||
|
||||
import type { ObservatoryEvidence } from "../../core/observatory/catalog";
|
||||
import type { ObservatoryLaboratorySetup } from "../../core/observatory/laboratorySetups";
|
||||
import type { ObservatoryRecordedRunBinding } from "../../core/observatory/recordedRun";
|
||||
import type { SetupPreflightState } from "../../core/observatory/useObservatoryLaboratorySetups";
|
||||
|
||||
const relationLabel: Readonly<Record<string, string>> = {
|
||||
"primary-visual": "Основной визуальный разбор",
|
||||
"compute-successor": "Связанное вычислительное доказательство",
|
||||
"canonical-projection": "Готовый записанный разбор",
|
||||
};
|
||||
|
||||
function workerLabel(value: string): string {
|
||||
const match = /^worker-(\d+)$/.exec(value);
|
||||
return match ? `Worker ${match[1]}` : value;
|
||||
}
|
||||
|
||||
export function ObservatorySetupDetail({
|
||||
setup,
|
||||
evidence,
|
||||
preflight,
|
||||
onCheck,
|
||||
onOpenExisting,
|
||||
}: {
|
||||
setup: ObservatoryLaboratorySetup;
|
||||
evidence: readonly ObservatoryEvidence[];
|
||||
preflight: SetupPreflightState;
|
||||
onCheck: () => void;
|
||||
onOpenExisting: (binding: ObservatoryRecordedRunBinding) => void;
|
||||
}) {
|
||||
const exactExisting = evidence.find(
|
||||
(candidate) => candidate.recordedRun
|
||||
&& setup.preflight.existingResultIds.includes(candidate.recordedRun.resultId),
|
||||
)?.recordedRun ?? null;
|
||||
const displayedReason = preflight.kind === "ready"
|
||||
? preflight.value.checks.find((check) => check.checkId === "executor")?.message
|
||||
?? setup.executor.reason
|
||||
: preflight.kind === "error"
|
||||
? preflight.message
|
||||
: setup.preflight.reason;
|
||||
|
||||
return (
|
||||
<GlassSurface className="observatory-setup-detail" padding="md">
|
||||
<div className="observatory-setup-detail__identity">
|
||||
<span className="section-eyebrow">СЕТАП ЛАБОРАТОРИИ</span>
|
||||
<h3>{setup.displayName}</h3>
|
||||
<p>{setup.description}</p>
|
||||
</div>
|
||||
<div className="observatory-setup-detail__badges">
|
||||
<StatusBadge tone={setup.compatibility.compatible ? "success" : "warning"}>
|
||||
{setup.compatibility.compatible ? "Совместим" : "Другая сессия"}
|
||||
</StatusBadge>
|
||||
<StatusBadge tone={setup.origin === "existing-result" ? "accent" : "neutral"}>
|
||||
{setup.origin === "existing-result" ? "Готовый результат" : "Архивный сетап"}
|
||||
</StatusBadge>
|
||||
</div>
|
||||
<dl className="observatory-setup-detail__facts">
|
||||
<div>
|
||||
<dt>Идентичность</dt>
|
||||
<dd>
|
||||
{setup.runDefinition
|
||||
? `Версия ${setup.runDefinition.version} · конфигурация зафиксирована`
|
||||
: "Готовый записанный результат без повторно запускаемой конфигурации"}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Исполнитель</dt>
|
||||
<dd>
|
||||
{setup.preflight.outcome === "existing"
|
||||
? "Не требуется для просмотра"
|
||||
: `${workerLabel(setup.executor.contourId)} · повторный запуск не подключён`}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Сохранено</dt>
|
||||
<dd>{setup.preservedResults.length} неизменяемых результатов</dd>
|
||||
</div>
|
||||
</dl>
|
||||
<ol className="observatory-setup-results" aria-label="Сохранённые результаты сетапа">
|
||||
{setup.preservedResults.map((result) => (
|
||||
<li key={result.resultId}>
|
||||
<Icon name={result.access === "evidence-only" ? "database" : "clipboard"} size={15} />
|
||||
<span>
|
||||
<strong>{relationLabel[result.relation] ?? "Сохранённый результат"}</strong>
|
||||
<small>
|
||||
{result.access === "observatory"
|
||||
? "Доступен в Обсерватории"
|
||||
: result.access === "legacy-lab"
|
||||
? "Сохранён в legacy LAB"
|
||||
: "Сохранено как связанное доказательство"}
|
||||
</small>
|
||||
</span>
|
||||
<StatusBadge tone={result.observatoryProjectionAvailable ? "success" : "neutral"}>
|
||||
{result.access === "observatory"
|
||||
? result.observatoryProjectionAvailable ? "Готов" : "Сохранён"
|
||||
: result.access === "legacy-lab" ? "Legacy LAB" : "Evidence"}
|
||||
</StatusBadge>
|
||||
</li>
|
||||
))}
|
||||
</ol>
|
||||
<div className="observatory-setup-detail__footer">
|
||||
<div className="observatory-setup-detail__reason" role={preflight.kind === "error" ? "alert" : "status"}>
|
||||
<Icon name={preflight.kind === "error" ? "alert" : "activity"} size={16} />
|
||||
<span>{displayedReason}</span>
|
||||
</div>
|
||||
<div className="observatory-setup-detail__actions">
|
||||
{exactExisting ? (
|
||||
<Button
|
||||
size="compact"
|
||||
variant="secondary"
|
||||
icon={<Icon name="eye" size={15} />}
|
||||
onClick={() => onOpenExisting(exactExisting)}
|
||||
>
|
||||
Открыть готовый результат
|
||||
</Button>
|
||||
) : (
|
||||
<Button
|
||||
size="compact"
|
||||
variant="secondary"
|
||||
disabled={preflight.kind === "checking"}
|
||||
onClick={onCheck}
|
||||
>
|
||||
{preflight.kind === "checking"
|
||||
? <ActivityIndicator size="compact" label="Проверяем совместимость" />
|
||||
: "Проверить совместимость"}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</GlassSurface>
|
||||
);
|
||||
}
|
||||
@@ -32,8 +32,9 @@ import {
|
||||
} from "../../core/observatory/catalogMutations";
|
||||
import { useObservatoryCatalog } from "../../core/observatory/useObservatoryCatalog";
|
||||
import { useObservatoryLaboratorySetups } from "../../core/observatory/useObservatoryLaboratorySetups";
|
||||
import { useObservatoryRecordedJobs } from "../../core/observatory/useObservatoryRecordedJobs";
|
||||
import type { ObservatoryRecordedJobState } from "../../core/observatory/recordedJobs";
|
||||
import type { WorkspaceDefinition } from "../../productModel";
|
||||
import { ObservatorySetupDetail } from "./ObservatorySetupDetail";
|
||||
|
||||
const MAX_PRESENTED_EVIDENCE = 6;
|
||||
const EMPTY_OBSERVATORY_ITEMS = [] as const;
|
||||
@@ -87,6 +88,27 @@ const modalityLabel: Record<string, string> = {
|
||||
telemetry: "Телеметрия",
|
||||
};
|
||||
|
||||
const recordedJobStatus: Record<
|
||||
ObservatoryRecordedJobState,
|
||||
{
|
||||
readonly label: string;
|
||||
readonly tone: "success" | "accent" | "warning" | "danger" | "neutral";
|
||||
}
|
||||
> = {
|
||||
accepted: { label: "Принят", tone: "neutral" },
|
||||
queued: { label: "Ждёт Worker", tone: "neutral" },
|
||||
claimed: { label: "Назначен Worker", tone: "accent" },
|
||||
running: { label: "Выполняется", tone: "accent" },
|
||||
paused: { label: "Пауза: live-поток", tone: "warning" },
|
||||
"preemption-pending": {
|
||||
label: "Ждём подтверждения остановки",
|
||||
tone: "warning",
|
||||
},
|
||||
succeeded: { label: "Готово", tone: "success" },
|
||||
failed: { label: "Ошибка расчёта", tone: "danger" },
|
||||
"reconciliation-required": { label: "Нужна сверка", tone: "warning" },
|
||||
};
|
||||
|
||||
function statusTone(
|
||||
status: ObservationSessionStatus,
|
||||
): "success" | "accent" | "warning" | "danger" | "neutral" {
|
||||
@@ -136,6 +158,10 @@ export function ObservatoryWorkspace({
|
||||
const controller = useObservatoryCatalog();
|
||||
const [selectedSessionId, setSelectedSessionId] = useState("");
|
||||
const setupController = useObservatoryLaboratorySetups(selectedSessionId);
|
||||
const recordedJobsController = useObservatoryRecordedJobs(
|
||||
selectedSessionId,
|
||||
setupController.selectedSetupId,
|
||||
);
|
||||
const [replay, setReplay] = useState<ObservatoryReplayState>({ kind: "closed" });
|
||||
const [renameTarget, setRenameTarget] = useState<ObservatoryEvidence | null>(null);
|
||||
const [renameValue, setRenameValue] = useState("");
|
||||
@@ -188,6 +214,29 @@ export function ObservatoryWorkspace({
|
||||
: "Несовместим с выбранной сессией",
|
||||
})) ?? []
|
||||
), [setupController.catalog]);
|
||||
const runPreflight = setupController.preflight.kind === "ready"
|
||||
? setupController.preflight.value
|
||||
: null;
|
||||
const queueSubmissionAllowed = Boolean(
|
||||
setupController.selectedSetup?.runDefinition
|
||||
&& setupController.selectedSetup.compatibility.compatible
|
||||
&& runPreflight?.outcome === "queueable"
|
||||
&& runPreflight.submissionAllowed,
|
||||
);
|
||||
const presentedJob = recordedJobsController.activeJob
|
||||
?? recordedJobsController.latestJob;
|
||||
const presentedJobStatus = presentedJob
|
||||
? recordedJobStatus[presentedJob.state]
|
||||
: null;
|
||||
const queueStatusError = setupController.preflight.kind === "error"
|
||||
? setupController.preflight.message
|
||||
: recordedJobsController.error;
|
||||
const queueStateBusy = recordedJobsController.state === "loading"
|
||||
|| recordedJobsController.state === "refreshing"
|
||||
|| recordedJobsController.state === "submitting";
|
||||
const canSubmitRecordedJob = queueSubmissionAllowed
|
||||
&& recordedJobsController.state === "ready"
|
||||
&& recordedJobsController.activeJob === null;
|
||||
const initialLoading = !controller.catalog
|
||||
&& ["idle", "loading"].includes(controller.state);
|
||||
const unavailable = !controller.catalog && controller.state === "error";
|
||||
@@ -374,11 +423,10 @@ export function ObservatoryWorkspace({
|
||||
</StatusBadge>
|
||||
</section>
|
||||
|
||||
<GlassSurface className="observatory-catalog-bar" padding="md">
|
||||
<GlassSurface className="observatory-catalog-bar" padding="sm">
|
||||
<div className="observatory-catalog-bar__copy">
|
||||
<span className="section-eyebrow">ИСТОЧНИК ДОКАЗАТЕЛЬСТВ</span>
|
||||
<h3>Сохранённая сессия</h3>
|
||||
<p>Выбор меняет только читаемую карточку и не готовит визуальный разбор в фоне.</p>
|
||||
</div>
|
||||
<div className="observatory-catalog-bar__controls">
|
||||
<Select
|
||||
@@ -418,10 +466,40 @@ export function ObservatoryWorkspace({
|
||||
onClick={() => {
|
||||
void controller.refresh();
|
||||
setupController.refresh();
|
||||
recordedJobsController.refresh();
|
||||
}}
|
||||
>
|
||||
Обновить
|
||||
</Button>
|
||||
<div className="observatory-catalog-bar__run" aria-live="polite">
|
||||
{queueStatusError ? (
|
||||
<StatusBadge tone="danger" title={queueStatusError}>
|
||||
Очередь недоступна
|
||||
</StatusBadge>
|
||||
) : recordedJobsController.state === "submitting" ? (
|
||||
<StatusBadge tone="neutral">Ставим в очередь</StatusBadge>
|
||||
) : presentedJobStatus ? (
|
||||
<StatusBadge
|
||||
tone={presentedJobStatus.tone}
|
||||
title={presentedJob?.terminalMessage ?? undefined}
|
||||
>
|
||||
{presentedJobStatus.label}
|
||||
</StatusBadge>
|
||||
) : setupController.preflight.kind === "checking" ? (
|
||||
<StatusBadge tone="neutral">Проверяем сетап</StatusBadge>
|
||||
) : queueStateBusy ? (
|
||||
<StatusBadge tone="neutral">Читаем очередь</StatusBadge>
|
||||
) : null}
|
||||
{canSubmitRecordedJob ? (
|
||||
<Button
|
||||
size="compact"
|
||||
variant="primary"
|
||||
onClick={() => { void recordedJobsController.submit(); }}
|
||||
>
|
||||
Рассчитать
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
</GlassSurface>
|
||||
|
||||
@@ -476,7 +554,7 @@ export function ObservatoryWorkspace({
|
||||
</GlassSurface>
|
||||
) : selectedSession ? (
|
||||
<section className="observatory-session-stack" aria-label="Выбранная сессия и связанные результаты">
|
||||
<GlassSurface className="observatory-session-summary" padding="md">
|
||||
<GlassSurface className="observatory-session-summary" padding="sm">
|
||||
<div className="observatory-session-summary__identity">
|
||||
<span className="section-eyebrow">ИСХОДНАЯ СЕССИЯ</span>
|
||||
<h3>{selectedSession.source.label}</h3>
|
||||
@@ -506,22 +584,6 @@ export function ObservatoryWorkspace({
|
||||
</div>
|
||||
</GlassSurface>
|
||||
|
||||
{setupController.selectedSetup ? (
|
||||
<ObservatorySetupDetail
|
||||
setup={setupController.selectedSetup}
|
||||
evidence={selectedSession.evidence}
|
||||
preflight={setupController.preflight}
|
||||
onCheck={() => { void setupController.check(); }}
|
||||
onOpenExisting={openReplay}
|
||||
/>
|
||||
) : setupController.error ? (
|
||||
<GlassSurface className="observatory-notice" padding="md" tone="soft" role="alert">
|
||||
<StatusBadge tone="warning">Сетапы недоступны</StatusBadge>
|
||||
<span className="observatory-notice__copy">{setupController.error}</span>
|
||||
<Button size="compact" variant="ghost" onClick={setupController.refresh}>Повторить</Button>
|
||||
</GlassSurface>
|
||||
) : null}
|
||||
|
||||
<section className="observatory-evidence">
|
||||
<header>
|
||||
<div>
|
||||
|
||||
@@ -156,6 +156,40 @@ test("Observatory preflight sends the exact selected definition and never submit
|
||||
assert.equal(preflight.submissionAllowed, false);
|
||||
});
|
||||
|
||||
test("Observatory dynamic preflight admits only an explicit queueable response", async () => {
|
||||
const selected = (await fetchObservatoryLaboratorySetups("source-a", {
|
||||
fetcher: async () => new Response(JSON.stringify({
|
||||
schema_version: "missioncore.observatory-laboratory-setup-catalog/v1",
|
||||
source_session_id: "source-a",
|
||||
setups: [setup()],
|
||||
authority,
|
||||
}), { status: 200 }),
|
||||
})).setups[0];
|
||||
|
||||
const preflight = await preflightObservatoryLaboratorySetup("source-a", selected, {
|
||||
fetcher: async () => new Response(JSON.stringify({
|
||||
schema_version: "missioncore.observatory-run-preflight/v1",
|
||||
source_session_id: "source-a",
|
||||
setup_id: selected.setupId,
|
||||
definition_sha256: "b".repeat(64),
|
||||
outcome: "queueable",
|
||||
submission_allowed: true,
|
||||
checks: [{
|
||||
check_id: "durable-queue",
|
||||
outcome: "pass",
|
||||
reason_code: "exact-binding-ready",
|
||||
message: "Точный источник и очередь готовы.",
|
||||
}],
|
||||
existing_result_ids: [],
|
||||
executor: setup().executor,
|
||||
authority,
|
||||
}), { status: 200 }),
|
||||
});
|
||||
|
||||
assert.equal(preflight.outcome, "queueable");
|
||||
assert.equal(preflight.submissionAllowed, true);
|
||||
});
|
||||
|
||||
test("Observatory setup contract rejects authority escalation and response drift", async () => {
|
||||
const escalated = setup();
|
||||
escalated.authority = { ...authority, commands_enabled: true };
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { after, before, test } from "node:test";
|
||||
|
||||
import { createServer } from "vite";
|
||||
|
||||
let server;
|
||||
let fetchObservatoryRecordedJobs;
|
||||
let submitObservatoryRecordedJob;
|
||||
let ObservatoryRecordedJobContractError;
|
||||
|
||||
const authority = {
|
||||
commands_enabled: false,
|
||||
actuation_allowed: false,
|
||||
navigation_or_safety_accepted: false,
|
||||
production_accepted: false,
|
||||
};
|
||||
|
||||
function job(state = "queued") {
|
||||
return {
|
||||
schema_version: "missioncore.observatory-recorded-job/v1",
|
||||
job_id: "observatory-run-0123456789abcdef0123456789abcdef",
|
||||
idempotency_key: "observatory-ui:source-a:m49-tgs:request-a",
|
||||
request_sha256: "1".repeat(64),
|
||||
identity_sha256: "2".repeat(64),
|
||||
submission_receipt_sha256: "3".repeat(64),
|
||||
source: {
|
||||
session_id: "source-a",
|
||||
catalog_sha256: "4".repeat(64),
|
||||
bundle_sha256: "5".repeat(64),
|
||||
capability_manifest_sha256: "6".repeat(64),
|
||||
adapter: {
|
||||
adapter_id: "ravnoves00-m49-source-pack",
|
||||
version: 1,
|
||||
adapter_sha256: "7".repeat(64),
|
||||
},
|
||||
},
|
||||
setup: {
|
||||
setup_id: "m49-tgs",
|
||||
definition_id: "m49-tgs",
|
||||
definition_version: 1,
|
||||
definition_sha256: "8".repeat(64),
|
||||
},
|
||||
executor: {
|
||||
release_id: "m49-tgs-release",
|
||||
release_sha256: "9".repeat(64),
|
||||
image_sha256: "a".repeat(64),
|
||||
model_release_ids: [],
|
||||
learned_models: [],
|
||||
model_manifest_sha256: "b".repeat(64),
|
||||
resource_profile_id: "m49-tgs-cpu",
|
||||
resource_profile_sha256: "c".repeat(64),
|
||||
},
|
||||
checkpoint_policy: {
|
||||
mode: "non-checkpointable",
|
||||
allowed_checkpoints: [],
|
||||
last_checkpoint_id: null,
|
||||
},
|
||||
priority: { class: "recorded", rank: 100, server_owned: true },
|
||||
state,
|
||||
preemption_requested: state === "preemption-pending",
|
||||
restart_from_zero: state === "paused",
|
||||
preemption_receipt_sha256: null,
|
||||
claim_generation: 0,
|
||||
result: state === "succeeded"
|
||||
? { result_id: "m49-result", sha256: "d".repeat(64) }
|
||||
: null,
|
||||
terminal: state === "failed"
|
||||
? { code: "worker-failed", message: "Worker завершил расчёт с ошибкой." }
|
||||
: null,
|
||||
created_at_utc: "2026-08-31T10:00:00Z",
|
||||
updated_at_utc: "2026-08-31T10:00:01Z",
|
||||
authority,
|
||||
};
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
server = await createServer({
|
||||
appType: "custom",
|
||||
logLevel: "silent",
|
||||
server: { middlewareMode: true },
|
||||
});
|
||||
({
|
||||
fetchObservatoryRecordedJobs,
|
||||
submitObservatoryRecordedJob,
|
||||
ObservatoryRecordedJobContractError,
|
||||
} = await server.ssrLoadModule("/src/core/observatory/recordedJobs.ts"));
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await server?.close();
|
||||
});
|
||||
|
||||
test("Observatory reads the exact session/setup queue without polling", async () => {
|
||||
const calls = [];
|
||||
const jobs = await fetchObservatoryRecordedJobs("source-a", "m49-tgs", {
|
||||
fetcher: async (input, init) => {
|
||||
calls.push({ input: String(input), init });
|
||||
return new Response(JSON.stringify({
|
||||
schema_version: "missioncore.observatory-recorded-job-list/v1",
|
||||
items: [job("running")],
|
||||
authority,
|
||||
}), { status: 200 });
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(
|
||||
calls[0].input,
|
||||
"/api/v1/observatory/runs?source_session_id=source-a&setup_id=m49-tgs&limit=20",
|
||||
);
|
||||
assert.equal(calls[0].init.method, "GET");
|
||||
assert.equal(jobs[0].state, "running");
|
||||
assert.equal(jobs[0].definitionSha256, "8".repeat(64));
|
||||
});
|
||||
|
||||
test("Observatory submits only public identities and accepts every durable state", async () => {
|
||||
const states = [
|
||||
"accepted",
|
||||
"queued",
|
||||
"claimed",
|
||||
"running",
|
||||
"paused",
|
||||
"preemption-pending",
|
||||
"succeeded",
|
||||
"failed",
|
||||
"reconciliation-required",
|
||||
];
|
||||
|
||||
for (const state of states) {
|
||||
let request;
|
||||
const result = await submitObservatoryRecordedJob(
|
||||
"source-a",
|
||||
"m49-tgs",
|
||||
"observatory-ui:source-a:m49-tgs:request-a",
|
||||
{
|
||||
fetcher: async (input, init) => {
|
||||
request = { input: String(input), init };
|
||||
return new Response(JSON.stringify(job(state)), { status: 200 });
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.state, state);
|
||||
assert.equal(request.input, "/api/v1/observatory/runs");
|
||||
assert.equal(request.init.method, "POST");
|
||||
assert.deepEqual(JSON.parse(request.init.body), {
|
||||
schema_version: "missioncore.observatory-recorded-run-submit/v1",
|
||||
idempotency_key: "observatory-ui:source-a:m49-tgs:request-a",
|
||||
source_session_id: "source-a",
|
||||
setup_id: "m49-tgs",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("Observatory queue contract rejects authority escalation and response drift", async () => {
|
||||
const escalated = job();
|
||||
escalated.authority = { ...authority, commands_enabled: true };
|
||||
await assert.rejects(
|
||||
fetchObservatoryRecordedJobs("source-a", "m49-tgs", {
|
||||
fetcher: async () => new Response(JSON.stringify({
|
||||
schema_version: "missioncore.observatory-recorded-job-list/v1",
|
||||
items: [escalated],
|
||||
authority,
|
||||
}), { status: 200 }),
|
||||
}),
|
||||
ObservatoryRecordedJobContractError,
|
||||
);
|
||||
|
||||
const drifted = job();
|
||||
drifted.worker_command = "forbidden";
|
||||
await assert.rejects(
|
||||
submitObservatoryRecordedJob(
|
||||
"source-a",
|
||||
"m49-tgs",
|
||||
"observatory-ui:source-a:m49-tgs:request-a",
|
||||
{ fetcher: async () => new Response(JSON.stringify(drifted), { status: 200 }) },
|
||||
),
|
||||
ObservatoryRecordedJobContractError,
|
||||
);
|
||||
});
|
||||
@@ -144,11 +144,11 @@ test("Observatory mounts the one shared canonical replay only after explicit adm
|
||||
);
|
||||
assert.match(
|
||||
styles,
|
||||
/\.observatory-catalog-bar h3,[\s\S]*\.observatory-session-summary h3,[\s\S]*\.observatory-setup-detail h3,[\s\S]*\.observatory-evidence h3 \{[\s\S]*font-size: var\(--nodedc-font-size-lg\);/,
|
||||
/\.observatory-catalog-bar h3,[\s\S]*\.observatory-session-summary h3,[\s\S]*\.observatory-evidence h3 \{[\s\S]*font-size: var\(--nodedc-font-size-lg\);/,
|
||||
);
|
||||
assert.match(
|
||||
styles,
|
||||
/@container observatory-session \(max-width: 56rem\) \{[\s\S]*\.observatory-session-summary \{[\s\S]*grid-template-columns: minmax\(0, 1fr\);/,
|
||||
/@container observatory-session \(max-width: 46rem\) \{[\s\S]*\.observatory-session-summary \{[\s\S]*grid-template-columns: minmax\(0, 1fr\);/,
|
||||
);
|
||||
assert.match(
|
||||
styles,
|
||||
@@ -206,23 +206,67 @@ test("Observatory rename and delete use admitted projection mutations and canoni
|
||||
assert.match(hook, /applyObservatoryCatalogMutationOverlay\(/);
|
||||
});
|
||||
|
||||
test("Observatory configurator keeps selection and preflight read-only", async () => {
|
||||
const [workspace, setupDetail, setupHook] = await Promise.all([
|
||||
test("Observatory keeps one compact selector axis without the obsolete setup detail", async () => {
|
||||
const [workspace, styles, setupHook, jobsHook] = await Promise.all([
|
||||
read("workspaces/observatory/ObservatoryWorkspace.tsx"),
|
||||
read("workspaces/observatory/ObservatorySetupDetail.tsx"),
|
||||
read("styles/observatory.css"),
|
||||
read("core/observatory/useObservatoryLaboratorySetups.ts"),
|
||||
read("core/observatory/useObservatoryRecordedJobs.ts"),
|
||||
]);
|
||||
|
||||
assert.match(workspace, /className="observatory-catalog-bar" padding="sm"/);
|
||||
assert.match(workspace, /label="Выбрать сохранённую сессию"/);
|
||||
assert.match(workspace, /label="Выбрать сетап лаборатории"/);
|
||||
assert.match(workspace, /Показан последний каталог сетапов/);
|
||||
assert.match(setupDetail, /Проверить совместимость/);
|
||||
assert.doesNotMatch(
|
||||
setupDetail,
|
||||
/Рассчитать лабораторию|definitionSha256|<small>\{result\.resultId\}/,
|
||||
assert.doesNotMatch(workspace, /Выбор меняет только читаемую карточку/);
|
||||
assert.doesNotMatch(workspace, /ObservatorySetupDetail|observatory-setup-detail/);
|
||||
assert.doesNotMatch(styles, /observatory-setup-detail|observatory-setup-results/);
|
||||
assert.match(workspace, /useObservatoryRecordedJobs/);
|
||||
assert.match(
|
||||
workspace,
|
||||
/runPreflight\?\.outcome === "queueable"[\s\S]*runPreflight\.submissionAllowed/,
|
||||
);
|
||||
assert.match(
|
||||
workspace,
|
||||
/canSubmitRecordedJob \? \([\s\S]*>\s*Рассчитать\s*<\/Button>/,
|
||||
);
|
||||
assert.match(workspace, /accepted: \{ label: "Принят"/);
|
||||
assert.match(workspace, /queued: \{ label: "Ждёт Worker"/);
|
||||
assert.match(workspace, /claimed: \{ label: "Назначен Worker"/);
|
||||
assert.match(workspace, /running: \{ label: "Выполняется"/);
|
||||
assert.match(workspace, /paused: \{ label: "Пауза: live-поток"/);
|
||||
assert.match(
|
||||
workspace,
|
||||
/"preemption-pending": \{[\s\S]*label: "Ждём подтверждения остановки"/,
|
||||
);
|
||||
assert.match(workspace, /succeeded: \{ label: "Готово"/);
|
||||
assert.match(workspace, /failed: \{ label: "Ошибка расчёта"/);
|
||||
assert.match(workspace, /"reconciliation-required": \{ label: "Нужна сверка"/);
|
||||
assert.match(
|
||||
jobsHook,
|
||||
/OPEN_STATES[\s\S]*"preemption-pending",[\s\S]*"reconciliation-required"/,
|
||||
);
|
||||
assert.match(jobsHook, /return `observatory-ui:\$\{entropy\}`;/);
|
||||
assert.doesNotMatch(jobsHook, /observatory-ui:[^`]*sourceSessionId|\.slice\(0, 160\)/);
|
||||
assert.match(workspace, /recordedJobsController\.refresh\(\)/);
|
||||
assert.doesNotMatch(`${workspace}\n${jobsHook}`, /setInterval|setTimeout/);
|
||||
assert.match(
|
||||
styles,
|
||||
/\.observatory-catalog-bar__controls \{[\s\S]*min-width: 0;[\s\S]*flex: 1 1 auto;[\s\S]*flex-wrap: nowrap;[\s\S]*justify-content: flex-end;/,
|
||||
);
|
||||
assert.match(
|
||||
styles,
|
||||
/\.observatory-catalog-bar__run \{[\s\S]*display: flex;[\s\S]*align-items: center;/,
|
||||
);
|
||||
assert.match(
|
||||
styles,
|
||||
/@media \(max-width: 920px\)[\s\S]*\.observatory-catalog-bar,[\s\S]*\.observatory-catalog-bar__controls,[\s\S]*flex-direction: column;/,
|
||||
);
|
||||
assert.match(setupHook, /catalog\?\.sourceSessionId === sourceSessionId/);
|
||||
assert.match(setupHook, /preflightRequest\.current\?\.abort\(\)/);
|
||||
assert.match(setupHook, /preflightRequest\.current !== request/);
|
||||
assert.doesNotMatch(`${workspace}\n${setupDetail}\n${setupHook}`, /\/api\/v1\/observatory\/runs/);
|
||||
assert.match(
|
||||
setupHook,
|
||||
/state !== "ready" \|\| !selectedSetup \|\| preflight\.kind !== "idle"[\s\S]*void check\(\)/,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
"setups": [
|
||||
{
|
||||
"setup_id": "m49-tgs-full-shadow-v1",
|
||||
"display_name": "M4.9T5 · TRAVEL TGS · полный source-paced shadow",
|
||||
"description": "Сохранённая CPU-конфигурация RAVNOVES00: 4 489 кадров, causal rolling 1 s, полный визуальный разбор и отдельное доказательство integrated graph.",
|
||||
"display_name": "M4.9T5 · TRAVEL TGS · CPU-only, без ML",
|
||||
"description": "Сохранённая конфигурация RAVNOVES00: TRAVEL TGS без ML-моделей, 4 489 кадров, causal rolling 1 s, полный визуальный разбор и отдельное доказательство integrated graph.",
|
||||
"origin": "archived-definition",
|
||||
"source": {
|
||||
"session_id": "20260720T065719Z_viewer_live",
|
||||
@@ -58,8 +58,8 @@
|
||||
},
|
||||
{
|
||||
"setup_id": "lab-v1-ravnoves004tree-final",
|
||||
"display_name": "LAB V1 · финальный записанный контур восприятия",
|
||||
"description": "Текущий точный RAVNOVES004TREE result: 6 830 кадров EoMT + DDRNet и синхронный записанный replay без доступа к управлению.",
|
||||
"display_name": "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||
"description": "Точный RAVNOVES004TREE result: 6 830 кадров EoMT Cityscapes Large 1024 + DDRNet-39 и синхронный записанный replay; полный TGS и независимый YOLOX отсутствуют.",
|
||||
"origin": "existing-result",
|
||||
"source": {
|
||||
"session_id": "20260828T130511Z_viewer_live",
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"schema_version": "missioncore.observatory-m49-recorded-queue-binding/v1",
|
||||
"binding_id": "m49-ravnoves00-recorded-queue-v1",
|
||||
"source": {
|
||||
"session_id": "20260720T065719Z_viewer_live",
|
||||
"label": "RAVNOVES00",
|
||||
"required_modalities": [
|
||||
"point-cloud",
|
||||
"trajectory",
|
||||
"video"
|
||||
],
|
||||
"source_pack": {
|
||||
"artifact_id": "e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b",
|
||||
"sha256": "0685d24219d8236caf8b7f1685e93f6d6b59e7fd015a768d88a92bbe8b154944",
|
||||
"byte_length": 72996000,
|
||||
"media_type": "application/vnd.nodedc.lidar-source-pack+npz",
|
||||
"expected_timeline_frames": 4489,
|
||||
"expected_available_lidar_frames": 3928
|
||||
}
|
||||
},
|
||||
"setup": {
|
||||
"setup_id": "m49-tgs-full-shadow-v1",
|
||||
"definition_id": "m49-tgs-full-shadow",
|
||||
"definition_version": 1,
|
||||
"definition_sha256": "836a66639e69f7ea00de2a9111c1c6c9c3c00f1abd726f1df596aeb4e3a88ae6"
|
||||
},
|
||||
"source_adapter": {
|
||||
"adapter_id": "ravnoves00-m49-source-pack",
|
||||
"version": 1
|
||||
},
|
||||
"executor": {
|
||||
"release_id": "m49-tgs-full-shadow-worker-release",
|
||||
"artifact_sha256": "5e0ea16c7a5cc760463836718b0cd8b0006ffc4b202e5f706a20a86ef2f912ab",
|
||||
"code_revision": "40c850b167dda366d8aa45d828520168affaf9fd",
|
||||
"service_installed": false,
|
||||
"image_set": {
|
||||
"schema_version": "missioncore.observatory-executor-image-set/v1",
|
||||
"images": {
|
||||
"travel": "7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f",
|
||||
"parity": "ceb13548617e4bd3f619766bfdff00af3fa5160946b367828da6d2233dcdcba0"
|
||||
}
|
||||
},
|
||||
"learned_models": []
|
||||
},
|
||||
"resource_profile": {
|
||||
"schema_version": "missioncore.observatory-recorded-resource-profile/v1",
|
||||
"profile_id": "worker006-cpu-single-run-v1",
|
||||
"contour_id": "worker-006",
|
||||
"concurrency": 1,
|
||||
"checkpoint_policy": "non-checkpointable",
|
||||
"restart_from_zero": true,
|
||||
"staging_discard_required": true,
|
||||
"resource_release_receipt_required": true
|
||||
},
|
||||
"authority": {
|
||||
"commands_enabled": false,
|
||||
"actuation_allowed": false,
|
||||
"navigation_or_safety_accepted": false,
|
||||
"production_accepted": false
|
||||
}
|
||||
}
|
||||
@@ -275,33 +275,64 @@ execution canaries. Once they prove one automatic run-to-evidence path, the LAB
|
||||
canonicalization slice closes and remaining historical work stays `legacy` unless
|
||||
an integrity or product need justifies a targeted migration.
|
||||
|
||||
## Observatory preparation and Worker dispatch
|
||||
## Observatory durable recorded queue and Worker dispatch
|
||||
|
||||
Selecting a source and a laboratory setup in Observatory is not itself a run.
|
||||
Before a remote executor is installed, Mission Core may persist only an immutable
|
||||
`missioncore.observatory-run-preparation/v1` receipt. That receipt binds the exact
|
||||
source-catalog snapshot, setup id, RunDefinition version and configuration digest
|
||||
to one idempotency key. Its only admitted terminal state is `blocked`; it has no
|
||||
`run_id`, dispatch receipt or execution authority.
|
||||
The public submission contains only `source_session_id`, `setup_id` and an
|
||||
idempotency key. It cannot supply commands, executable text, filesystem paths,
|
||||
container images, model identities, resource limits or priority. The server
|
||||
resolves the allowlisted pair and seals all executable identity into one durable
|
||||
record:
|
||||
|
||||
A real Worker run begins only after a separate durable dispatcher has atomically
|
||||
published an immutable dispatch receipt. The executor must be an additive,
|
||||
allowlisted service owned outside the K1 acquisition path. It may accept bounded
|
||||
versioned data, but never an arbitrary command, filesystem path or PowerShell
|
||||
fragment from the UI. Exact retries return the original receipt; a reused key with
|
||||
different identity fails closed. Transport uncertainty produces an explicit
|
||||
reconciliation state and never an automatic duplicate run.
|
||||
- the current source-catalog snapshot captured at admission, plus immutable source
|
||||
bundle and source-capability-manifest SHA-256 identities;
|
||||
- source-adapter id, version and SHA-256;
|
||||
- setup RunDefinition id, version and SHA-256;
|
||||
- executor release and image SHA-256 identities;
|
||||
- learned-model manifest and resource-profile identities;
|
||||
- checkpoint policy and the server-owned priority class.
|
||||
|
||||
The first executable Observatory setup remains the exact recorded
|
||||
RAVNOVES00/M4.9T5 definition. Its full and integrated successors are different
|
||||
definitions and results. The current RAVNOVES004TREE LAB V1 projection remains
|
||||
replay-only until a new, independently versioned RunDefinition is reconstructed;
|
||||
Mission Core does not invent a configuration for an already published result.
|
||||
An exact retry returns the existing job. Reusing an idempotency key for another
|
||||
identity fails closed. The ordinary lifecycle advances through `accepted`,
|
||||
`queued`, `claimed` and `running`, then terminates as `succeeded` or `failed`.
|
||||
`paused`, `preemption-pending` and `reconciliation-required` are explicit safety
|
||||
branches rather than hidden retries. The durable queue serializes one recorded
|
||||
compute owner and does not equate a queued receipt with a Worker execution
|
||||
receipt.
|
||||
|
||||
Worker telemetry is secondary observation evidence. It does not replace the
|
||||
authoritative run ledger, dispatch receipt, result validation or common laboratory
|
||||
receipt. K1 control, Simulation/Gaussian runtimes and legacy LAB projections are
|
||||
outside this executor boundary and are not restarted or migrated by it.
|
||||
The only currently queueable pair is the exact `RAVNOVES00` source and
|
||||
`M4.9T5 · TRAVEL TGS · CPU-only, без ML` RunDefinition. It seals an explicitly
|
||||
empty learned-model list because TGS is an algorithmic CPU pipeline, not an
|
||||
unknown model dependency. The binding pins the exact source pack rather than a
|
||||
volatile whole-catalog digest; the server captures and seals the current catalog
|
||||
snapshot into each admitted job. `LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39`
|
||||
remains a result-only replay: no independently versioned durable executor exists
|
||||
for that published result, so Mission Core does not invent one. Every other
|
||||
session/setup pair fails closed until its own source adapter, capability manifest,
|
||||
RunDefinition and executor identities are admitted.
|
||||
|
||||
Recorded work has priority rank `100`. A future live K1 lease has rank `0` and
|
||||
closes new recorded claims while it is pending or active. Cooperative executors
|
||||
yield only at an allowlisted checkpoint. A non-checkpointable monolith must be
|
||||
cancelled by the scheduler, discard all staging output and later restart from
|
||||
zero. Cancellation uses a crash-safe two-phase protocol: Mission Core first
|
||||
persists one stable cancellation intent and activates live work only after an
|
||||
identity-bound receipt proves resource release and staging discard. A retry uses
|
||||
the same cancellation identity. A missing callback or receipt remains durably
|
||||
pending for retry; a conflicting or indeterminate receipt enters reconciliation.
|
||||
Every unresolved form blocks live activation and concurrent ownership.
|
||||
An explicit live terminal trigger requeues paused recorded work.
|
||||
|
||||
ADR 0046 records this decision. The durable queue and identity contracts are
|
||||
implemented; Worker claim transport, installation of the exact M4.9T5 executor
|
||||
and wiring from the real K1 lifecycle to live-lease triggers remain pending.
|
||||
Therefore a submitted M4.9T5 job may honestly wait in `queued` without implying
|
||||
that Worker 006 can execute it yet.
|
||||
|
||||
Worker telemetry remains secondary observation evidence. It does not replace the
|
||||
authoritative queue ledger, result validation or common laboratory receipt. K1
|
||||
acquisition and control, the legacy LAB archive, and Simulation/Gaussian runtimes
|
||||
are outside this boundary and are not restarted, migrated or modified by it.
|
||||
|
||||
## Planning discipline
|
||||
|
||||
|
||||
@@ -113,11 +113,11 @@ The next bounded slice adds an explicit laboratory Setup selection beside the so
|
||||
Setup is a versioned, immutable compatibility contract, not a mutable bag of UI parameters. The
|
||||
catalog keeps two independently named historical facts:
|
||||
|
||||
- `M4.9T5 · TRAVEL TGS · полный source-paced shadow` is an archived RunDefinition bound exactly to
|
||||
- `M4.9T5 · TRAVEL TGS · CPU-only, без ML` is an archived RunDefinition bound exactly to
|
||||
`20260720T065719Z_viewer_live`. Its configuration references are content-addressed, its primary
|
||||
visual result remains in legacy LAB, and the integrated-graph result is retained as a compute
|
||||
successor rather than presented as a second viewer.
|
||||
- `LAB V1 · финальный записанный контур восприятия` is the current exact RAV004 result bound to
|
||||
- `LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39` is the current exact RAV004 result bound to
|
||||
`20260828T130511Z_viewer_live`. It predates the product RunDefinition contract and therefore keeps
|
||||
`run_definition = null`; the UI identifies it as an existing result and never fabricates a config
|
||||
digest for it.
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
# ADR 0046: Durable Observatory recorded queue and live K1 priority
|
||||
|
||||
Date: 2026-08-31
|
||||
Status: accepted; durable queue core implemented; Worker and live-trigger wiring pending
|
||||
|
||||
## Context
|
||||
|
||||
Observatory must let an operator apply an admitted laboratory setup to a saved
|
||||
session without turning the browser into an execution client. The current compute
|
||||
host is a shared, finite Worker rather than an elastic pool. Recorded experiments
|
||||
may wait, but a future live K1 perception process must acquire that compute
|
||||
immediately and exclusively when its real acquisition lifecycle begins.
|
||||
|
||||
Historical LAB results are not automatically executable definitions. In
|
||||
particular, the existing LAB V1 result proves a recorded EoMT and DDRNet analysis,
|
||||
but it does not retain one independently versioned, durable executor contract that
|
||||
can safely be reconstructed from its display metadata. Conversely, M4.9T5 has an
|
||||
exact accepted RAVNOVES00 source pack and CPU-only TRAVEL TGS release which can be
|
||||
sealed without inventing learned-model dependencies.
|
||||
|
||||
The queue must also survive process restarts and ambiguity around preemption. A
|
||||
database flag alone cannot prove that a non-checkpointable Worker process released
|
||||
CPU, memory and staging storage. Starting live K1 work before physical cancellation
|
||||
is confirmed would permit two owners of the same constrained resource.
|
||||
|
||||
## Decision
|
||||
|
||||
Mission Core owns a bounded SQLite-backed recorded-job queue. The browser submits
|
||||
only:
|
||||
|
||||
- `source_session_id`;
|
||||
- `setup_id`;
|
||||
- one idempotency key.
|
||||
|
||||
The request contains no command, script, path, environment variable, image,
|
||||
model, resource limit or priority. The server resolves an allowlisted source/setup
|
||||
pair and seals the following identities into the accepted job:
|
||||
|
||||
1. the current source-catalog snapshot SHA-256 captured at admission;
|
||||
2. immutable source-bundle SHA-256;
|
||||
3. source-capability-manifest SHA-256;
|
||||
4. source-adapter id, version and SHA-256;
|
||||
5. setup RunDefinition id, version and SHA-256;
|
||||
6. executor release and container-image SHA-256;
|
||||
7. learned-model release list and model-manifest SHA-256;
|
||||
8. resource-profile SHA-256 and checkpoint policy.
|
||||
|
||||
The queue owns idempotency. An exact retry returns the original job and receipt.
|
||||
The same key with changed identity is a conflict. Jobs advance through `accepted`,
|
||||
`queued`, `claimed` and `running`, then terminate as `succeeded` or `failed`.
|
||||
`paused`, `preemption-pending` and `reconciliation-required` expose safety-relevant
|
||||
branches. Mission Core admits only one active recorded compute owner.
|
||||
|
||||
The queue is an orchestration and evidence boundary, not a Worker transport. A
|
||||
queued job proves durable admission, not remote execution or eventual success.
|
||||
Worker claims, terminal result publication and telemetry must later use a separate
|
||||
authenticated, versioned service boundary; no arbitrary PowerShell or filesystem
|
||||
payload is added to this contract.
|
||||
|
||||
## Current setup admission
|
||||
|
||||
The current executable matrix contains one exact pair:
|
||||
|
||||
- source: `RAVNOVES00`;
|
||||
- setup: `M4.9T5 · TRAVEL TGS · CPU-only, без ML`;
|
||||
- learned-model releases: `[]`;
|
||||
- checkpoint policy: `non-checkpointable`.
|
||||
|
||||
The empty model list is affirmative provenance: this TGS configuration is a
|
||||
CPU-only algorithmic pipeline. It is not a placeholder for unidentified weights.
|
||||
Admission still reads and seals the current source-catalog snapshot and verifies
|
||||
the pinned source bundle and capability manifest before the job is accepted. The
|
||||
binding deliberately does not pin a digest of the whole mutable catalog: the
|
||||
exact compute input is the immutable source pack, while the catalog snapshot is
|
||||
job-specific provenance.
|
||||
|
||||
`LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39` remains result-only. The
|
||||
published evidence stays viewable, but the setup cannot be queued until a new
|
||||
independently versioned source adapter, RunDefinition and executable release are
|
||||
created and accepted. The same fail-closed rule applies to every other
|
||||
session/setup combination. The product may grow toward any admitted setup over
|
||||
any compatible recorded source, but compatibility is evidence, not an assumption.
|
||||
|
||||
## Priority and preemption protocol
|
||||
|
||||
Priority is server-owned:
|
||||
|
||||
- live K1 lease: rank `0`;
|
||||
- recorded replay job: rank `100`.
|
||||
|
||||
A pending or active live lease blocks new recorded claims. A running cooperative
|
||||
job yields at an allowlisted checkpoint and moves to `paused` before live
|
||||
activation. A non-checkpointable job cannot pretend to pause. Its transition is:
|
||||
|
||||
1. persist a stable cancellation intent bound to the job claim, executor release,
|
||||
image, resource profile and live trigger;
|
||||
2. invoke the scheduler-owned physical cancellation boundary;
|
||||
3. persist an identity-bound cancellation receipt proving resource release,
|
||||
staging discard and restart-from-zero;
|
||||
4. only then activate the live K1 lease.
|
||||
|
||||
The cancellation identity is reused after a crash or retry. A missing callback or
|
||||
receipt leaves the durable intent pending for retry; a conflicting or
|
||||
indeterminate receipt moves the record to reconciliation. Both forms block live
|
||||
activation instead of automatically launching duplicate work. When an
|
||||
explicit live terminal trigger completes, fails or cancels the lease, paused
|
||||
recorded jobs return to the queue. A non-checkpointable job starts again from
|
||||
zero; partial staging never becomes evidence.
|
||||
|
||||
## Acceptance boundary
|
||||
|
||||
This decision accepts the durable queue core and its identity, lifecycle,
|
||||
idempotency and preemption contracts. It does not claim that the complete compute
|
||||
loop is deployed. The following remain implementation gates:
|
||||
|
||||
- authenticated Worker claim and result transport;
|
||||
- installation of the exact M4.9T5 executor on the Worker;
|
||||
- binding real K1 acquisition start/terminal events to live-lease triggers;
|
||||
- physical acceptance that cancellation releases the required Worker resources
|
||||
and that recorded work resumes only after the live lease terminates.
|
||||
|
||||
Until those gates close, an M4.9T5 submission may remain honestly `queued`. LAB V1
|
||||
cannot be submitted, and no UI state may present it as executable.
|
||||
|
||||
## Consequences
|
||||
|
||||
- The UI stays a bounded selector and submitter; executable authority remains on
|
||||
the server.
|
||||
- Every accepted job is reproducible from sealed source, setup, executor, model
|
||||
and resource identities rather than mutable host paths.
|
||||
- Live K1 can receive hard priority without silently losing or concurrently
|
||||
running recorded work.
|
||||
- Non-checkpointable replay pays the deliberate cost of discard and
|
||||
restart-from-zero after preemption.
|
||||
- K1 acquisition/control principles, the legacy LAB archive and
|
||||
Simulation/Gaussian remain unchanged.
|
||||
- Adding a setup or source requires an admitted adapter and RunDefinition; a
|
||||
display name or historical result is insufficient.
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -42,6 +42,15 @@ from k1link.observatory import (
|
||||
ObservatoryRunPreparationLedger,
|
||||
load_observatory_run_preparation_ledger,
|
||||
)
|
||||
from k1link.observatory.m49_queue_binding import (
|
||||
M49QueueBindingConfig,
|
||||
M49QueueBindingError,
|
||||
M49RecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueError,
|
||||
)
|
||||
from k1link.sessions import (
|
||||
MaterializedRecording,
|
||||
RecordedCameraFrameService,
|
||||
@@ -242,6 +251,34 @@ OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR: str | None
|
||||
OBSERVATORY_RUN_PREPARATION_LEDGER,
|
||||
OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR,
|
||||
) = load_observatory_run_preparation_ledger(session_store.data_dir)
|
||||
OBSERVATORY_RECORDED_BINDING_SERVICE: M49RecordedQueueBindingService | None
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE: ObservatoryRecordedJobQueue | None
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR: str | None
|
||||
try:
|
||||
if OBSERVATORY_LABORATORY_SETUP_REGISTRY is None:
|
||||
raise M49QueueBindingError("observatory setup registry is unavailable")
|
||||
OBSERVATORY_RECORDED_BINDING_SERVICE = M49RecordedQueueBindingService(
|
||||
data_dir=session_store.data_dir,
|
||||
session_store=session_store,
|
||||
setup_registry=OBSERVATORY_LABORATORY_SETUP_REGISTRY,
|
||||
config=M49QueueBindingConfig.from_file(
|
||||
REPOSITORY_ROOT
|
||||
/ "config"
|
||||
/ "observatory-m49-recorded-queue-binding.json"
|
||||
),
|
||||
)
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE = ObservatoryRecordedJobQueue(
|
||||
session_store.data_dir,
|
||||
definitions=OBSERVATORY_RECORDED_BINDING_SERVICE.definitions,
|
||||
)
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = None
|
||||
except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError) as exc:
|
||||
# Recorded execution remains an optional observation-only slice. A drifted
|
||||
# seal or queue must fail closed without preventing K1, Simulation or legacy
|
||||
# LAB from starting.
|
||||
OBSERVATORY_RECORDED_BINDING_SERVICE = None
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE = None
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = str(exc)
|
||||
simulation_project_store = SimulationProjectStore(session_store.data_dir)
|
||||
simulation_project_service = SimulationProjectService(simulation_project_store)
|
||||
session_artifact_gateway = configured_artifact_gateway(session_store.data_dir)
|
||||
@@ -725,6 +762,9 @@ app.include_router(
|
||||
setup_registry_error=OBSERVATORY_LABORATORY_SETUP_REGISTRY_ERROR,
|
||||
run_preparation_ledger=OBSERVATORY_RUN_PREPARATION_LEDGER,
|
||||
run_preparation_ledger_error=OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR,
|
||||
recorded_binding_service=OBSERVATORY_RECORDED_BINDING_SERVICE,
|
||||
recorded_job_queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
recorded_job_queue_error=OBSERVATORY_RECORDED_JOB_QUEUE_ERROR,
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
|
||||
@@ -17,6 +17,18 @@ from k1link.observatory import (
|
||||
is_admitted_observatory_recorded_result,
|
||||
observatory_run_preparation_request_sha256,
|
||||
)
|
||||
from k1link.observatory.m49_queue_binding import (
|
||||
M49QueueBindingError,
|
||||
M49QueueBindingIntegrityError,
|
||||
M49RecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueCapacityError,
|
||||
ObservatoryRecordedQueueConflictError,
|
||||
ObservatoryRecordedQueueError,
|
||||
ObservatoryRecordedQueueNotFoundError,
|
||||
)
|
||||
from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore
|
||||
from k1link.sessions.models import SessionSummary
|
||||
|
||||
@@ -32,6 +44,19 @@ OBSERVATORY_RUN_PREFLIGHT_REQUEST_SCHEMA: Literal[
|
||||
OBSERVATORY_RUN_PREFLIGHT_SCHEMA: Literal[
|
||||
"missioncore.observatory-run-preflight/v1"
|
||||
] = "missioncore.observatory-run-preflight/v1"
|
||||
OBSERVATORY_RECORDED_RUN_SUBMIT_SCHEMA: Literal[
|
||||
"missioncore.observatory-recorded-run-submit/v1"
|
||||
] = "missioncore.observatory-recorded-run-submit/v1"
|
||||
OBSERVATORY_RECORDED_JOB_LIST_SCHEMA: Literal[
|
||||
"missioncore.observatory-recorded-job-list/v1"
|
||||
] = "missioncore.observatory-recorded-job-list/v1"
|
||||
|
||||
_OBSERVATION_ONLY_AUTHORITY: dict[str, bool] = {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
|
||||
|
||||
class _StrictApiModel(BaseModel):
|
||||
@@ -88,6 +113,25 @@ class ObservatoryRunPreparationRequest(_StrictApiModel):
|
||||
definition_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
|
||||
class ObservatoryRecordedRunSubmitRequest(_StrictApiModel):
|
||||
schema_version: Literal["missioncore.observatory-recorded-run-submit/v1"]
|
||||
idempotency_key: str = Field(
|
||||
min_length=1,
|
||||
max_length=160,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$",
|
||||
)
|
||||
source_session_id: str = Field(
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||
)
|
||||
setup_id: str = Field(
|
||||
min_length=3,
|
||||
max_length=96,
|
||||
pattern=r"^[a-z][a-z0-9-]{2,95}$",
|
||||
)
|
||||
|
||||
|
||||
def build_observatory_router(
|
||||
store: SessionStore,
|
||||
*,
|
||||
@@ -95,6 +139,9 @@ def build_observatory_router(
|
||||
setup_registry_error: str | None = None,
|
||||
run_preparation_ledger: ObservatoryRunPreparationLedger | None = None,
|
||||
run_preparation_ledger_error: str | None = None,
|
||||
recorded_binding_service: M49RecordedQueueBindingService | None = None,
|
||||
recorded_job_queue: ObservatoryRecordedJobQueue | None = None,
|
||||
recorded_job_queue_error: str | None = None,
|
||||
) -> APIRouter:
|
||||
"""Build bounded catalog-only mutations for typed Observatory projections."""
|
||||
|
||||
@@ -224,6 +271,34 @@ def build_observatory_router(
|
||||
assert isinstance(executor, dict)
|
||||
compatible = compatibility.get("compatible") is True
|
||||
existing = preflight.get("outcome") == "existing"
|
||||
queue_binding_ready = False
|
||||
queue_binding_reason: str | None = None
|
||||
binding_service = recorded_binding_service
|
||||
exact_queue_setup = (
|
||||
binding_service is not None
|
||||
and request.setup_id
|
||||
== binding_service.config.setup.setup_id
|
||||
)
|
||||
if (
|
||||
compatible
|
||||
and not existing
|
||||
and binding_service is not None
|
||||
and exact_queue_setup
|
||||
and isinstance(expected_digest, str)
|
||||
):
|
||||
try:
|
||||
binding_service.check(
|
||||
source_session_id=request.source_session_id,
|
||||
setup_id=request.setup_id,
|
||||
definition_sha256=expected_digest,
|
||||
)
|
||||
queue_binding_ready = True
|
||||
except M49QueueBindingError:
|
||||
queue_binding_reason = (
|
||||
"Точная привязка источника и исполняемого релиза не прошла "
|
||||
"проверку целостности."
|
||||
)
|
||||
queueable = queue_binding_ready and recorded_job_queue is not None
|
||||
checks: list[dict[str, Any]] = [
|
||||
{
|
||||
"check_id": "source-compatibility",
|
||||
@@ -247,26 +322,66 @@ def build_observatory_router(
|
||||
},
|
||||
{
|
||||
"check_id": "executor",
|
||||
"outcome": "not-applicable" if existing else "fail",
|
||||
"outcome": (
|
||||
"not-applicable"
|
||||
if existing
|
||||
else "pass"
|
||||
if queue_binding_ready
|
||||
else "fail"
|
||||
),
|
||||
"reason_code": (
|
||||
"existing-result-does-not-require-executor"
|
||||
if existing
|
||||
else "executor-release-sealed"
|
||||
if queue_binding_ready
|
||||
else str(executor.get("reason_code"))
|
||||
),
|
||||
"message": (
|
||||
"Готовый результат открывается без повторного запуска Worker."
|
||||
if existing
|
||||
else (
|
||||
"Исполняемый релиз и его ресурсы запечатаны; отдельный "
|
||||
"Worker service заберёт расчёт из очереди после установки."
|
||||
)
|
||||
if queue_binding_ready
|
||||
else str(executor.get("reason"))
|
||||
),
|
||||
},
|
||||
{
|
||||
"check_id": "durable-queue",
|
||||
"outcome": (
|
||||
"not-applicable"
|
||||
if existing
|
||||
else "pass"
|
||||
if queueable
|
||||
else "fail"
|
||||
),
|
||||
"reason_code": (
|
||||
"existing-result-does-not-require-queue"
|
||||
if existing
|
||||
else "durable-queue-ready"
|
||||
if queueable
|
||||
else "durable-queue-unavailable"
|
||||
),
|
||||
"message": (
|
||||
"Готовый результат не требует постановки в очередь."
|
||||
if existing
|
||||
else "Durable-очередь готова принять расчёт."
|
||||
if queueable
|
||||
else queue_binding_reason
|
||||
or "Этот источник и сетап пока нельзя поставить в очередь."
|
||||
),
|
||||
},
|
||||
]
|
||||
return {
|
||||
"schema_version": OBSERVATORY_RUN_PREFLIGHT_SCHEMA,
|
||||
"source_session_id": request.source_session_id,
|
||||
"setup_id": request.setup_id,
|
||||
"definition_sha256": expected_digest,
|
||||
"outcome": "existing" if existing else "blocked",
|
||||
"submission_allowed": False,
|
||||
"outcome": (
|
||||
"existing" if existing else "queueable" if queueable else "blocked"
|
||||
),
|
||||
"submission_allowed": queueable,
|
||||
"checks": checks,
|
||||
"existing_result_ids": preflight.get("existing_result_ids", []),
|
||||
"executor": executor,
|
||||
@@ -508,6 +623,236 @@ def build_observatory_router(
|
||||
detail="Подготовка расчётов Обсерватории недоступна.",
|
||||
)
|
||||
|
||||
if (
|
||||
setup_registry is not None
|
||||
and recorded_binding_service is not None
|
||||
and recorded_job_queue is not None
|
||||
):
|
||||
|
||||
@router.post("/api/v1/observatory/runs", status_code=202)
|
||||
def submit_observatory_recorded_run(
|
||||
request: ObservatoryRecordedRunSubmitRequest,
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
existing_job = recorded_job_queue.get_by_idempotency_key(
|
||||
request.idempotency_key
|
||||
)
|
||||
except ObservatoryRecordedQueueNotFoundError:
|
||||
existing_job = None
|
||||
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Durable-очередь расчётов недоступна.",
|
||||
) from exc
|
||||
if existing_job is not None:
|
||||
if (
|
||||
existing_job.source_session_id != request.source_session_id
|
||||
or existing_job.setup_id != request.setup_id
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Ключ идемпотентности уже связан с другим расчётом.",
|
||||
)
|
||||
return existing_job.as_dict()
|
||||
|
||||
source = source_summary(request.source_session_id)
|
||||
try:
|
||||
setup_registry.setup(request.setup_id)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="Сетап лаборатории не найден.",
|
||||
) from exc
|
||||
catalog = setup_registry.catalog(
|
||||
source,
|
||||
available_observatory_result_ids=available_observatory_results(
|
||||
request.source_session_id
|
||||
),
|
||||
)
|
||||
setups = catalog.get("setups")
|
||||
if not isinstance(setups, list):
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Каталог сетапов Обсерватории нарушил контракт.",
|
||||
)
|
||||
projected = next(
|
||||
(
|
||||
item
|
||||
for item in setups
|
||||
if isinstance(item, dict) and item.get("setup_id") == request.setup_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if projected is None:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Каталог сетапов Обсерватории нарушил контракт.",
|
||||
)
|
||||
compatibility = projected.get("compatibility")
|
||||
preflight = projected.get("preflight")
|
||||
definition = projected.get("run_definition")
|
||||
if not isinstance(compatibility, dict) or not isinstance(preflight, dict):
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Каталог сетапов Обсерватории нарушил контракт.",
|
||||
)
|
||||
if compatibility.get("compatible") is not True:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Исходная сессия несовместима с выбранным сетапом.",
|
||||
)
|
||||
if preflight.get("outcome") == "existing":
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Точный результат уже существует; новый расчёт не создан.",
|
||||
)
|
||||
if not isinstance(definition, dict):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Для сетапа нет воспроизводимой RunDefinition.",
|
||||
)
|
||||
definition_sha256 = definition.get("definition_sha256")
|
||||
if not isinstance(definition_sha256, str):
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Каталог сетапов Обсерватории нарушил контракт.",
|
||||
)
|
||||
try:
|
||||
admission = recorded_binding_service.admit(
|
||||
source_session_id=request.source_session_id,
|
||||
setup_id=request.setup_id,
|
||||
definition_sha256=definition_sha256,
|
||||
)
|
||||
job, _created = recorded_job_queue.submit(
|
||||
admission.intent(idempotency_key=request.idempotency_key),
|
||||
enqueue=True,
|
||||
)
|
||||
return job.as_dict()
|
||||
except M49QueueBindingIntegrityError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
"Точная привязка источника и исполняемого релиза не прошла "
|
||||
"проверку целостности."
|
||||
),
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueConflictError as exc:
|
||||
try:
|
||||
existing_job = recorded_job_queue.get_by_idempotency_key(
|
||||
request.idempotency_key
|
||||
)
|
||||
except ObservatoryRecordedQueueNotFoundError:
|
||||
existing_job = None
|
||||
except (ObservatoryRecordedQueueError, ValueError) as lookup_exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Durable-очередь расчётов недоступна.",
|
||||
) from lookup_exc
|
||||
if (
|
||||
existing_job is not None
|
||||
and existing_job.source_session_id == request.source_session_id
|
||||
and existing_job.setup_id == request.setup_id
|
||||
):
|
||||
return existing_job.as_dict()
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Ключ идемпотентности уже связан с другим расчётом.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueCapacityError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Квота durable-очереди расчётов исчерпана.",
|
||||
) from exc
|
||||
except (M49QueueBindingError, ObservatoryRecordedQueueError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Durable-очередь расчётов недоступна.",
|
||||
) from exc
|
||||
|
||||
@router.get("/api/v1/observatory/runs")
|
||||
def list_observatory_recorded_runs(
|
||||
source_session_id: str = Query(
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||
),
|
||||
setup_id: str = Query(
|
||||
min_length=3,
|
||||
max_length=96,
|
||||
pattern=r"^[a-z][a-z0-9-]{2,95}$",
|
||||
),
|
||||
limit: int = Query(default=20, ge=1, le=100),
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
jobs = recorded_job_queue.list_jobs(
|
||||
source_session_id=source_session_id,
|
||||
setup_id=setup_id,
|
||||
limit=limit,
|
||||
)
|
||||
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Durable-очередь расчётов недоступна.",
|
||||
) from exc
|
||||
return {
|
||||
"schema_version": OBSERVATORY_RECORDED_JOB_LIST_SCHEMA,
|
||||
"items": [job.as_dict() for job in jobs],
|
||||
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
|
||||
@router.get("/api/v1/observatory/runs/{job_id}")
|
||||
def get_observatory_recorded_run(
|
||||
job_id: str = ApiPath(
|
||||
min_length=48,
|
||||
max_length=48,
|
||||
pattern=r"^observatory-run-[a-f0-9]{32}$",
|
||||
),
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
return recorded_job_queue.get(job_id).as_dict()
|
||||
except ObservatoryRecordedQueueNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="Расчёт Обсерватории не найден.",
|
||||
) from exc
|
||||
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Durable-очередь расчётов недоступна.",
|
||||
) from exc
|
||||
|
||||
elif recorded_job_queue_error is not None:
|
||||
|
||||
@router.post("/api/v1/observatory/runs")
|
||||
def unavailable_observatory_recorded_run(
|
||||
request: ObservatoryRecordedRunSubmitRequest,
|
||||
) -> None:
|
||||
del request
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Durable-очередь расчётов недоступна.",
|
||||
)
|
||||
|
||||
@router.get("/api/v1/observatory/runs")
|
||||
def unavailable_observatory_recorded_runs(
|
||||
source_session_id: str = Query(
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||
),
|
||||
setup_id: str = Query(
|
||||
min_length=3,
|
||||
max_length=96,
|
||||
pattern=r"^[a-z][a-z0-9-]{2,95}$",
|
||||
),
|
||||
limit: int = Query(default=20, ge=1, le=100),
|
||||
) -> None:
|
||||
del source_session_id, setup_id, limit
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Durable-очередь расчётов недоступна.",
|
||||
)
|
||||
|
||||
@router.patch(
|
||||
"/api/v1/observatory/lab-projections/{session_id}",
|
||||
response_model=ObservatoryProjectionDocument,
|
||||
|
||||
@@ -0,0 +1,555 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
import k1link.observatory.m49_queue_binding as binding_module
|
||||
from k1link.artifact_gateway import (
|
||||
ArtifactMember,
|
||||
CentralArtifactStore,
|
||||
LocalArtifactCache,
|
||||
)
|
||||
from k1link.observatory.m49_queue_binding import (
|
||||
OBSERVATORY_EXECUTOR_BINDING_SCHEMA,
|
||||
OBSERVATORY_SOURCE_BUNDLE_SCHEMA,
|
||||
OBSERVATORY_SOURCE_CAPABILITY_SCHEMA,
|
||||
SOURCE_DOCUMENT_STORE_DIRECTORY,
|
||||
M49QueueBindingConfig,
|
||||
M49QueueBindingIntegrityError,
|
||||
M49RecordedQueueBindingService,
|
||||
M49SourcePackIdentity,
|
||||
M49SourcePackVerifier,
|
||||
_verify_source_pack_from_cache,
|
||||
)
|
||||
from k1link.observatory.setups import LaboratorySetupRegistry
|
||||
from k1link.sessions.models import (
|
||||
SessionArtifact,
|
||||
SessionDetail,
|
||||
SessionSource,
|
||||
SessionSummary,
|
||||
)
|
||||
from k1link.sessions.store import SessionStore
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
BINDING_PATH = (
|
||||
REPOSITORY_ROOT / "config" / "observatory-m49-recorded-queue-binding.json"
|
||||
)
|
||||
SETUP_REGISTRY_PATH = (
|
||||
REPOSITORY_ROOT / "config" / "observatory-laboratory-setups.json"
|
||||
)
|
||||
SOURCE_SESSION_ID = "20260720T065719Z_viewer_live"
|
||||
SOURCE_LABEL = "RAVNOVES00"
|
||||
SETUP_ID = "m49-tgs-full-shadow-v1"
|
||||
DEFINITION_SHA256 = (
|
||||
"836a66639e69f7ea00de2a9111c1c6c9c3c00f1abd726f1df596aeb4e3a88ae6"
|
||||
)
|
||||
SOURCE_PACK_SHA256 = (
|
||||
"0685d24219d8236caf8b7f1685e93f6d6b59e7fd015a768d88a92bbe8b154944"
|
||||
)
|
||||
EXECUTOR_ARTIFACT_SHA256 = (
|
||||
"5e0ea16c7a5cc760463836718b0cd8b0006ffc4b202e5f706a20a86ef2f912ab"
|
||||
)
|
||||
CODE_REVISION = "40c850b167dda366d8aa45d828520168affaf9fd"
|
||||
TRAVEL_IMAGE_SHA256 = (
|
||||
"7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f"
|
||||
)
|
||||
PARITY_IMAGE_SHA256 = (
|
||||
"ceb13548617e4bd3f619766bfdff00af3fa5160946b367828da6d2233dcdcba0"
|
||||
)
|
||||
SOURCE_CATALOG_SHA256 = (
|
||||
"24207af81b67de515ba9f1b899577bd877a00d058c3472269508980541b1e185"
|
||||
)
|
||||
|
||||
|
||||
class _SessionStore:
|
||||
def __init__(
|
||||
self,
|
||||
data_dir: Path,
|
||||
detail: SessionDetail,
|
||||
catalog_sha256: str,
|
||||
) -> None:
|
||||
self.data_dir = data_dir.resolve()
|
||||
self.detail = detail
|
||||
self.catalog_sha256 = catalog_sha256
|
||||
|
||||
def get_session_with_catalog_snapshot(
|
||||
self, session_id: str
|
||||
) -> tuple[SessionDetail, str]:
|
||||
if session_id != self.detail.summary.session_id:
|
||||
raise KeyError(session_id)
|
||||
return self.detail, self.catalog_sha256
|
||||
|
||||
|
||||
def _canonical_sha256(value: object) -> str:
|
||||
payload = json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode()
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def test_first_immutable_contract_store_syncs_directory_chain_bottom_up(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
payload = b'{"schema_version":"test/v1"}'
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
root = tmp_path / "observatory-source-contracts"
|
||||
synced: list[Path] = []
|
||||
monkeypatch.setattr(
|
||||
binding_module,
|
||||
"_fsync_directory",
|
||||
lambda path: synced.append(path),
|
||||
)
|
||||
|
||||
binding_module._write_immutable_document(root, digest, payload)
|
||||
|
||||
assert synced == [
|
||||
root / "objects" / "sha256" / digest[:2],
|
||||
root / "objects" / "sha256",
|
||||
root / "objects",
|
||||
root,
|
||||
tmp_path,
|
||||
]
|
||||
|
||||
|
||||
def test_existing_immutable_contract_replays_directory_chain_sync(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
payload = b'{"schema_version":"test/v1"}'
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
root = tmp_path / "observatory-source-contracts"
|
||||
binding_module._write_immutable_document(root, digest, payload)
|
||||
synced: list[Path] = []
|
||||
monkeypatch.setattr(
|
||||
binding_module,
|
||||
"_fsync_directory",
|
||||
lambda path: synced.append(path),
|
||||
)
|
||||
|
||||
binding_module._write_immutable_document(root, digest, payload)
|
||||
|
||||
assert synced == [
|
||||
root / "objects" / "sha256" / digest[:2],
|
||||
root / "objects" / "sha256",
|
||||
root / "objects",
|
||||
root,
|
||||
tmp_path,
|
||||
]
|
||||
|
||||
|
||||
def test_immutable_contract_store_rejects_intermediate_symlink(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
payload = b'{"schema_version":"test/v1"}'
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
root = tmp_path / "observatory-source-contracts"
|
||||
outside = tmp_path / "outside"
|
||||
root.mkdir()
|
||||
outside.mkdir()
|
||||
(root / "objects").symlink_to(outside, target_is_directory=True)
|
||||
|
||||
with pytest.raises(
|
||||
M49QueueBindingIntegrityError,
|
||||
match="store directory is invalid",
|
||||
):
|
||||
binding_module._write_immutable_document(root, digest, payload)
|
||||
|
||||
assert list(outside.iterdir()) == []
|
||||
|
||||
|
||||
def _detail() -> SessionDetail:
|
||||
summary = SessionSummary(
|
||||
session_id=SOURCE_SESSION_ID,
|
||||
display_name=SOURCE_LABEL,
|
||||
status="ready",
|
||||
started_at_utc="2026-07-20T06:57:20.888Z",
|
||||
completed_at_utc="2026-07-20T07:06:16.599Z",
|
||||
duration_seconds=535.717620042,
|
||||
modalities=("point-cloud", "trajectory", "video"),
|
||||
source_count=3,
|
||||
total_bytes=100,
|
||||
replayable=True,
|
||||
origin="xgrids-k1.viewer-live.evidence",
|
||||
)
|
||||
sources = (
|
||||
SessionSource(
|
||||
source_id="sensor.camera.right",
|
||||
semantic_channel_id="camera.video.recorded",
|
||||
modality="video",
|
||||
status="recorded",
|
||||
seekable=True,
|
||||
artifact_id="recorded-video",
|
||||
),
|
||||
SessionSource(
|
||||
source_id="sensor.lidar.primary",
|
||||
semantic_channel_id="spatial.point-cloud.recorded",
|
||||
modality="point-cloud",
|
||||
status="recorded",
|
||||
seekable=True,
|
||||
artifact_id="raw-primary",
|
||||
),
|
||||
SessionSource(
|
||||
source_id="spatial.trajectory",
|
||||
semantic_channel_id="spatial.pose.recorded",
|
||||
modality="trajectory",
|
||||
status="recorded",
|
||||
seekable=True,
|
||||
artifact_id="raw-primary",
|
||||
),
|
||||
)
|
||||
artifacts = (
|
||||
SessionArtifact(
|
||||
artifact_id="raw-primary",
|
||||
kind="raw-transport",
|
||||
media_type="application/x-nodedc-k1mqtt",
|
||||
byte_length=64,
|
||||
sha256="d" * 64,
|
||||
integrity_status="verified",
|
||||
),
|
||||
SessionArtifact(
|
||||
artifact_id="recorded-video",
|
||||
kind="recorded-video",
|
||||
media_type="video/mp4",
|
||||
byte_length=36,
|
||||
sha256=None,
|
||||
integrity_status="validated-structure",
|
||||
),
|
||||
)
|
||||
return SessionDetail(
|
||||
summary=summary,
|
||||
sources=sources,
|
||||
artifacts=artifacts,
|
||||
plugin_id="xgrids-k1",
|
||||
archive_id="viewer-live",
|
||||
)
|
||||
|
||||
|
||||
def _registry() -> LaboratorySetupRegistry:
|
||||
return LaboratorySetupRegistry.from_file(
|
||||
SETUP_REGISTRY_PATH,
|
||||
repository_root=REPOSITORY_ROOT,
|
||||
)
|
||||
|
||||
|
||||
def _artifact_cache(
|
||||
data_dir: Path,
|
||||
tmp_path: Path,
|
||||
payload: bytes,
|
||||
identity: M49SourcePackIdentity,
|
||||
) -> LocalArtifactCache:
|
||||
source = tmp_path / "source-pack.npz"
|
||||
source.write_bytes(payload)
|
||||
central = CentralArtifactStore(tmp_path / "central", create=True)
|
||||
published = central.publish_file(source)
|
||||
assert published.sha256 == identity.sha256
|
||||
cache = LocalArtifactCache(
|
||||
data_dir / "artifact-cache",
|
||||
max_bytes=1024 * 1024,
|
||||
free_space_reserve_bytes=0,
|
||||
)
|
||||
member = ArtifactMember(
|
||||
role="lidar-source-pack",
|
||||
media_type=identity.media_type,
|
||||
sha256=published.sha256,
|
||||
byte_length=published.byte_length,
|
||||
)
|
||||
cache.fetch(central, member, pin_id=f"session:{SOURCE_SESSION_ID}")
|
||||
return cache
|
||||
|
||||
|
||||
def _accept_source_pack(
|
||||
_cache: LocalArtifactCache,
|
||||
identity: M49SourcePackIdentity,
|
||||
) -> None:
|
||||
assert identity.sha256 == SOURCE_PACK_SHA256
|
||||
assert identity.byte_length == 72_996_000
|
||||
|
||||
|
||||
def _service(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
catalog_sha256: str = SOURCE_CATALOG_SHA256,
|
||||
setup_registry: LaboratorySetupRegistry | None = None,
|
||||
source_pack_verifier: M49SourcePackVerifier = _accept_source_pack,
|
||||
) -> tuple[M49RecordedQueueBindingService, LocalArtifactCache]:
|
||||
data_dir = tmp_path / "data"
|
||||
data_dir.mkdir()
|
||||
config = M49QueueBindingConfig.from_file(BINDING_PATH)
|
||||
cache = LocalArtifactCache(
|
||||
data_dir / "artifact-cache",
|
||||
max_bytes=1024 * 1024,
|
||||
free_space_reserve_bytes=0,
|
||||
)
|
||||
store = _SessionStore(data_dir, _detail(), catalog_sha256)
|
||||
service = M49RecordedQueueBindingService(
|
||||
data_dir=data_dir,
|
||||
session_store=cast(SessionStore, store),
|
||||
setup_registry=setup_registry or _registry(),
|
||||
config=config,
|
||||
artifact_cache=cache,
|
||||
source_pack_verifier=source_pack_verifier,
|
||||
)
|
||||
return service, cache
|
||||
|
||||
|
||||
def test_bundled_binding_freezes_the_accepted_exact_m49_identity() -> None:
|
||||
config = M49QueueBindingConfig.from_file(BINDING_PATH)
|
||||
definition = config.recorded_definition()
|
||||
image_set = config.image_set_document()
|
||||
resource = config.resource_profile.document()
|
||||
|
||||
assert config.source.session_id == SOURCE_SESSION_ID
|
||||
assert config.source.label == SOURCE_LABEL
|
||||
assert config.source.source_pack.sha256 == SOURCE_PACK_SHA256
|
||||
assert config.source.source_pack.byte_length == 72_996_000
|
||||
assert config.setup.setup_id == SETUP_ID
|
||||
assert config.setup.definition_sha256 == DEFINITION_SHA256
|
||||
assert config.executor.artifact_sha256 == EXECUTOR_ARTIFACT_SHA256
|
||||
assert config.executor.code_revision == CODE_REVISION
|
||||
assert config.executor.service_installed is False
|
||||
assert image_set["images"] == [
|
||||
{"role": "parity", "sha256": PARITY_IMAGE_SHA256},
|
||||
{"role": "travel", "sha256": TRAVEL_IMAGE_SHA256},
|
||||
]
|
||||
assert definition.executor_image_sha256 == _canonical_sha256(image_set)
|
||||
assert definition.model_release_ids == ()
|
||||
assert definition.learned_models == ()
|
||||
assert definition.checkpoint_policy == "non-checkpointable"
|
||||
assert definition.allowed_checkpoints == ()
|
||||
assert resource["restart_from_zero"] is True
|
||||
assert resource["staging_discard_required"] is True
|
||||
assert resource["resource_release_receipt_required"] is True
|
||||
|
||||
|
||||
def test_check_is_read_only_and_admit_seals_path_free_source_contracts(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service, cache = _service(tmp_path)
|
||||
document_root = service.data_dir / SOURCE_DOCUMENT_STORE_DIRECTORY
|
||||
cache_database_mtime_ns = cache.database_path.stat().st_mtime_ns
|
||||
|
||||
checked = service.check(
|
||||
source_session_id=SOURCE_SESSION_ID,
|
||||
setup_id=SETUP_ID,
|
||||
definition_sha256=DEFINITION_SHA256,
|
||||
)
|
||||
|
||||
assert not document_root.exists()
|
||||
assert cache.database_path.stat().st_mtime_ns == cache_database_mtime_ns
|
||||
assert checked.source_bundle.schema_version == OBSERVATORY_SOURCE_BUNDLE_SCHEMA
|
||||
assert (
|
||||
checked.capability_manifest.schema_version
|
||||
== OBSERVATORY_SOURCE_CAPABILITY_SCHEMA
|
||||
)
|
||||
assert checked.executor_binding.schema_version == OBSERVATORY_EXECUTOR_BINDING_SCHEMA
|
||||
assert checked.source_catalog_sha256 == SOURCE_CATALOG_SHA256
|
||||
assert checked.definition.definition_sha256 == DEFINITION_SHA256
|
||||
assert checked.definition.model_release_ids == ()
|
||||
assert checked.definition.checkpoint_policy == "non-checkpointable"
|
||||
assert checked.source_bundle.document()["members"] == [
|
||||
{
|
||||
"role": "lidar-source-pack",
|
||||
"artifact_id": service.config.source.source_pack.artifact_id,
|
||||
"media_type": service.config.source.source_pack.media_type,
|
||||
"sha256": service.config.source.source_pack.sha256,
|
||||
"byte_length": service.config.source.source_pack.byte_length,
|
||||
}
|
||||
]
|
||||
assert "path" not in json.dumps(checked.as_dict()).lower()
|
||||
|
||||
admitted = service.admit(
|
||||
source_session_id=SOURCE_SESSION_ID,
|
||||
setup_id=SETUP_ID,
|
||||
definition_sha256=DEFINITION_SHA256,
|
||||
)
|
||||
repeated = service.admit(
|
||||
source_session_id=SOURCE_SESSION_ID,
|
||||
setup_id=SETUP_ID,
|
||||
definition_sha256=DEFINITION_SHA256,
|
||||
)
|
||||
sealed_files = tuple(document_root.rglob("*.json"))
|
||||
|
||||
assert admitted == checked
|
||||
assert repeated == admitted
|
||||
assert len(sealed_files) == 3
|
||||
assert {path.stem for path in sealed_files} == {
|
||||
admitted.source_bundle.sha256,
|
||||
admitted.capability_manifest.sha256,
|
||||
admitted.executor_binding.sha256,
|
||||
}
|
||||
assert all(path.read_bytes() in {
|
||||
admitted.source_bundle.payload,
|
||||
admitted.capability_manifest.payload,
|
||||
admitted.executor_binding.payload,
|
||||
} for path in sealed_files)
|
||||
assert service.definitions.resolve(SETUP_ID, DEFINITION_SHA256) == (
|
||||
admitted.definition
|
||||
)
|
||||
intent = admitted.intent(idempotency_key="m49-recorded-submit-001")
|
||||
assert intent.source_session_id == SOURCE_SESSION_ID
|
||||
assert intent.source_catalog_sha256 == SOURCE_CATALOG_SHA256
|
||||
assert intent.source_bundle_sha256 == admitted.source_bundle.sha256
|
||||
assert intent.source_capability_manifest_sha256 == (
|
||||
admitted.capability_manifest.sha256
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("source_session_id", "setup_id", "definition_sha256", "message"),
|
||||
[
|
||||
("other-session", SETUP_ID, DEFINITION_SHA256, "exact source session"),
|
||||
(SOURCE_SESSION_ID, "other-setup", DEFINITION_SHA256, "exact setup"),
|
||||
(SOURCE_SESSION_ID, SETUP_ID, "f" * 64, "definition digest"),
|
||||
],
|
||||
)
|
||||
def test_binding_rejects_every_other_session_setup_or_definition(
|
||||
tmp_path: Path,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
message: str,
|
||||
) -> None:
|
||||
service, _cache = _service(tmp_path)
|
||||
|
||||
with pytest.raises(M49QueueBindingIntegrityError, match=message):
|
||||
service.check(
|
||||
source_session_id=source_session_id,
|
||||
setup_id=setup_id,
|
||||
definition_sha256=definition_sha256,
|
||||
)
|
||||
|
||||
|
||||
def test_binding_captures_the_current_session_catalog_snapshot(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service, _cache = _service(tmp_path, catalog_sha256="e" * 64)
|
||||
|
||||
admission = service.check(
|
||||
source_session_id=SOURCE_SESSION_ID,
|
||||
setup_id=SETUP_ID,
|
||||
definition_sha256=DEFINITION_SHA256,
|
||||
)
|
||||
|
||||
assert admission.source_catalog_sha256 == "e" * 64
|
||||
assert admission.source_bundle.document()["catalog_sha256"] == "e" * 64
|
||||
|
||||
|
||||
def test_binding_fails_closed_when_the_cached_source_pack_changes(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
payload = b"sealed source pack fixture\n"
|
||||
identity = M49SourcePackIdentity(
|
||||
artifact_id="fixture-source-pack",
|
||||
sha256=hashlib.sha256(payload).hexdigest(),
|
||||
byte_length=len(payload),
|
||||
media_type="application/vnd.nodedc.lidar-source-pack+npz",
|
||||
expected_timeline_frames=2,
|
||||
expected_available_lidar_frames=1,
|
||||
)
|
||||
data_dir = tmp_path / "data"
|
||||
data_dir.mkdir()
|
||||
cache = _artifact_cache(data_dir, tmp_path, payload, identity)
|
||||
_verify_source_pack_from_cache(cache, identity)
|
||||
cache.object_path(identity.sha256).write_bytes(b"x" * len(payload))
|
||||
|
||||
with pytest.raises(M49QueueBindingIntegrityError, match="artifact-cache metadata"):
|
||||
_verify_source_pack_from_cache(cache, identity)
|
||||
|
||||
|
||||
def test_binding_fails_closed_when_the_setup_registry_definition_changes(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
document = json.loads(SETUP_REGISTRY_PATH.read_text(encoding="utf-8"))
|
||||
document["setups"][0]["run_definition"]["definition_id"] = (
|
||||
"m49-tgs-full-shadow-drift"
|
||||
)
|
||||
path = tmp_path / "setup-registry.json"
|
||||
path.write_text(json.dumps(document), encoding="utf-8")
|
||||
drifted_registry = LaboratorySetupRegistry.from_file(
|
||||
path,
|
||||
repository_root=REPOSITORY_ROOT,
|
||||
)
|
||||
service, _cache = _service(tmp_path, setup_registry=drifted_registry)
|
||||
|
||||
with pytest.raises(M49QueueBindingIntegrityError, match="setup definition changed"):
|
||||
service.check(
|
||||
source_session_id=SOURCE_SESSION_ID,
|
||||
setup_id=SETUP_ID,
|
||||
definition_sha256=DEFINITION_SHA256,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutation",
|
||||
["unknown-key", "installed", "learned-model", "source-session"],
|
||||
)
|
||||
def test_binding_loader_rejects_authority_or_contract_expansion(
|
||||
tmp_path: Path,
|
||||
mutation: str,
|
||||
) -> None:
|
||||
document = json.loads(BINDING_PATH.read_text(encoding="utf-8"))
|
||||
if mutation == "unknown-key":
|
||||
document["command"] = "run arbitrary payload"
|
||||
elif mutation == "installed":
|
||||
document["executor"]["service_installed"] = True
|
||||
elif mutation == "learned-model":
|
||||
document["executor"]["learned_models"] = ["invented-model"]
|
||||
else:
|
||||
document["source"]["session_id"] = "different-source"
|
||||
path = tmp_path / "invalid-binding.json"
|
||||
path.write_text(json.dumps(document), encoding="utf-8")
|
||||
|
||||
with pytest.raises(M49QueueBindingIntegrityError):
|
||||
M49QueueBindingConfig.from_file(path)
|
||||
|
||||
|
||||
def test_binding_requires_the_session_store_data_dir_artifact_cache(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service, cache = _service(tmp_path)
|
||||
config = service.config
|
||||
foreign = LocalArtifactCache(
|
||||
tmp_path / "foreign-cache",
|
||||
max_bytes=1024 * 1024,
|
||||
free_space_reserve_bytes=0,
|
||||
)
|
||||
|
||||
with pytest.raises(M49QueueBindingIntegrityError, match="data-dir artifact cache"):
|
||||
M49RecordedQueueBindingService(
|
||||
data_dir=service.data_dir,
|
||||
session_store=service._session_store, # noqa: SLF001 - exact boundary test
|
||||
setup_registry=_registry(),
|
||||
config=config,
|
||||
artifact_cache=foreign,
|
||||
)
|
||||
|
||||
assert cache.root == service.data_dir / "artifact-cache"
|
||||
|
||||
|
||||
def test_binding_wraps_unavailable_default_artifact_cache(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
data_dir = tmp_path / "data"
|
||||
data_dir.mkdir()
|
||||
(data_dir / "artifact-cache").write_text("not a directory", encoding="utf-8")
|
||||
store = _SessionStore(data_dir, _detail(), SOURCE_CATALOG_SHA256)
|
||||
|
||||
with pytest.raises(
|
||||
M49QueueBindingIntegrityError,
|
||||
match="artifact cache is unavailable",
|
||||
):
|
||||
M49RecordedQueueBindingService(
|
||||
data_dir=data_dir,
|
||||
session_store=cast(SessionStore, store),
|
||||
setup_registry=_registry(),
|
||||
config=M49QueueBindingConfig.from_file(BINDING_PATH),
|
||||
)
|
||||
@@ -0,0 +1,728 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
LIVE_K1_PRIORITY_RANK,
|
||||
RECORDED_JOB_DATABASE_NAME,
|
||||
RECORDED_PRIORITY_RANK,
|
||||
ObservatoryLiveLeaseIntent,
|
||||
ObservatoryNonCheckpointableCancellationReceipt,
|
||||
ObservatoryRecordedJobIntent,
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedPreemptionError,
|
||||
ObservatoryRecordedQueueBusyError,
|
||||
ObservatoryRecordedQueueConflictError,
|
||||
ObservatoryRecordedQueueIntegrityError,
|
||||
ObservatoryRecordedQueueStaleClaimError,
|
||||
RecordedRunDefinition,
|
||||
RecordedRunDefinitionRegistry,
|
||||
)
|
||||
|
||||
NOW = "2026-08-30T21:00:00.000Z"
|
||||
DEFINITION_SHA = "a" * 64
|
||||
NON_CHECKPOINTABLE_DEFINITION_SHA = "b" * 64
|
||||
ADAPTER_SHA = "c" * 64
|
||||
CATALOG_SHA = "d" * 64
|
||||
RESULT_SHA = "e" * 64
|
||||
LIVE_EPOCH_SHA = "f" * 64
|
||||
SOURCE_BUNDLE_SHA = "1" * 64
|
||||
SOURCE_CAPABILITIES_SHA = "2" * 64
|
||||
EXECUTOR_RELEASE_SHA = "3" * 64
|
||||
EXECUTOR_IMAGE_SHA = "4" * 64
|
||||
MODEL_MANIFEST_SHA = "5" * 64
|
||||
RESOURCE_PROFILE_SHA = "6" * 64
|
||||
|
||||
|
||||
def _definitions() -> RecordedRunDefinitionRegistry:
|
||||
return RecordedRunDefinitionRegistry(
|
||||
(
|
||||
RecordedRunDefinition(
|
||||
setup_id="travel-tgs-eomt-v1",
|
||||
definition_id="travel-tgs-eomt",
|
||||
definition_version=1,
|
||||
definition_sha256=DEFINITION_SHA,
|
||||
source_adapter_id="sealed-session-bundle",
|
||||
source_adapter_version=1,
|
||||
source_adapter_sha256=ADAPTER_SHA,
|
||||
executor_release_id="travel-tgs-executor-v1",
|
||||
executor_release_sha256=EXECUTOR_RELEASE_SHA,
|
||||
executor_image_sha256=EXECUTOR_IMAGE_SHA,
|
||||
model_release_ids=("eomt-v1", "travel-tgs-v1"),
|
||||
model_manifest_sha256=MODEL_MANIFEST_SHA,
|
||||
resource_profile_id="worker006-single-gpu-v1",
|
||||
resource_profile_sha256=RESOURCE_PROFILE_SHA,
|
||||
checkpoint_policy="cooperative",
|
||||
allowed_checkpoints=(
|
||||
"source-bundle-ready",
|
||||
"model-batch-finished",
|
||||
"evidence-sealed",
|
||||
),
|
||||
),
|
||||
RecordedRunDefinition(
|
||||
setup_id="legacy-monolith-v1",
|
||||
definition_id="legacy-monolith",
|
||||
definition_version=1,
|
||||
definition_sha256=NON_CHECKPOINTABLE_DEFINITION_SHA,
|
||||
source_adapter_id="sealed-session-bundle",
|
||||
source_adapter_version=1,
|
||||
source_adapter_sha256=ADAPTER_SHA,
|
||||
executor_release_id="legacy-monolith-executor-v1",
|
||||
executor_release_sha256=EXECUTOR_RELEASE_SHA,
|
||||
executor_image_sha256=EXECUTOR_IMAGE_SHA,
|
||||
model_release_ids=("legacy-monolith-v1",),
|
||||
model_manifest_sha256=MODEL_MANIFEST_SHA,
|
||||
resource_profile_id="worker006-single-gpu-v1",
|
||||
resource_profile_sha256=RESOURCE_PROFILE_SHA,
|
||||
checkpoint_policy="non-checkpointable",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _queue(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
with_non_checkpointable_preemptor: bool = False,
|
||||
) -> ObservatoryRecordedJobQueue:
|
||||
preemptor = None
|
||||
if with_non_checkpointable_preemptor:
|
||||
def preemptor(request):
|
||||
return ObservatoryNonCheckpointableCancellationReceipt.sealed(
|
||||
request,
|
||||
cancellation_id=f"cancel-{request.cancellation_request_id}",
|
||||
)
|
||||
|
||||
return ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=_definitions(),
|
||||
clock=lambda: NOW,
|
||||
non_checkpointable_preemptor=preemptor,
|
||||
)
|
||||
|
||||
|
||||
def _intent(
|
||||
*,
|
||||
idempotency_key: str = "recorded-request-001",
|
||||
source_session_id: str = "20260828T130511Z_viewer_live",
|
||||
source_catalog_sha256: str = CATALOG_SHA,
|
||||
setup_id: str = "travel-tgs-eomt-v1",
|
||||
definition_sha256: str = DEFINITION_SHA,
|
||||
) -> ObservatoryRecordedJobIntent:
|
||||
return ObservatoryRecordedJobIntent(
|
||||
idempotency_key=idempotency_key,
|
||||
source_session_id=source_session_id,
|
||||
source_catalog_sha256=source_catalog_sha256,
|
||||
source_bundle_sha256=SOURCE_BUNDLE_SHA,
|
||||
source_capability_manifest_sha256=SOURCE_CAPABILITIES_SHA,
|
||||
setup_id=setup_id,
|
||||
definition_sha256=definition_sha256,
|
||||
)
|
||||
|
||||
|
||||
def _live_intent(
|
||||
*,
|
||||
trigger_id: str = "k1-live-start-001",
|
||||
live_session_id: str = "20260830T210000Z_k1_live",
|
||||
acquisition_epoch_sha256: str = LIVE_EPOCH_SHA,
|
||||
) -> ObservatoryLiveLeaseIntent:
|
||||
return ObservatoryLiveLeaseIntent(
|
||||
trigger_id=trigger_id,
|
||||
live_session_id=live_session_id,
|
||||
acquisition_epoch_sha256=acquisition_epoch_sha256,
|
||||
)
|
||||
|
||||
|
||||
def _running_job(
|
||||
queue: ObservatoryRecordedJobQueue,
|
||||
*,
|
||||
intent: ObservatoryRecordedJobIntent | None = None,
|
||||
claim_request_id: str = "worker-claim-001",
|
||||
):
|
||||
job, created = queue.submit(intent or _intent())
|
||||
assert created is True
|
||||
queue.enqueue(job.job_id)
|
||||
claim = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id=claim_request_id
|
||||
)
|
||||
assert claim is not None
|
||||
running = queue.start(job.job_id, claim_token=claim.claim_token)
|
||||
assert running.state == "running"
|
||||
return running, claim
|
||||
|
||||
|
||||
def test_submission_is_durable_exactly_idempotent_and_path_free(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
first, first_created = queue.submit(_intent())
|
||||
second, second_created = queue.submit(_intent())
|
||||
restored = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=_definitions(),
|
||||
clock=lambda: NOW,
|
||||
).get(first.job_id)
|
||||
|
||||
assert first_created is True
|
||||
assert second_created is False
|
||||
assert second == first
|
||||
assert restored == first
|
||||
assert first.state == "accepted"
|
||||
assert first.priority_class == "recorded"
|
||||
assert first.priority_rank == RECORDED_PRIORITY_RANK
|
||||
assert first.source_adapter_id == "sealed-session-bundle"
|
||||
assert first.checkpoint_policy == "cooperative"
|
||||
projection = first.as_dict()
|
||||
assert projection["priority"] == {
|
||||
"class": "recorded",
|
||||
"rank": RECORDED_PRIORITY_RANK,
|
||||
"server_owned": True,
|
||||
}
|
||||
assert "path" not in str(projection).lower()
|
||||
assert not hasattr(_intent(), "command")
|
||||
assert not hasattr(_intent(), "image")
|
||||
assert projection["source"]["bundle_sha256"] == SOURCE_BUNDLE_SHA
|
||||
assert projection["source"]["capability_manifest_sha256"] == (
|
||||
SOURCE_CAPABILITIES_SHA
|
||||
)
|
||||
assert projection["executor"]["release_sha256"] == EXECUTOR_RELEASE_SHA
|
||||
assert projection["executor"]["image_sha256"] == EXECUTOR_IMAGE_SHA
|
||||
assert projection["executor"]["model_manifest_sha256"] == MODEL_MANIFEST_SHA
|
||||
assert projection["executor"]["resource_profile_sha256"] == (
|
||||
RESOURCE_PROFILE_SHA
|
||||
)
|
||||
assert queue.database_path.name == RECORDED_JOB_DATABASE_NAME
|
||||
assert queue.database_path.stat().st_mode & 0o777 == 0o600
|
||||
|
||||
with pytest.raises(ObservatoryRecordedQueueConflictError, match="idempotency"):
|
||||
queue.submit(replace(_intent(), source_session_id="another-session"))
|
||||
|
||||
|
||||
def test_submission_and_enqueue_can_commit_as_one_idempotent_transaction(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
|
||||
first, created = queue.submit(_intent(), enqueue=True)
|
||||
repeated, repeated_created = queue.submit(_intent(), enqueue=True)
|
||||
|
||||
assert created is True
|
||||
assert repeated_created is False
|
||||
assert first.state == "queued"
|
||||
assert repeated == first
|
||||
|
||||
|
||||
def test_session_setup_identity_changes_for_each_exact_combination(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
first, _ = queue.submit(_intent())
|
||||
second, _ = queue.submit(
|
||||
_intent(
|
||||
idempotency_key="recorded-request-002",
|
||||
source_session_id="another-session",
|
||||
)
|
||||
)
|
||||
third, _ = queue.submit(
|
||||
_intent(
|
||||
idempotency_key="recorded-request-003",
|
||||
setup_id="legacy-monolith-v1",
|
||||
definition_sha256=NON_CHECKPOINTABLE_DEFINITION_SHA,
|
||||
)
|
||||
)
|
||||
|
||||
assert len({first.identity_sha256, second.identity_sha256, third.identity_sha256}) == 3
|
||||
assert second.source_adapter_sha256 == first.source_adapter_sha256
|
||||
assert third.checkpoint_policy == "non-checkpointable"
|
||||
|
||||
with pytest.raises(ObservatoryRecordedQueueConflictError, match="allowlisted"):
|
||||
queue.submit(
|
||||
_intent(
|
||||
idempotency_key="recorded-request-004",
|
||||
definition_sha256="0" * 64,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_normal_recorded_job_lifecycle_and_terminal_idempotency(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
running, claim = _running_job(queue)
|
||||
|
||||
checkpointed = queue.checkpoint(
|
||||
running.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
checkpoint_id="model-batch-finished",
|
||||
)
|
||||
assert checkpointed.state == "running"
|
||||
assert checkpointed.last_checkpoint_id == "model-batch-finished"
|
||||
|
||||
succeeded = queue.succeed(
|
||||
running.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
result_id="recorded-result-001",
|
||||
result_sha256=RESULT_SHA,
|
||||
)
|
||||
replayed = queue.succeed(
|
||||
running.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
result_id="recorded-result-001",
|
||||
result_sha256=RESULT_SHA,
|
||||
)
|
||||
assert succeeded.state == "succeeded"
|
||||
assert replayed == succeeded
|
||||
assert succeeded.result_sha256 == RESULT_SHA
|
||||
assert succeeded.active_claim_token is None
|
||||
|
||||
with pytest.raises(ObservatoryRecordedQueueConflictError, match="terminal"):
|
||||
queue.fail(
|
||||
running.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
error_code="worker-failed",
|
||||
message="Synthetic failure.",
|
||||
)
|
||||
|
||||
|
||||
def test_success_requires_running_and_cannot_publish_during_preemption(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job, _ = queue.submit(_intent())
|
||||
queue.enqueue(job.job_id)
|
||||
claim = queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="worker-claim-001",
|
||||
)
|
||||
assert claim is not None
|
||||
|
||||
with pytest.raises(ObservatoryRecordedQueueConflictError, match="claimed"):
|
||||
queue.succeed(
|
||||
job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
result_id="recorded-result-001",
|
||||
result_sha256=RESULT_SHA,
|
||||
)
|
||||
|
||||
queue.start(job.job_id, claim_token=claim.claim_token)
|
||||
queue.request_live(_live_intent())
|
||||
assert queue.get(job.job_id).preemption_requested is True
|
||||
with pytest.raises(
|
||||
ObservatoryRecordedQueueConflictError,
|
||||
match="preemption",
|
||||
):
|
||||
queue.succeed(
|
||||
job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
result_id="recorded-result-001",
|
||||
result_sha256=RESULT_SHA,
|
||||
)
|
||||
|
||||
|
||||
def test_claim_is_exactly_idempotent_including_empty_result(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
|
||||
empty = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="empty-poll-001"
|
||||
)
|
||||
assert empty is None
|
||||
job, _ = queue.submit(_intent())
|
||||
queue.enqueue(job.job_id)
|
||||
assert (
|
||||
queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="empty-poll-001"
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
claim = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="non-empty-poll-001"
|
||||
)
|
||||
retry = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="non-empty-poll-001"
|
||||
)
|
||||
assert claim is not None
|
||||
assert retry is not None
|
||||
assert retry.claim_token == claim.claim_token
|
||||
assert retry.job.job_id == job.job_id
|
||||
|
||||
with pytest.raises(ObservatoryRecordedQueueConflictError, match="claim"):
|
||||
queue.claim_next(
|
||||
claimant_id="another-worker", claim_request_id="non-empty-poll-001"
|
||||
)
|
||||
|
||||
|
||||
def test_single_worker_resource_has_only_one_recorded_owner(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
first, _ = queue.submit(_intent())
|
||||
second, _ = queue.submit(
|
||||
_intent(
|
||||
idempotency_key="recorded-request-002",
|
||||
source_session_id="another-session",
|
||||
)
|
||||
)
|
||||
queue.enqueue(first.job_id)
|
||||
queue.enqueue(second.job_id)
|
||||
first_claim = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="owner-poll-001"
|
||||
)
|
||||
assert first_claim is not None
|
||||
assert (
|
||||
queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="owner-poll-002"
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
completed_job_id = first_claim.job.job_id
|
||||
queue.fail(
|
||||
completed_job_id,
|
||||
claim_token=first_claim.claim_token,
|
||||
error_code="synthetic-failure",
|
||||
message="Release the synthetic single-Worker ownership.",
|
||||
)
|
||||
second_claim = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="owner-poll-003"
|
||||
)
|
||||
assert second_claim is not None
|
||||
assert second_claim.job.job_id in {first.job_id, second.job_id} - {completed_job_id}
|
||||
|
||||
|
||||
def test_live_lease_cooperatively_pauses_and_resumes_recorded_job(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
running, claim = _running_job(queue)
|
||||
|
||||
lease, created = queue.request_live(_live_intent())
|
||||
exact_retry, retry_created = queue.request_live(_live_intent())
|
||||
requested_job = queue.get(running.job_id)
|
||||
assert created is True
|
||||
assert retry_created is False
|
||||
assert exact_retry == lease
|
||||
assert lease.state == "pending"
|
||||
assert lease.priority_rank == LIVE_K1_PRIORITY_RANK
|
||||
assert requested_job.state == "running"
|
||||
assert requested_job.preemption_requested is True
|
||||
assert queue.admission_gate().blocked is True
|
||||
|
||||
with pytest.raises(ObservatoryRecordedQueueBusyError, match="safe preemption"):
|
||||
queue.activate_live(lease.lease_id)
|
||||
|
||||
paused = queue.checkpoint(
|
||||
running.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
checkpoint_id="model-batch-finished",
|
||||
)
|
||||
assert paused.state == "paused"
|
||||
active = queue.activate_live(lease.lease_id)
|
||||
assert active.state == "active"
|
||||
|
||||
another, _ = queue.submit(
|
||||
_intent(
|
||||
idempotency_key="recorded-request-002",
|
||||
source_session_id="another-session",
|
||||
)
|
||||
)
|
||||
queue.enqueue(another.job_id)
|
||||
assert (
|
||||
queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="blocked-poll-001"
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
completed = queue.finish_live(
|
||||
lease.lease_id,
|
||||
terminal_trigger_id="k1-live-finish-001",
|
||||
outcome="completed",
|
||||
)
|
||||
completed_retry = queue.finish_live(
|
||||
lease.lease_id,
|
||||
terminal_trigger_id="k1-live-finish-001",
|
||||
outcome="completed",
|
||||
)
|
||||
assert completed.state == "completed"
|
||||
assert completed_retry == completed
|
||||
assert queue.admission_gate().blocked is False
|
||||
resumed = queue.get(running.job_id)
|
||||
assert resumed.state == "queued"
|
||||
assert resumed.preemption_requested is False
|
||||
assert resumed.active_claim_token is None
|
||||
|
||||
with pytest.raises(ObservatoryRecordedQueueConflictError, match="terminal"):
|
||||
queue.finish_live(
|
||||
lease.lease_id,
|
||||
terminal_trigger_id="different-terminal-trigger",
|
||||
outcome="failed",
|
||||
)
|
||||
|
||||
|
||||
def test_cancelled_pending_live_lease_releases_cooperative_preemption_request(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
running, claim = _running_job(queue)
|
||||
lease, _ = queue.request_live(_live_intent())
|
||||
assert queue.get(running.job_id).preemption_requested is True
|
||||
|
||||
cancelled = queue.finish_live(
|
||||
lease.lease_id,
|
||||
terminal_trigger_id="k1-live-cancel-001",
|
||||
outcome="cancelled",
|
||||
)
|
||||
released = queue.get(running.job_id)
|
||||
|
||||
assert cancelled.state == "cancelled"
|
||||
assert released.state == "running"
|
||||
assert released.preemption_requested is False
|
||||
assert released.active_claim_token == claim.claim_token
|
||||
|
||||
|
||||
def test_live_request_pauses_claimed_job_before_execution(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job, _ = queue.submit(_intent())
|
||||
queue.enqueue(job.job_id)
|
||||
claim = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="worker-claim-001"
|
||||
)
|
||||
assert claim is not None
|
||||
|
||||
lease, _ = queue.request_live(_live_intent())
|
||||
paused = queue.get(job.job_id)
|
||||
assert paused.state == "paused"
|
||||
assert queue.start(job.job_id, claim_token=claim.claim_token).state == "paused"
|
||||
assert queue.activate_live(lease.lease_id).state == "active"
|
||||
queue.finish_live(
|
||||
lease.lease_id,
|
||||
terminal_trigger_id="k1-live-finish-001",
|
||||
outcome="completed",
|
||||
)
|
||||
|
||||
with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"):
|
||||
queue.start(job.job_id, claim_token=claim.claim_token)
|
||||
|
||||
|
||||
def test_non_checkpointable_job_is_cancelled_and_restarts_from_zero_for_live(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
queue = _queue(tmp_path, with_non_checkpointable_preemptor=True)
|
||||
running, claim = _running_job(
|
||||
queue,
|
||||
intent=_intent(
|
||||
setup_id="legacy-monolith-v1",
|
||||
definition_sha256=NON_CHECKPOINTABLE_DEFINITION_SHA,
|
||||
),
|
||||
)
|
||||
lease, _ = queue.request_live(_live_intent())
|
||||
paused = queue.get(running.job_id)
|
||||
assert paused.state == "paused"
|
||||
assert paused.preemption_requested is True
|
||||
assert paused.restart_from_zero is True
|
||||
assert paused.preemption_receipt_sha256 is not None
|
||||
assert paused.active_claim_token is None
|
||||
|
||||
with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"):
|
||||
queue.checkpoint(
|
||||
running.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
checkpoint_id="model-batch-finished",
|
||||
)
|
||||
assert queue.activate_live(lease.lease_id).state == "active"
|
||||
assert (
|
||||
queue.finish_live(
|
||||
lease.lease_id,
|
||||
terminal_trigger_id="k1-live-finish-001",
|
||||
outcome="failed",
|
||||
).state
|
||||
== "failed"
|
||||
)
|
||||
resumed = queue.get(running.job_id)
|
||||
assert resumed.state == "queued"
|
||||
assert resumed.restart_from_zero is True
|
||||
|
||||
new_claim = queue.claim_next(
|
||||
claimant_id="recorded-worker", claim_request_id="worker-claim-002"
|
||||
)
|
||||
assert new_claim is not None
|
||||
assert new_claim.job.job_id == running.job_id
|
||||
assert new_claim.job.restart_from_zero is True
|
||||
|
||||
|
||||
def test_live_request_fails_closed_without_non_checkpointable_canceler(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
running, _claim = _running_job(
|
||||
queue,
|
||||
intent=_intent(
|
||||
setup_id="legacy-monolith-v1",
|
||||
definition_sha256=NON_CHECKPOINTABLE_DEFINITION_SHA,
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(ObservatoryRecordedPreemptionError, match="scheduler-owned"):
|
||||
queue.request_live(_live_intent())
|
||||
|
||||
assert queue.admission_gate().blocked is True
|
||||
assert queue.get(running.job_id).state == "preemption-pending"
|
||||
with sqlite3.connect(queue.database_path) as connection:
|
||||
intent_state = connection.execute(
|
||||
"SELECT state FROM observatory_recorded_preemptions"
|
||||
).fetchone()[0]
|
||||
assert intent_state == "pending"
|
||||
|
||||
recovered = _queue(tmp_path, with_non_checkpointable_preemptor=True)
|
||||
lease, created = recovered.request_live(_live_intent())
|
||||
assert created is False
|
||||
assert recovered.get(running.job_id).state == "paused"
|
||||
assert recovered.get(running.job_id).restart_from_zero is True
|
||||
assert recovered.activate_live(lease.lease_id).state == "active"
|
||||
|
||||
|
||||
def test_durable_cancel_intent_survives_callback_crash_and_reuses_identity(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
cancellation_request_ids: list[str] = []
|
||||
|
||||
def crash_after_external_side_effect(request):
|
||||
cancellation_request_ids.append(request.cancellation_request_id)
|
||||
raise OSError("synthetic scheduler process loss after cancellation")
|
||||
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=_definitions(),
|
||||
clock=lambda: NOW,
|
||||
non_checkpointable_preemptor=crash_after_external_side_effect,
|
||||
)
|
||||
running, _claim = _running_job(
|
||||
queue,
|
||||
intent=_intent(
|
||||
setup_id="legacy-monolith-v1",
|
||||
definition_sha256=NON_CHECKPOINTABLE_DEFINITION_SHA,
|
||||
),
|
||||
)
|
||||
with pytest.raises(ObservatoryRecordedPreemptionError, match="did not release"):
|
||||
queue.request_live(_live_intent())
|
||||
|
||||
assert queue.get(running.job_id).state == "preemption-pending"
|
||||
|
||||
def replay_exact_cancel(request):
|
||||
cancellation_request_ids.append(request.cancellation_request_id)
|
||||
return ObservatoryNonCheckpointableCancellationReceipt.sealed(
|
||||
request,
|
||||
cancellation_id=f"cancel-{request.cancellation_request_id}",
|
||||
)
|
||||
|
||||
recovered = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=_definitions(),
|
||||
clock=lambda: NOW,
|
||||
non_checkpointable_preemptor=replay_exact_cancel,
|
||||
)
|
||||
lease, created = recovered.request_live(_live_intent())
|
||||
|
||||
assert created is False
|
||||
assert len(cancellation_request_ids) == 2
|
||||
assert cancellation_request_ids[0] == cancellation_request_ids[1]
|
||||
assert recovered.get(running.job_id).state == "paused"
|
||||
assert recovered.activate_live(lease.lease_id).state == "active"
|
||||
|
||||
|
||||
def test_learned_model_release_list_may_be_empty_for_algorithm_only_tgs() -> None:
|
||||
definition = RecordedRunDefinition(
|
||||
setup_id="travel-tgs-algorithm-v1",
|
||||
definition_id="travel-tgs-algorithm",
|
||||
definition_version=1,
|
||||
definition_sha256="7" * 64,
|
||||
source_adapter_id="sealed-session-bundle",
|
||||
source_adapter_version=1,
|
||||
source_adapter_sha256=ADAPTER_SHA,
|
||||
executor_release_id="travel-tgs-executor-v1",
|
||||
executor_release_sha256=EXECUTOR_RELEASE_SHA,
|
||||
executor_image_sha256=EXECUTOR_IMAGE_SHA,
|
||||
model_release_ids=(),
|
||||
model_manifest_sha256=MODEL_MANIFEST_SHA,
|
||||
resource_profile_id="worker006-single-gpu-v1",
|
||||
resource_profile_sha256=RESOURCE_PROFILE_SHA,
|
||||
checkpoint_policy="cooperative",
|
||||
allowed_checkpoints=("model-batch-finished",),
|
||||
)
|
||||
|
||||
assert definition.model_release_ids == ()
|
||||
assert definition.learned_models == ()
|
||||
assert definition.model_manifest_sha256 == MODEL_MANIFEST_SHA
|
||||
|
||||
|
||||
def test_reconciliation_required_is_durable_terminal_state(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
running, claim = _running_job(queue)
|
||||
|
||||
uncertain = queue.require_reconciliation(
|
||||
running.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
reason_code="worker-outcome-unknown",
|
||||
message="Dispatch was accepted but its terminal receipt is unavailable.",
|
||||
)
|
||||
restored = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=_definitions(),
|
||||
clock=lambda: NOW,
|
||||
).get(running.job_id)
|
||||
assert uncertain.state == "reconciliation-required"
|
||||
assert restored == uncertain
|
||||
|
||||
|
||||
def test_reconciliation_required_quarantines_recorded_and_live_ownership(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
running, claim = _running_job(queue)
|
||||
queue.require_reconciliation(
|
||||
running.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
reason_code="worker-outcome-unknown",
|
||||
message="Worker ownership cannot be proven released.",
|
||||
)
|
||||
waiting, _ = queue.submit(
|
||||
_intent(idempotency_key="recorded-request-002")
|
||||
)
|
||||
queue.enqueue(waiting.job_id)
|
||||
|
||||
assert queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="worker-claim-after-reconciliation",
|
||||
) is None
|
||||
lease, _ = queue.request_live(_live_intent())
|
||||
with pytest.raises(ObservatoryRecordedQueueBusyError, match="recorded work"):
|
||||
queue.activate_live(lease.lease_id)
|
||||
|
||||
|
||||
def test_live_lease_requires_explicit_valid_terminal_transition(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
lease, _ = queue.request_live(_live_intent())
|
||||
|
||||
with pytest.raises(ObservatoryRecordedQueueConflictError, match="complete"):
|
||||
queue.finish_live(
|
||||
lease.lease_id,
|
||||
terminal_trigger_id="k1-live-finish-001",
|
||||
outcome="completed",
|
||||
)
|
||||
cancelled = queue.finish_live(
|
||||
lease.lease_id,
|
||||
terminal_trigger_id="k1-live-cancel-001",
|
||||
outcome="cancelled",
|
||||
)
|
||||
assert cancelled.state == "cancelled"
|
||||
assert queue.admission_gate().blocked is False
|
||||
|
||||
|
||||
def test_queue_detects_mutated_immutable_identity(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job, _ = queue.submit(_intent())
|
||||
with sqlite3.connect(queue.database_path) as connection:
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs SET source_catalog_sha256 = ? "
|
||||
"WHERE job_id = ?",
|
||||
("0" * 64, job.job_id),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
with pytest.raises(ObservatoryRecordedQueueIntegrityError, match="identity"):
|
||||
queue.get(job.job_id)
|
||||
@@ -0,0 +1,253 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from k1link.observatory.m49_queue_binding import M49QueueBindingIntegrityError
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobIntent,
|
||||
ObservatoryRecordedJobQueue,
|
||||
RecordedRunDefinition,
|
||||
RecordedRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.setups import LaboratorySetupRegistry
|
||||
from k1link.sessions import SessionNotFoundError
|
||||
from k1link.sessions.models import SessionSummary
|
||||
from k1link.web.observatory_api import build_observatory_router
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-laboratory-setups.json"
|
||||
RAV00_SESSION_ID = "20260720T065719Z_viewer_live"
|
||||
SETUP_ID = "m49-tgs-full-shadow-v1"
|
||||
CATALOG_SHA256 = "1" * 64
|
||||
SOURCE_BUNDLE_SHA256 = "2" * 64
|
||||
CAPABILITY_SHA256 = "3" * 64
|
||||
|
||||
|
||||
def _source() -> SessionSummary:
|
||||
return SessionSummary(
|
||||
session_id=RAV00_SESSION_ID,
|
||||
display_name="RAVNOVES00",
|
||||
status="ready",
|
||||
started_at_utc="2026-07-20T06:57:19Z",
|
||||
completed_at_utc="2026-07-20T07:06:15Z",
|
||||
duration_seconds=536.0,
|
||||
modalities=("point-cloud", "trajectory", "video"),
|
||||
source_count=3,
|
||||
total_bytes=1,
|
||||
replayable=True,
|
||||
origin="recorded",
|
||||
)
|
||||
|
||||
|
||||
class _Store:
|
||||
def get_session(self, session_id: str) -> SimpleNamespace:
|
||||
if session_id != RAV00_SESSION_ID:
|
||||
raise SessionNotFoundError(session_id)
|
||||
return SimpleNamespace(summary=_source())
|
||||
|
||||
|
||||
class _Admission:
|
||||
def __init__(
|
||||
self,
|
||||
definition: RecordedRunDefinition,
|
||||
catalog_sha256: str,
|
||||
) -> None:
|
||||
self.definition = definition
|
||||
self.catalog_sha256 = catalog_sha256
|
||||
|
||||
def intent(self, *, idempotency_key: str) -> ObservatoryRecordedJobIntent:
|
||||
return ObservatoryRecordedJobIntent(
|
||||
idempotency_key=idempotency_key,
|
||||
source_session_id=RAV00_SESSION_ID,
|
||||
source_catalog_sha256=self.catalog_sha256,
|
||||
source_bundle_sha256=SOURCE_BUNDLE_SHA256,
|
||||
source_capability_manifest_sha256=CAPABILITY_SHA256,
|
||||
setup_id=self.definition.setup_id,
|
||||
definition_sha256=self.definition.definition_sha256,
|
||||
)
|
||||
|
||||
|
||||
class _BindingService:
|
||||
def __init__(self, definition: RecordedRunDefinition) -> None:
|
||||
self.config = SimpleNamespace(setup=SimpleNamespace(setup_id=SETUP_ID))
|
||||
self.definition = definition
|
||||
self.catalog_sha256 = CATALOG_SHA256
|
||||
self.check_count = 0
|
||||
self.admit_count = 0
|
||||
|
||||
def check(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> _Admission:
|
||||
self.check_count += 1
|
||||
return self._resolve(source_session_id, setup_id, definition_sha256)
|
||||
|
||||
def admit(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> _Admission:
|
||||
self.admit_count += 1
|
||||
return self._resolve(source_session_id, setup_id, definition_sha256)
|
||||
|
||||
def _resolve(
|
||||
self,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> _Admission:
|
||||
if (
|
||||
source_session_id != RAV00_SESSION_ID
|
||||
or setup_id != self.definition.setup_id
|
||||
or definition_sha256 != self.definition.definition_sha256
|
||||
):
|
||||
raise M49QueueBindingIntegrityError("not the exact M4.9 binding")
|
||||
return _Admission(self.definition, self.catalog_sha256)
|
||||
|
||||
|
||||
def _services(
|
||||
tmp_path: Path,
|
||||
) -> tuple[
|
||||
LaboratorySetupRegistry,
|
||||
_BindingService,
|
||||
ObservatoryRecordedJobQueue,
|
||||
RecordedRunDefinition,
|
||||
]:
|
||||
registry = LaboratorySetupRegistry.from_file(
|
||||
REGISTRY_PATH,
|
||||
repository_root=REPOSITORY_ROOT,
|
||||
)
|
||||
catalog = registry.catalog(_source())
|
||||
setups = cast(list[dict[str, Any]], catalog["setups"])
|
||||
definition_document = cast(dict[str, object], setups[0]["run_definition"])
|
||||
definition = RecordedRunDefinition(
|
||||
setup_id=SETUP_ID,
|
||||
definition_id=str(definition_document["definition_id"]),
|
||||
definition_version=int(definition_document["version"]),
|
||||
definition_sha256=str(definition_document["definition_sha256"]),
|
||||
source_adapter_id="ravnoves00-m49-source-pack",
|
||||
source_adapter_version=1,
|
||||
source_adapter_sha256="4" * 64,
|
||||
executor_release_id="m49-tgs-full-shadow-worker-release",
|
||||
executor_release_sha256="5" * 64,
|
||||
executor_image_sha256="6" * 64,
|
||||
model_release_ids=(),
|
||||
model_manifest_sha256="7" * 64,
|
||||
resource_profile_id="worker006-cpu-single-run-v1",
|
||||
resource_profile_sha256="8" * 64,
|
||||
checkpoint_policy="non-checkpointable",
|
||||
)
|
||||
binding = _BindingService(definition)
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=RecordedRunDefinitionRegistry((definition,)),
|
||||
)
|
||||
return registry, binding, queue, definition
|
||||
|
||||
|
||||
def test_exact_m49_preflight_is_queueable_and_submit_is_idempotently_queued(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry, binding, queue, definition = _services(tmp_path)
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_observatory_router(
|
||||
_Store(), # type: ignore[arg-type]
|
||||
setup_registry=registry,
|
||||
recorded_binding_service=binding, # type: ignore[arg-type]
|
||||
recorded_job_queue=queue,
|
||||
)
|
||||
)
|
||||
client = TestClient(app)
|
||||
|
||||
preflight = client.post(
|
||||
"/api/v1/observatory/run-preflights",
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-run-preflight-request/v1",
|
||||
"source_session_id": RAV00_SESSION_ID,
|
||||
"setup_id": SETUP_ID,
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
},
|
||||
)
|
||||
|
||||
assert preflight.status_code == 200
|
||||
assert preflight.json()["outcome"] == "queueable"
|
||||
assert preflight.json()["submission_allowed"] is True
|
||||
assert next(
|
||||
check
|
||||
for check in preflight.json()["checks"]
|
||||
if check["check_id"] == "durable-queue"
|
||||
)["outcome"] == "pass"
|
||||
assert binding.check_count == 1
|
||||
assert binding.admit_count == 0
|
||||
|
||||
request = {
|
||||
"schema_version": "missioncore.observatory-recorded-run-submit/v1",
|
||||
"idempotency_key": "observatory-ui:m49:stable-request",
|
||||
"source_session_id": RAV00_SESSION_ID,
|
||||
"setup_id": SETUP_ID,
|
||||
}
|
||||
submitted = client.post("/api/v1/observatory/runs", json=request)
|
||||
binding.catalog_sha256 = "9" * 64
|
||||
repeated = client.post("/api/v1/observatory/runs", json=request)
|
||||
|
||||
assert submitted.status_code == 202
|
||||
assert repeated.status_code == 202
|
||||
assert submitted.json()["job_id"] == repeated.json()["job_id"]
|
||||
assert submitted.json()["state"] == "queued"
|
||||
assert submitted.json()["source"]["session_id"] == RAV00_SESSION_ID
|
||||
assert submitted.json()["setup"]["setup_id"] == SETUP_ID
|
||||
assert submitted.json()["executor"]["learned_models"] == []
|
||||
assert submitted.json()["priority"] == {
|
||||
"class": "recorded",
|
||||
"rank": 100,
|
||||
"server_owned": True,
|
||||
}
|
||||
assert binding.admit_count == 1
|
||||
|
||||
listed = client.get(
|
||||
"/api/v1/observatory/runs",
|
||||
params={"source_session_id": RAV00_SESSION_ID, "setup_id": SETUP_ID},
|
||||
)
|
||||
assert listed.status_code == 200
|
||||
assert listed.json()["schema_version"] == (
|
||||
"missioncore.observatory-recorded-job-list/v1"
|
||||
)
|
||||
assert [item["job_id"] for item in listed.json()["items"]] == [
|
||||
submitted.json()["job_id"]
|
||||
]
|
||||
assert queue.admission_gate().blocked is False
|
||||
|
||||
|
||||
def test_recorded_run_routes_fail_closed_when_queue_initialization_failed() -> None:
|
||||
registry = LaboratorySetupRegistry.from_file(
|
||||
REGISTRY_PATH,
|
||||
repository_root=REPOSITORY_ROOT,
|
||||
)
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_observatory_router(
|
||||
_Store(), # type: ignore[arg-type]
|
||||
setup_registry=registry,
|
||||
recorded_job_queue_error="queue database is incompatible",
|
||||
)
|
||||
)
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get(
|
||||
"/api/v1/observatory/runs",
|
||||
params={"source_session_id": RAV00_SESSION_ID, "setup_id": SETUP_ID},
|
||||
)
|
||||
|
||||
assert response.status_code == 503
|
||||
assert response.json()["detail"] == "Durable-очередь расчётов недоступна."
|
||||
@@ -127,6 +127,7 @@ def test_repository_setup_registry_keeps_real_definition_and_pre_definition_resu
|
||||
]
|
||||
rav00 = registry.catalog(_source(RAV00_SESSION_ID, "RAVNOVES00"))
|
||||
m49, current = rav00["setups"]
|
||||
assert m49["display_name"] == "M4.9T5 · TRAVEL TGS · CPU-only, без ML"
|
||||
assert m49["origin"] == "archived-definition"
|
||||
assert m49["compatibility"]["compatible"] is True
|
||||
assert m49["preflight"] == {
|
||||
@@ -153,6 +154,8 @@ def test_repository_setup_registry_keeps_real_definition_and_pre_definition_resu
|
||||
available_observatory_result_ids=frozenset({RAV004_RESULT_ID}),
|
||||
)
|
||||
existing = rav004["setups"][1]
|
||||
assert existing["display_name"] == "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39"
|
||||
assert "полный TGS и независимый YOLOX отсутствуют" in existing["description"]
|
||||
assert existing["origin"] == "existing-result"
|
||||
assert existing["run_definition"] is None
|
||||
assert existing["preflight"]["outcome"] == "existing"
|
||||
|
||||
Reference in New Issue
Block a user