feat(observatory): add durable recorded compute queue
This commit is contained in:
@@ -61,8 +61,8 @@ export interface ObservatoryLaboratoryRunPreflight {
|
|||||||
readonly sourceSessionId: string;
|
readonly sourceSessionId: string;
|
||||||
readonly setupId: string;
|
readonly setupId: string;
|
||||||
readonly definitionSha256: string | null;
|
readonly definitionSha256: string | null;
|
||||||
readonly outcome: "existing" | "blocked";
|
readonly outcome: "existing" | "queueable" | "blocked";
|
||||||
readonly submissionAllowed: false;
|
readonly submissionAllowed: boolean;
|
||||||
readonly checks: readonly {
|
readonly checks: readonly {
|
||||||
readonly checkId: string;
|
readonly checkId: string;
|
||||||
readonly outcome: "pass" | "fail" | "not-applicable";
|
readonly outcome: "pass" | "fail" | "not-applicable";
|
||||||
@@ -287,8 +287,15 @@ function decodePreflight(value: unknown): ObservatoryLaboratoryRunPreflight {
|
|||||||
sourceSessionId: text(row.source_session_id, "preflight source_session_id"),
|
sourceSessionId: text(row.source_session_id, "preflight source_session_id"),
|
||||||
setupId: text(row.setup_id, "preflight setup_id"),
|
setupId: text(row.setup_id, "preflight setup_id"),
|
||||||
definitionSha256: digest,
|
definitionSha256: digest,
|
||||||
outcome: oneOf(row.outcome, ["existing", "blocked"] as const, "preflight outcome"),
|
outcome: oneOf(
|
||||||
submissionAllowed: exact(row.submission_allowed, false, "preflight submission_allowed"),
|
row.outcome,
|
||||||
|
["existing", "queueable", "blocked"] as const,
|
||||||
|
"preflight outcome",
|
||||||
|
),
|
||||||
|
submissionAllowed: boolean(
|
||||||
|
row.submission_allowed,
|
||||||
|
"preflight submission_allowed",
|
||||||
|
),
|
||||||
checks: array(row.checks, "preflight checks").map((item) => {
|
checks: array(row.checks, "preflight checks").map((item) => {
|
||||||
const check = record(item, "preflight check");
|
const check = record(item, "preflight check");
|
||||||
exactKeys(check, ["check_id", "message", "outcome", "reason_code"], "preflight check");
|
exactKeys(check, ["check_id", "message", "outcome", "reason_code"], "preflight check");
|
||||||
|
|||||||
@@ -0,0 +1,241 @@
|
|||||||
|
const SUBMIT_SCHEMA = "missioncore.observatory-recorded-run-submit/v1";
|
||||||
|
const JOB_SCHEMA = "missioncore.observatory-recorded-job/v1";
|
||||||
|
const JOB_LIST_SCHEMA = "missioncore.observatory-recorded-job-list/v1";
|
||||||
|
const SHA256 = /^[a-f0-9]{64}$/;
|
||||||
|
|
||||||
|
export type ObservatoryRecordedJobState =
|
||||||
|
| "accepted"
|
||||||
|
| "queued"
|
||||||
|
| "claimed"
|
||||||
|
| "running"
|
||||||
|
| "paused"
|
||||||
|
| "preemption-pending"
|
||||||
|
| "succeeded"
|
||||||
|
| "failed"
|
||||||
|
| "reconciliation-required";
|
||||||
|
|
||||||
|
export interface ObservatoryRecordedJob {
|
||||||
|
readonly jobId: string;
|
||||||
|
readonly idempotencyKey: string;
|
||||||
|
readonly sourceSessionId: string;
|
||||||
|
readonly setupId: string;
|
||||||
|
readonly definitionSha256: string;
|
||||||
|
readonly state: ObservatoryRecordedJobState;
|
||||||
|
readonly restartFromZero: boolean;
|
||||||
|
readonly resultId: string | null;
|
||||||
|
readonly terminalMessage: string | null;
|
||||||
|
readonly createdAtUtc: string;
|
||||||
|
readonly updatedAtUtc: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ObservatoryRecordedJobFetch = (
|
||||||
|
input: RequestInfo | URL,
|
||||||
|
init?: RequestInit,
|
||||||
|
) => Promise<Response>;
|
||||||
|
|
||||||
|
export class ObservatoryRecordedJobContractError extends Error {
|
||||||
|
readonly status: number | null;
|
||||||
|
|
||||||
|
constructor(message: string, status: number | null = null) {
|
||||||
|
super(message);
|
||||||
|
this.name = "ObservatoryRecordedJobContractError";
|
||||||
|
this.status = status;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchObservatoryRecordedJobs(
|
||||||
|
sourceSessionId: string,
|
||||||
|
setupId: string,
|
||||||
|
{
|
||||||
|
signal,
|
||||||
|
fetcher = globalThis.fetch,
|
||||||
|
}: {
|
||||||
|
signal?: AbortSignal;
|
||||||
|
fetcher?: ObservatoryRecordedJobFetch;
|
||||||
|
} = {},
|
||||||
|
): Promise<readonly ObservatoryRecordedJob[]> {
|
||||||
|
const query = new URLSearchParams({
|
||||||
|
source_session_id: sourceSessionId,
|
||||||
|
setup_id: setupId,
|
||||||
|
limit: "20",
|
||||||
|
});
|
||||||
|
const response = await request(fetcher, `/api/v1/observatory/runs?${query}`, {
|
||||||
|
method: "GET",
|
||||||
|
headers: { Accept: "application/json" },
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
const body = await responseBody(response);
|
||||||
|
if (!response.ok) throw apiError(body, response.status);
|
||||||
|
const row = record(body, "список расчётов");
|
||||||
|
exactKeys(row, ["authority", "items", "schema_version"], "список расчётов");
|
||||||
|
exact(row.schema_version, JOB_LIST_SCHEMA, "schema_version списка расчётов");
|
||||||
|
observationAuthority(row.authority);
|
||||||
|
return array(row.items, "items").map(decodeJob).filter((job) => (
|
||||||
|
job.sourceSessionId === sourceSessionId && job.setupId === setupId
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function submitObservatoryRecordedJob(
|
||||||
|
sourceSessionId: string,
|
||||||
|
setupId: string,
|
||||||
|
idempotencyKey: string,
|
||||||
|
{
|
||||||
|
signal,
|
||||||
|
fetcher = globalThis.fetch,
|
||||||
|
}: {
|
||||||
|
signal?: AbortSignal;
|
||||||
|
fetcher?: ObservatoryRecordedJobFetch;
|
||||||
|
} = {},
|
||||||
|
): Promise<ObservatoryRecordedJob> {
|
||||||
|
const response = await request(fetcher, "/api/v1/observatory/runs", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { Accept: "application/json", "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
schema_version: SUBMIT_SCHEMA,
|
||||||
|
idempotency_key: idempotencyKey,
|
||||||
|
source_session_id: sourceSessionId,
|
||||||
|
setup_id: setupId,
|
||||||
|
}),
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
const body = await responseBody(response);
|
||||||
|
if (!response.ok) throw apiError(body, response.status);
|
||||||
|
const job = decodeJob(body);
|
||||||
|
if (job.sourceSessionId !== sourceSessionId || job.setupId !== setupId) {
|
||||||
|
throw new ObservatoryRecordedJobContractError(
|
||||||
|
"Расчёт относится к другой сессии или сетапу.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
function decodeJob(value: unknown): ObservatoryRecordedJob {
|
||||||
|
const row = record(value, "расчёт");
|
||||||
|
exactKeys(row, [
|
||||||
|
"authority", "checkpoint_policy", "claim_generation", "created_at_utc", "executor",
|
||||||
|
"idempotency_key", "identity_sha256", "job_id", "preemption_receipt_sha256",
|
||||||
|
"preemption_requested", "priority", "request_sha256", "restart_from_zero", "result",
|
||||||
|
"schema_version", "setup", "source", "state", "submission_receipt_sha256", "terminal",
|
||||||
|
"updated_at_utc",
|
||||||
|
], "расчёт");
|
||||||
|
exact(row.schema_version, JOB_SCHEMA, "schema_version расчёта");
|
||||||
|
observationAuthority(row.authority);
|
||||||
|
digest(row.request_sha256, "request_sha256");
|
||||||
|
digest(row.identity_sha256, "identity_sha256");
|
||||||
|
digest(row.submission_receipt_sha256, "submission_receipt_sha256");
|
||||||
|
const source = record(row.source, "source");
|
||||||
|
exactKeys(source, [
|
||||||
|
"adapter", "bundle_sha256", "capability_manifest_sha256", "catalog_sha256", "session_id",
|
||||||
|
], "source");
|
||||||
|
digest(source.catalog_sha256, "source.catalog_sha256");
|
||||||
|
digest(source.bundle_sha256, "source.bundle_sha256");
|
||||||
|
digest(source.capability_manifest_sha256, "source.capability_manifest_sha256");
|
||||||
|
const setup = record(row.setup, "setup");
|
||||||
|
exactKeys(setup, ["definition_id", "definition_sha256", "definition_version", "setup_id"], "setup");
|
||||||
|
digest(setup.definition_sha256, "setup.definition_sha256");
|
||||||
|
const state = oneOf(row.state, [
|
||||||
|
"accepted", "queued", "claimed", "running", "paused", "preemption-pending",
|
||||||
|
"succeeded", "failed", "reconciliation-required",
|
||||||
|
] as const, "state");
|
||||||
|
const result = row.result === null ? null : record(row.result, "result");
|
||||||
|
if (result !== null) exactKeys(result, ["result_id", "sha256"], "result");
|
||||||
|
const terminal = row.terminal === null ? null : record(row.terminal, "terminal");
|
||||||
|
if (terminal !== null) exactKeys(terminal, ["code", "message"], "terminal");
|
||||||
|
return {
|
||||||
|
jobId: text(row.job_id, "job_id"),
|
||||||
|
idempotencyKey: text(row.idempotency_key, "idempotency_key"),
|
||||||
|
sourceSessionId: text(source.session_id, "source.session_id"),
|
||||||
|
setupId: text(setup.setup_id, "setup.setup_id"),
|
||||||
|
definitionSha256: String(setup.definition_sha256),
|
||||||
|
state,
|
||||||
|
restartFromZero: boolean(row.restart_from_zero, "restart_from_zero"),
|
||||||
|
resultId: result === null ? null : text(result.result_id, "result.result_id"),
|
||||||
|
terminalMessage: terminal === null || terminal.message === null
|
||||||
|
? null
|
||||||
|
: text(terminal.message, "terminal.message"),
|
||||||
|
createdAtUtc: text(row.created_at_utc, "created_at_utc"),
|
||||||
|
updatedAtUtc: text(row.updated_at_utc, "updated_at_utc"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function observationAuthority(value: unknown): void {
|
||||||
|
const row = record(value, "authority");
|
||||||
|
const keys = ["commands_enabled", "actuation_allowed", "navigation_or_safety_accepted", "production_accepted"];
|
||||||
|
exactKeys(row, keys, "authority");
|
||||||
|
keys.forEach((key) => exact(row[key], false, `authority.${key}`));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function request(fetcher: ObservatoryRecordedJobFetch, input: string, init: RequestInit): Promise<Response> {
|
||||||
|
try {
|
||||||
|
return await fetcher(input, init);
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
||||||
|
throw new ObservatoryRecordedJobContractError("Очередь расчётов недоступна.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function responseBody(response: Response): Promise<unknown> {
|
||||||
|
const body = await response.text();
|
||||||
|
if (!body) return undefined;
|
||||||
|
try { return JSON.parse(body) as unknown; } catch { return body; }
|
||||||
|
}
|
||||||
|
|
||||||
|
function apiError(body: unknown, status: number): ObservatoryRecordedJobContractError {
|
||||||
|
const detail = body && typeof body === "object" && !Array.isArray(body)
|
||||||
|
? (body as Record<string, unknown>).detail
|
||||||
|
: null;
|
||||||
|
return new ObservatoryRecordedJobContractError(
|
||||||
|
typeof detail === "string" && detail.trim() ? detail : `Observatory API вернул HTTP ${status}.`,
|
||||||
|
status,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function record(value: unknown, label: string): Record<string, unknown> {
|
||||||
|
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||||
|
throw new ObservatoryRecordedJobContractError(`${label}: ожидался объект.`);
|
||||||
|
}
|
||||||
|
return value as Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function array(value: unknown, label: string): unknown[] {
|
||||||
|
if (!Array.isArray(value)) throw new ObservatoryRecordedJobContractError(`${label}: ожидался массив.`);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function text(value: unknown, label: string): string {
|
||||||
|
if (typeof value !== "string" || !value.trim()) {
|
||||||
|
throw new ObservatoryRecordedJobContractError(`${label}: ожидался текст.`);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function digest(value: unknown, label: string): string {
|
||||||
|
const valueText = text(value, label);
|
||||||
|
if (!SHA256.test(valueText)) throw new ObservatoryRecordedJobContractError(`${label}: некорректный digest.`);
|
||||||
|
return valueText;
|
||||||
|
}
|
||||||
|
|
||||||
|
function boolean(value: unknown, label: string): boolean {
|
||||||
|
if (typeof value !== "boolean") throw new ObservatoryRecordedJobContractError(`${label}: ожидался boolean.`);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function exact<T>(value: unknown, expected: T, label: string): T {
|
||||||
|
if (value !== expected) throw new ObservatoryRecordedJobContractError(`${label}: значение изменилось.`);
|
||||||
|
return expected;
|
||||||
|
}
|
||||||
|
|
||||||
|
function oneOf<const T extends readonly string[]>(value: unknown, allowed: T, label: string): T[number] {
|
||||||
|
if (typeof value !== "string" || !allowed.includes(value)) {
|
||||||
|
throw new ObservatoryRecordedJobContractError(`${label}: значение не поддерживается.`);
|
||||||
|
}
|
||||||
|
return value as T[number];
|
||||||
|
}
|
||||||
|
|
||||||
|
function exactKeys(value: Record<string, unknown>, expected: readonly string[], label: string): void {
|
||||||
|
const actual = Object.keys(value).sort();
|
||||||
|
const sortedExpected = [...expected].sort();
|
||||||
|
if (actual.length !== sortedExpected.length || actual.some((key, index) => key !== sortedExpected[index])) {
|
||||||
|
throw new ObservatoryRecordedJobContractError(`${label}: обнаружены неизвестные поля.`);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -111,6 +111,11 @@ export function useObservatoryLaboratorySetups(sourceSessionId: string) {
|
|||||||
}
|
}
|
||||||
}, [preflight.kind, selectedSetup, sourceSessionId]);
|
}, [preflight.kind, selectedSetup, sourceSessionId]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (state !== "ready" || !selectedSetup || preflight.kind !== "idle") return;
|
||||||
|
void check();
|
||||||
|
}, [check, preflight.kind, selectedSetup, state]);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
catalog: activeCatalog,
|
catalog: activeCatalog,
|
||||||
state,
|
state,
|
||||||
|
|||||||
@@ -0,0 +1,112 @@
|
|||||||
|
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||||
|
|
||||||
|
import {
|
||||||
|
fetchObservatoryRecordedJobs,
|
||||||
|
submitObservatoryRecordedJob,
|
||||||
|
type ObservatoryRecordedJob,
|
||||||
|
} from "./recordedJobs";
|
||||||
|
|
||||||
|
type RecordedJobsState = "idle" | "loading" | "ready" | "refreshing" | "submitting" | "error";
|
||||||
|
|
||||||
|
const OPEN_STATES = new Set([
|
||||||
|
"accepted", "queued", "claimed", "running", "paused", "preemption-pending",
|
||||||
|
"reconciliation-required",
|
||||||
|
]);
|
||||||
|
|
||||||
|
export function useObservatoryRecordedJobs(sourceSessionId: string, setupId: string) {
|
||||||
|
const [jobs, setJobs] = useState<readonly ObservatoryRecordedJob[]>([]);
|
||||||
|
const [state, setState] = useState<RecordedJobsState>("idle");
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const [revision, setRevision] = useState(0);
|
||||||
|
const requestRef = useRef<AbortController | null>(null);
|
||||||
|
const requestSequence = useRef(0);
|
||||||
|
const idempotencyKeys = useRef(new Map<string, string>());
|
||||||
|
const selectionKey = `${sourceSessionId}\u0000${setupId}`;
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
requestRef.current?.abort();
|
||||||
|
requestRef.current = null;
|
||||||
|
if (!sourceSessionId || !setupId) {
|
||||||
|
setJobs([]);
|
||||||
|
setState("idle");
|
||||||
|
setError(null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const sequence = ++requestSequence.current;
|
||||||
|
const request = new AbortController();
|
||||||
|
requestRef.current = request;
|
||||||
|
setState((current) => jobs.length > 0 && current !== "idle" ? "refreshing" : "loading");
|
||||||
|
setError(null);
|
||||||
|
void fetchObservatoryRecordedJobs(sourceSessionId, setupId, { signal: request.signal })
|
||||||
|
.then((next) => {
|
||||||
|
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
||||||
|
setJobs(next);
|
||||||
|
setState("ready");
|
||||||
|
})
|
||||||
|
.catch((caught: unknown) => {
|
||||||
|
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
||||||
|
setState("error");
|
||||||
|
setError(caught instanceof Error && caught.message.trim()
|
||||||
|
? caught.message
|
||||||
|
: "Очередь расчётов недоступна.");
|
||||||
|
});
|
||||||
|
return () => request.abort();
|
||||||
|
}, [revision, selectionKey]);
|
||||||
|
|
||||||
|
useEffect(() => () => requestRef.current?.abort(), []);
|
||||||
|
|
||||||
|
const latestJob = jobs[0] ?? null;
|
||||||
|
const activeJob = useMemo(
|
||||||
|
() => jobs.find((job) => OPEN_STATES.has(job.state)) ?? null,
|
||||||
|
[jobs],
|
||||||
|
);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!latestJob || activeJob) return;
|
||||||
|
idempotencyKeys.current.delete(selectionKey);
|
||||||
|
}, [activeJob, latestJob, selectionKey]);
|
||||||
|
|
||||||
|
const refresh = useCallback(() => setRevision((value) => value + 1), []);
|
||||||
|
|
||||||
|
const submit = useCallback(async (): Promise<ObservatoryRecordedJob | null> => {
|
||||||
|
if (!sourceSessionId || !setupId || state === "submitting") return null;
|
||||||
|
if (activeJob) return activeJob;
|
||||||
|
requestRef.current?.abort();
|
||||||
|
const request = new AbortController();
|
||||||
|
requestRef.current = request;
|
||||||
|
const key = idempotencyKeys.current.get(selectionKey)
|
||||||
|
?? createIdempotencyKey();
|
||||||
|
idempotencyKeys.current.set(selectionKey, key);
|
||||||
|
setState("submitting");
|
||||||
|
setError(null);
|
||||||
|
try {
|
||||||
|
const job = await submitObservatoryRecordedJob(sourceSessionId, setupId, key, {
|
||||||
|
signal: request.signal,
|
||||||
|
});
|
||||||
|
if (request.signal.aborted || requestRef.current !== request) return null;
|
||||||
|
setJobs((current) => [job, ...current.filter((candidate) => candidate.jobId !== job.jobId)]);
|
||||||
|
setState("ready");
|
||||||
|
return job;
|
||||||
|
} catch (caught) {
|
||||||
|
if (request.signal.aborted || requestRef.current !== request) return null;
|
||||||
|
setState("error");
|
||||||
|
setError(caught instanceof Error && caught.message.trim()
|
||||||
|
? caught.message
|
||||||
|
: "Не удалось поставить расчёт в очередь.");
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
if (requestRef.current === request) requestRef.current = null;
|
||||||
|
}
|
||||||
|
}, [activeJob, selectionKey, setupId, sourceSessionId, state]);
|
||||||
|
|
||||||
|
return { jobs, latestJob, activeJob, state, error, refresh, submit };
|
||||||
|
}
|
||||||
|
|
||||||
|
function createIdempotencyKey(): string {
|
||||||
|
const entropy = typeof globalThis.crypto?.randomUUID === "function"
|
||||||
|
? globalThis.crypto.randomUUID()
|
||||||
|
: `${Date.now().toString(36)}-${Math.random().toString(16).slice(2)}`;
|
||||||
|
return `observatory-ui:${entropy}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ObservatoryRecordedJobsController = ReturnType<typeof useObservatoryRecordedJobs>;
|
||||||
@@ -20,8 +20,6 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.observatory-lead,
|
.observatory-lead,
|
||||||
.observatory-catalog-bar,
|
|
||||||
.observatory-catalog-bar__controls,
|
|
||||||
.observatory-notice {
|
.observatory-notice {
|
||||||
flex-wrap: wrap;
|
flex-wrap: wrap;
|
||||||
}
|
}
|
||||||
@@ -35,14 +33,12 @@
|
|||||||
.observatory-lead h2,
|
.observatory-lead h2,
|
||||||
.observatory-catalog-bar h3,
|
.observatory-catalog-bar h3,
|
||||||
.observatory-session-summary h3,
|
.observatory-session-summary h3,
|
||||||
.observatory-setup-detail h3,
|
|
||||||
.observatory-evidence h3 {
|
.observatory-evidence h3 {
|
||||||
margin: 0.3rem 0 0;
|
margin: 0.3rem 0 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
.observatory-catalog-bar h3,
|
.observatory-catalog-bar h3,
|
||||||
.observatory-session-summary h3,
|
.observatory-session-summary h3,
|
||||||
.observatory-setup-detail h3,
|
|
||||||
.observatory-evidence h3 {
|
.observatory-evidence h3 {
|
||||||
font-size: var(--nodedc-font-size-lg);
|
font-size: var(--nodedc-font-size-lg);
|
||||||
letter-spacing: -0.02em;
|
letter-spacing: -0.02em;
|
||||||
@@ -50,7 +46,6 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.observatory-lead p,
|
.observatory-lead p,
|
||||||
.observatory-catalog-bar p,
|
|
||||||
.observatory-state p,
|
.observatory-state p,
|
||||||
.observatory-evidence-empty p {
|
.observatory-evidence-empty p {
|
||||||
max-width: 52rem;
|
max-width: 52rem;
|
||||||
@@ -67,17 +62,37 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.observatory-catalog-bar__copy {
|
.observatory-catalog-bar__copy {
|
||||||
flex: 1 1 auto;
|
display: grid;
|
||||||
|
flex: 0 0 auto;
|
||||||
|
align-content: center;
|
||||||
|
gap: 0.1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.observatory-catalog-bar h3 {
|
||||||
|
margin-top: 0.1rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
.observatory-catalog-bar__controls {
|
.observatory-catalog-bar__controls {
|
||||||
flex: 2 1 48rem;
|
min-width: 0;
|
||||||
|
flex: 1 1 auto;
|
||||||
|
flex-wrap: nowrap;
|
||||||
justify-content: flex-end;
|
justify-content: flex-end;
|
||||||
}
|
}
|
||||||
|
|
||||||
.observatory-catalog-bar__controls .nodedc-select-anchor {
|
.observatory-catalog-bar__controls .nodedc-select-anchor {
|
||||||
flex: 1 1 18rem;
|
flex: 1 1 18rem;
|
||||||
min-width: min(22rem, 100%);
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.observatory-catalog-bar__controls > .nodedc-button {
|
||||||
|
flex: 0 0 auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.observatory-catalog-bar__run {
|
||||||
|
display: flex;
|
||||||
|
flex: 0 0 auto;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.55rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
.observatory-state {
|
.observatory-state {
|
||||||
@@ -105,9 +120,9 @@
|
|||||||
|
|
||||||
.observatory-session-summary {
|
.observatory-session-summary {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: minmax(14rem, 1.1fr) minmax(22rem, 1.6fr) minmax(14rem, auto);
|
grid-template-columns: minmax(11rem, 1.05fr) minmax(20rem, 1.6fr) minmax(11rem, auto);
|
||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 1rem;
|
gap: 0.75rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
.observatory-session-summary__identity {
|
.observatory-session-summary__identity {
|
||||||
@@ -131,7 +146,7 @@
|
|||||||
.observatory-session-summary__facts {
|
.observatory-session-summary__facts {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: repeat(3, minmax(6.5rem, 1fr));
|
grid-template-columns: repeat(3, minmax(6.5rem, 1fr));
|
||||||
gap: 1rem;
|
gap: 0.75rem;
|
||||||
margin: 0;
|
margin: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -159,7 +174,7 @@
|
|||||||
display: grid;
|
display: grid;
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
justify-items: end;
|
justify-items: end;
|
||||||
gap: 0.6rem;
|
gap: 0.4rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
.observatory-modalities {
|
.observatory-modalities {
|
||||||
@@ -171,135 +186,6 @@
|
|||||||
color: var(--nodedc-text-muted);
|
color: var(--nodedc-text-muted);
|
||||||
}
|
}
|
||||||
|
|
||||||
.observatory-setup-detail {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: minmax(0, 1fr) auto;
|
|
||||||
gap: 0.85rem 1rem;
|
|
||||||
container-name: observatory-setup;
|
|
||||||
container-type: inline-size;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__identity {
|
|
||||||
min-width: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__identity h3 {
|
|
||||||
margin: 0.15rem 0 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__identity p {
|
|
||||||
max-width: 68rem;
|
|
||||||
margin: 0.3rem 0 0;
|
|
||||||
color: var(--nodedc-text-muted);
|
|
||||||
font-size: var(--nodedc-font-size-sm);
|
|
||||||
line-height: 1.35;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__badges,
|
|
||||||
.observatory-setup-detail__actions,
|
|
||||||
.observatory-setup-detail__reason {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 0.55rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__badges {
|
|
||||||
align-self: start;
|
|
||||||
justify-content: flex-end;
|
|
||||||
flex-wrap: wrap;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__facts {
|
|
||||||
display: grid;
|
|
||||||
grid-column: 1 / -1;
|
|
||||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
|
||||||
gap: 0.75rem;
|
|
||||||
margin: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__facts > div {
|
|
||||||
display: grid;
|
|
||||||
min-width: 0;
|
|
||||||
gap: 0.2rem;
|
|
||||||
padding: 0.65rem 0.75rem;
|
|
||||||
border-radius: var(--nodedc-radius-control-compact);
|
|
||||||
background: var(--nodedc-glass-control-bg);
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__facts dt {
|
|
||||||
color: var(--nodedc-text-muted);
|
|
||||||
font-size: var(--nodedc-font-size-xs);
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__facts dd {
|
|
||||||
min-width: 0;
|
|
||||||
margin: 0;
|
|
||||||
overflow-wrap: anywhere;
|
|
||||||
color: var(--nodedc-text-secondary);
|
|
||||||
font-size: var(--nodedc-font-size-sm);
|
|
||||||
line-height: 1.35;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-results {
|
|
||||||
display: grid;
|
|
||||||
grid-column: 1 / -1;
|
|
||||||
gap: 0.35rem;
|
|
||||||
margin: 0;
|
|
||||||
padding: 0;
|
|
||||||
list-style: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-results li {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: auto minmax(0, 1fr) auto;
|
|
||||||
align-items: center;
|
|
||||||
gap: 0.65rem;
|
|
||||||
min-height: 3.15rem;
|
|
||||||
padding: 0.5rem 0.7rem;
|
|
||||||
border-radius: var(--nodedc-radius-control-compact);
|
|
||||||
background: var(--nodedc-glass-control-bg);
|
|
||||||
color: var(--nodedc-text-secondary);
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-results span {
|
|
||||||
display: grid;
|
|
||||||
min-width: 0;
|
|
||||||
gap: 0.1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-results strong,
|
|
||||||
.observatory-setup-results small {
|
|
||||||
overflow: hidden;
|
|
||||||
text-overflow: ellipsis;
|
|
||||||
white-space: nowrap;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-results small {
|
|
||||||
color: var(--nodedc-text-muted);
|
|
||||||
font-size: var(--nodedc-font-size-sm);
|
|
||||||
line-height: 1.35;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__footer {
|
|
||||||
display: flex;
|
|
||||||
grid-column: 1 / -1;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: space-between;
|
|
||||||
gap: 0.8rem;
|
|
||||||
padding-top: 0.1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__reason {
|
|
||||||
min-width: 0;
|
|
||||||
color: var(--nodedc-text-muted);
|
|
||||||
font-size: var(--nodedc-font-size-sm);
|
|
||||||
line-height: 1.4;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__actions {
|
|
||||||
flex: 0 0 auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-evidence {
|
.observatory-evidence {
|
||||||
display: grid;
|
display: grid;
|
||||||
gap: 0.65rem;
|
gap: 0.65rem;
|
||||||
@@ -454,7 +340,7 @@
|
|||||||
font-size: var(--nodedc-font-size-sm);
|
font-size: var(--nodedc-font-size-sm);
|
||||||
}
|
}
|
||||||
|
|
||||||
@container observatory-session (max-width: 56rem) {
|
@container observatory-session (max-width: 46rem) {
|
||||||
.observatory-session-summary {
|
.observatory-session-summary {
|
||||||
grid-template-columns: minmax(0, 1fr);
|
grid-template-columns: minmax(0, 1fr);
|
||||||
align-items: start;
|
align-items: start;
|
||||||
@@ -467,29 +353,6 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@container observatory-setup (max-width: 48rem) {
|
|
||||||
.observatory-setup-detail {
|
|
||||||
grid-template-columns: minmax(0, 1fr);
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__badges {
|
|
||||||
justify-content: flex-start;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__facts {
|
|
||||||
grid-template-columns: minmax(0, 1fr);
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__footer {
|
|
||||||
align-items: stretch;
|
|
||||||
flex-direction: column;
|
|
||||||
}
|
|
||||||
|
|
||||||
.observatory-setup-detail__actions {
|
|
||||||
flex-wrap: wrap;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@container observatory-session (max-width: 48rem) {
|
@container observatory-session (max-width: 48rem) {
|
||||||
.observatory-evidence-card {
|
.observatory-evidence-card {
|
||||||
grid-template-columns: auto minmax(0, 1fr) auto;
|
grid-template-columns: auto minmax(0, 1fr) auto;
|
||||||
@@ -535,4 +398,8 @@
|
|||||||
min-width: 0;
|
min-width: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.observatory-catalog-bar__run {
|
||||||
|
justify-content: flex-end;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,139 +0,0 @@
|
|||||||
import {
|
|
||||||
ActivityIndicator,
|
|
||||||
Button,
|
|
||||||
GlassSurface,
|
|
||||||
Icon,
|
|
||||||
StatusBadge,
|
|
||||||
} from "@nodedc/ui-react";
|
|
||||||
|
|
||||||
import type { ObservatoryEvidence } from "../../core/observatory/catalog";
|
|
||||||
import type { ObservatoryLaboratorySetup } from "../../core/observatory/laboratorySetups";
|
|
||||||
import type { ObservatoryRecordedRunBinding } from "../../core/observatory/recordedRun";
|
|
||||||
import type { SetupPreflightState } from "../../core/observatory/useObservatoryLaboratorySetups";
|
|
||||||
|
|
||||||
const relationLabel: Readonly<Record<string, string>> = {
|
|
||||||
"primary-visual": "Основной визуальный разбор",
|
|
||||||
"compute-successor": "Связанное вычислительное доказательство",
|
|
||||||
"canonical-projection": "Готовый записанный разбор",
|
|
||||||
};
|
|
||||||
|
|
||||||
function workerLabel(value: string): string {
|
|
||||||
const match = /^worker-(\d+)$/.exec(value);
|
|
||||||
return match ? `Worker ${match[1]}` : value;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function ObservatorySetupDetail({
|
|
||||||
setup,
|
|
||||||
evidence,
|
|
||||||
preflight,
|
|
||||||
onCheck,
|
|
||||||
onOpenExisting,
|
|
||||||
}: {
|
|
||||||
setup: ObservatoryLaboratorySetup;
|
|
||||||
evidence: readonly ObservatoryEvidence[];
|
|
||||||
preflight: SetupPreflightState;
|
|
||||||
onCheck: () => void;
|
|
||||||
onOpenExisting: (binding: ObservatoryRecordedRunBinding) => void;
|
|
||||||
}) {
|
|
||||||
const exactExisting = evidence.find(
|
|
||||||
(candidate) => candidate.recordedRun
|
|
||||||
&& setup.preflight.existingResultIds.includes(candidate.recordedRun.resultId),
|
|
||||||
)?.recordedRun ?? null;
|
|
||||||
const displayedReason = preflight.kind === "ready"
|
|
||||||
? preflight.value.checks.find((check) => check.checkId === "executor")?.message
|
|
||||||
?? setup.executor.reason
|
|
||||||
: preflight.kind === "error"
|
|
||||||
? preflight.message
|
|
||||||
: setup.preflight.reason;
|
|
||||||
|
|
||||||
return (
|
|
||||||
<GlassSurface className="observatory-setup-detail" padding="md">
|
|
||||||
<div className="observatory-setup-detail__identity">
|
|
||||||
<span className="section-eyebrow">СЕТАП ЛАБОРАТОРИИ</span>
|
|
||||||
<h3>{setup.displayName}</h3>
|
|
||||||
<p>{setup.description}</p>
|
|
||||||
</div>
|
|
||||||
<div className="observatory-setup-detail__badges">
|
|
||||||
<StatusBadge tone={setup.compatibility.compatible ? "success" : "warning"}>
|
|
||||||
{setup.compatibility.compatible ? "Совместим" : "Другая сессия"}
|
|
||||||
</StatusBadge>
|
|
||||||
<StatusBadge tone={setup.origin === "existing-result" ? "accent" : "neutral"}>
|
|
||||||
{setup.origin === "existing-result" ? "Готовый результат" : "Архивный сетап"}
|
|
||||||
</StatusBadge>
|
|
||||||
</div>
|
|
||||||
<dl className="observatory-setup-detail__facts">
|
|
||||||
<div>
|
|
||||||
<dt>Идентичность</dt>
|
|
||||||
<dd>
|
|
||||||
{setup.runDefinition
|
|
||||||
? `Версия ${setup.runDefinition.version} · конфигурация зафиксирована`
|
|
||||||
: "Готовый записанный результат без повторно запускаемой конфигурации"}
|
|
||||||
</dd>
|
|
||||||
</div>
|
|
||||||
<div>
|
|
||||||
<dt>Исполнитель</dt>
|
|
||||||
<dd>
|
|
||||||
{setup.preflight.outcome === "existing"
|
|
||||||
? "Не требуется для просмотра"
|
|
||||||
: `${workerLabel(setup.executor.contourId)} · повторный запуск не подключён`}
|
|
||||||
</dd>
|
|
||||||
</div>
|
|
||||||
<div>
|
|
||||||
<dt>Сохранено</dt>
|
|
||||||
<dd>{setup.preservedResults.length} неизменяемых результатов</dd>
|
|
||||||
</div>
|
|
||||||
</dl>
|
|
||||||
<ol className="observatory-setup-results" aria-label="Сохранённые результаты сетапа">
|
|
||||||
{setup.preservedResults.map((result) => (
|
|
||||||
<li key={result.resultId}>
|
|
||||||
<Icon name={result.access === "evidence-only" ? "database" : "clipboard"} size={15} />
|
|
||||||
<span>
|
|
||||||
<strong>{relationLabel[result.relation] ?? "Сохранённый результат"}</strong>
|
|
||||||
<small>
|
|
||||||
{result.access === "observatory"
|
|
||||||
? "Доступен в Обсерватории"
|
|
||||||
: result.access === "legacy-lab"
|
|
||||||
? "Сохранён в legacy LAB"
|
|
||||||
: "Сохранено как связанное доказательство"}
|
|
||||||
</small>
|
|
||||||
</span>
|
|
||||||
<StatusBadge tone={result.observatoryProjectionAvailable ? "success" : "neutral"}>
|
|
||||||
{result.access === "observatory"
|
|
||||||
? result.observatoryProjectionAvailable ? "Готов" : "Сохранён"
|
|
||||||
: result.access === "legacy-lab" ? "Legacy LAB" : "Evidence"}
|
|
||||||
</StatusBadge>
|
|
||||||
</li>
|
|
||||||
))}
|
|
||||||
</ol>
|
|
||||||
<div className="observatory-setup-detail__footer">
|
|
||||||
<div className="observatory-setup-detail__reason" role={preflight.kind === "error" ? "alert" : "status"}>
|
|
||||||
<Icon name={preflight.kind === "error" ? "alert" : "activity"} size={16} />
|
|
||||||
<span>{displayedReason}</span>
|
|
||||||
</div>
|
|
||||||
<div className="observatory-setup-detail__actions">
|
|
||||||
{exactExisting ? (
|
|
||||||
<Button
|
|
||||||
size="compact"
|
|
||||||
variant="secondary"
|
|
||||||
icon={<Icon name="eye" size={15} />}
|
|
||||||
onClick={() => onOpenExisting(exactExisting)}
|
|
||||||
>
|
|
||||||
Открыть готовый результат
|
|
||||||
</Button>
|
|
||||||
) : (
|
|
||||||
<Button
|
|
||||||
size="compact"
|
|
||||||
variant="secondary"
|
|
||||||
disabled={preflight.kind === "checking"}
|
|
||||||
onClick={onCheck}
|
|
||||||
>
|
|
||||||
{preflight.kind === "checking"
|
|
||||||
? <ActivityIndicator size="compact" label="Проверяем совместимость" />
|
|
||||||
: "Проверить совместимость"}
|
|
||||||
</Button>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</GlassSurface>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -32,8 +32,9 @@ import {
|
|||||||
} from "../../core/observatory/catalogMutations";
|
} from "../../core/observatory/catalogMutations";
|
||||||
import { useObservatoryCatalog } from "../../core/observatory/useObservatoryCatalog";
|
import { useObservatoryCatalog } from "../../core/observatory/useObservatoryCatalog";
|
||||||
import { useObservatoryLaboratorySetups } from "../../core/observatory/useObservatoryLaboratorySetups";
|
import { useObservatoryLaboratorySetups } from "../../core/observatory/useObservatoryLaboratorySetups";
|
||||||
|
import { useObservatoryRecordedJobs } from "../../core/observatory/useObservatoryRecordedJobs";
|
||||||
|
import type { ObservatoryRecordedJobState } from "../../core/observatory/recordedJobs";
|
||||||
import type { WorkspaceDefinition } from "../../productModel";
|
import type { WorkspaceDefinition } from "../../productModel";
|
||||||
import { ObservatorySetupDetail } from "./ObservatorySetupDetail";
|
|
||||||
|
|
||||||
const MAX_PRESENTED_EVIDENCE = 6;
|
const MAX_PRESENTED_EVIDENCE = 6;
|
||||||
const EMPTY_OBSERVATORY_ITEMS = [] as const;
|
const EMPTY_OBSERVATORY_ITEMS = [] as const;
|
||||||
@@ -87,6 +88,27 @@ const modalityLabel: Record<string, string> = {
|
|||||||
telemetry: "Телеметрия",
|
telemetry: "Телеметрия",
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const recordedJobStatus: Record<
|
||||||
|
ObservatoryRecordedJobState,
|
||||||
|
{
|
||||||
|
readonly label: string;
|
||||||
|
readonly tone: "success" | "accent" | "warning" | "danger" | "neutral";
|
||||||
|
}
|
||||||
|
> = {
|
||||||
|
accepted: { label: "Принят", tone: "neutral" },
|
||||||
|
queued: { label: "Ждёт Worker", tone: "neutral" },
|
||||||
|
claimed: { label: "Назначен Worker", tone: "accent" },
|
||||||
|
running: { label: "Выполняется", tone: "accent" },
|
||||||
|
paused: { label: "Пауза: live-поток", tone: "warning" },
|
||||||
|
"preemption-pending": {
|
||||||
|
label: "Ждём подтверждения остановки",
|
||||||
|
tone: "warning",
|
||||||
|
},
|
||||||
|
succeeded: { label: "Готово", tone: "success" },
|
||||||
|
failed: { label: "Ошибка расчёта", tone: "danger" },
|
||||||
|
"reconciliation-required": { label: "Нужна сверка", tone: "warning" },
|
||||||
|
};
|
||||||
|
|
||||||
function statusTone(
|
function statusTone(
|
||||||
status: ObservationSessionStatus,
|
status: ObservationSessionStatus,
|
||||||
): "success" | "accent" | "warning" | "danger" | "neutral" {
|
): "success" | "accent" | "warning" | "danger" | "neutral" {
|
||||||
@@ -136,6 +158,10 @@ export function ObservatoryWorkspace({
|
|||||||
const controller = useObservatoryCatalog();
|
const controller = useObservatoryCatalog();
|
||||||
const [selectedSessionId, setSelectedSessionId] = useState("");
|
const [selectedSessionId, setSelectedSessionId] = useState("");
|
||||||
const setupController = useObservatoryLaboratorySetups(selectedSessionId);
|
const setupController = useObservatoryLaboratorySetups(selectedSessionId);
|
||||||
|
const recordedJobsController = useObservatoryRecordedJobs(
|
||||||
|
selectedSessionId,
|
||||||
|
setupController.selectedSetupId,
|
||||||
|
);
|
||||||
const [replay, setReplay] = useState<ObservatoryReplayState>({ kind: "closed" });
|
const [replay, setReplay] = useState<ObservatoryReplayState>({ kind: "closed" });
|
||||||
const [renameTarget, setRenameTarget] = useState<ObservatoryEvidence | null>(null);
|
const [renameTarget, setRenameTarget] = useState<ObservatoryEvidence | null>(null);
|
||||||
const [renameValue, setRenameValue] = useState("");
|
const [renameValue, setRenameValue] = useState("");
|
||||||
@@ -188,6 +214,29 @@ export function ObservatoryWorkspace({
|
|||||||
: "Несовместим с выбранной сессией",
|
: "Несовместим с выбранной сессией",
|
||||||
})) ?? []
|
})) ?? []
|
||||||
), [setupController.catalog]);
|
), [setupController.catalog]);
|
||||||
|
const runPreflight = setupController.preflight.kind === "ready"
|
||||||
|
? setupController.preflight.value
|
||||||
|
: null;
|
||||||
|
const queueSubmissionAllowed = Boolean(
|
||||||
|
setupController.selectedSetup?.runDefinition
|
||||||
|
&& setupController.selectedSetup.compatibility.compatible
|
||||||
|
&& runPreflight?.outcome === "queueable"
|
||||||
|
&& runPreflight.submissionAllowed,
|
||||||
|
);
|
||||||
|
const presentedJob = recordedJobsController.activeJob
|
||||||
|
?? recordedJobsController.latestJob;
|
||||||
|
const presentedJobStatus = presentedJob
|
||||||
|
? recordedJobStatus[presentedJob.state]
|
||||||
|
: null;
|
||||||
|
const queueStatusError = setupController.preflight.kind === "error"
|
||||||
|
? setupController.preflight.message
|
||||||
|
: recordedJobsController.error;
|
||||||
|
const queueStateBusy = recordedJobsController.state === "loading"
|
||||||
|
|| recordedJobsController.state === "refreshing"
|
||||||
|
|| recordedJobsController.state === "submitting";
|
||||||
|
const canSubmitRecordedJob = queueSubmissionAllowed
|
||||||
|
&& recordedJobsController.state === "ready"
|
||||||
|
&& recordedJobsController.activeJob === null;
|
||||||
const initialLoading = !controller.catalog
|
const initialLoading = !controller.catalog
|
||||||
&& ["idle", "loading"].includes(controller.state);
|
&& ["idle", "loading"].includes(controller.state);
|
||||||
const unavailable = !controller.catalog && controller.state === "error";
|
const unavailable = !controller.catalog && controller.state === "error";
|
||||||
@@ -374,11 +423,10 @@ export function ObservatoryWorkspace({
|
|||||||
</StatusBadge>
|
</StatusBadge>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<GlassSurface className="observatory-catalog-bar" padding="md">
|
<GlassSurface className="observatory-catalog-bar" padding="sm">
|
||||||
<div className="observatory-catalog-bar__copy">
|
<div className="observatory-catalog-bar__copy">
|
||||||
<span className="section-eyebrow">ИСТОЧНИК ДОКАЗАТЕЛЬСТВ</span>
|
<span className="section-eyebrow">ИСТОЧНИК ДОКАЗАТЕЛЬСТВ</span>
|
||||||
<h3>Сохранённая сессия</h3>
|
<h3>Сохранённая сессия</h3>
|
||||||
<p>Выбор меняет только читаемую карточку и не готовит визуальный разбор в фоне.</p>
|
|
||||||
</div>
|
</div>
|
||||||
<div className="observatory-catalog-bar__controls">
|
<div className="observatory-catalog-bar__controls">
|
||||||
<Select
|
<Select
|
||||||
@@ -418,10 +466,40 @@ export function ObservatoryWorkspace({
|
|||||||
onClick={() => {
|
onClick={() => {
|
||||||
void controller.refresh();
|
void controller.refresh();
|
||||||
setupController.refresh();
|
setupController.refresh();
|
||||||
|
recordedJobsController.refresh();
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
Обновить
|
Обновить
|
||||||
</Button>
|
</Button>
|
||||||
|
<div className="observatory-catalog-bar__run" aria-live="polite">
|
||||||
|
{queueStatusError ? (
|
||||||
|
<StatusBadge tone="danger" title={queueStatusError}>
|
||||||
|
Очередь недоступна
|
||||||
|
</StatusBadge>
|
||||||
|
) : recordedJobsController.state === "submitting" ? (
|
||||||
|
<StatusBadge tone="neutral">Ставим в очередь</StatusBadge>
|
||||||
|
) : presentedJobStatus ? (
|
||||||
|
<StatusBadge
|
||||||
|
tone={presentedJobStatus.tone}
|
||||||
|
title={presentedJob?.terminalMessage ?? undefined}
|
||||||
|
>
|
||||||
|
{presentedJobStatus.label}
|
||||||
|
</StatusBadge>
|
||||||
|
) : setupController.preflight.kind === "checking" ? (
|
||||||
|
<StatusBadge tone="neutral">Проверяем сетап</StatusBadge>
|
||||||
|
) : queueStateBusy ? (
|
||||||
|
<StatusBadge tone="neutral">Читаем очередь</StatusBadge>
|
||||||
|
) : null}
|
||||||
|
{canSubmitRecordedJob ? (
|
||||||
|
<Button
|
||||||
|
size="compact"
|
||||||
|
variant="primary"
|
||||||
|
onClick={() => { void recordedJobsController.submit(); }}
|
||||||
|
>
|
||||||
|
Рассчитать
|
||||||
|
</Button>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</GlassSurface>
|
</GlassSurface>
|
||||||
|
|
||||||
@@ -476,7 +554,7 @@ export function ObservatoryWorkspace({
|
|||||||
</GlassSurface>
|
</GlassSurface>
|
||||||
) : selectedSession ? (
|
) : selectedSession ? (
|
||||||
<section className="observatory-session-stack" aria-label="Выбранная сессия и связанные результаты">
|
<section className="observatory-session-stack" aria-label="Выбранная сессия и связанные результаты">
|
||||||
<GlassSurface className="observatory-session-summary" padding="md">
|
<GlassSurface className="observatory-session-summary" padding="sm">
|
||||||
<div className="observatory-session-summary__identity">
|
<div className="observatory-session-summary__identity">
|
||||||
<span className="section-eyebrow">ИСХОДНАЯ СЕССИЯ</span>
|
<span className="section-eyebrow">ИСХОДНАЯ СЕССИЯ</span>
|
||||||
<h3>{selectedSession.source.label}</h3>
|
<h3>{selectedSession.source.label}</h3>
|
||||||
@@ -506,22 +584,6 @@ export function ObservatoryWorkspace({
|
|||||||
</div>
|
</div>
|
||||||
</GlassSurface>
|
</GlassSurface>
|
||||||
|
|
||||||
{setupController.selectedSetup ? (
|
|
||||||
<ObservatorySetupDetail
|
|
||||||
setup={setupController.selectedSetup}
|
|
||||||
evidence={selectedSession.evidence}
|
|
||||||
preflight={setupController.preflight}
|
|
||||||
onCheck={() => { void setupController.check(); }}
|
|
||||||
onOpenExisting={openReplay}
|
|
||||||
/>
|
|
||||||
) : setupController.error ? (
|
|
||||||
<GlassSurface className="observatory-notice" padding="md" tone="soft" role="alert">
|
|
||||||
<StatusBadge tone="warning">Сетапы недоступны</StatusBadge>
|
|
||||||
<span className="observatory-notice__copy">{setupController.error}</span>
|
|
||||||
<Button size="compact" variant="ghost" onClick={setupController.refresh}>Повторить</Button>
|
|
||||||
</GlassSurface>
|
|
||||||
) : null}
|
|
||||||
|
|
||||||
<section className="observatory-evidence">
|
<section className="observatory-evidence">
|
||||||
<header>
|
<header>
|
||||||
<div>
|
<div>
|
||||||
|
|||||||
@@ -156,6 +156,40 @@ test("Observatory preflight sends the exact selected definition and never submit
|
|||||||
assert.equal(preflight.submissionAllowed, false);
|
assert.equal(preflight.submissionAllowed, false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("Observatory dynamic preflight admits only an explicit queueable response", async () => {
|
||||||
|
const selected = (await fetchObservatoryLaboratorySetups("source-a", {
|
||||||
|
fetcher: async () => new Response(JSON.stringify({
|
||||||
|
schema_version: "missioncore.observatory-laboratory-setup-catalog/v1",
|
||||||
|
source_session_id: "source-a",
|
||||||
|
setups: [setup()],
|
||||||
|
authority,
|
||||||
|
}), { status: 200 }),
|
||||||
|
})).setups[0];
|
||||||
|
|
||||||
|
const preflight = await preflightObservatoryLaboratorySetup("source-a", selected, {
|
||||||
|
fetcher: async () => new Response(JSON.stringify({
|
||||||
|
schema_version: "missioncore.observatory-run-preflight/v1",
|
||||||
|
source_session_id: "source-a",
|
||||||
|
setup_id: selected.setupId,
|
||||||
|
definition_sha256: "b".repeat(64),
|
||||||
|
outcome: "queueable",
|
||||||
|
submission_allowed: true,
|
||||||
|
checks: [{
|
||||||
|
check_id: "durable-queue",
|
||||||
|
outcome: "pass",
|
||||||
|
reason_code: "exact-binding-ready",
|
||||||
|
message: "Точный источник и очередь готовы.",
|
||||||
|
}],
|
||||||
|
existing_result_ids: [],
|
||||||
|
executor: setup().executor,
|
||||||
|
authority,
|
||||||
|
}), { status: 200 }),
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.equal(preflight.outcome, "queueable");
|
||||||
|
assert.equal(preflight.submissionAllowed, true);
|
||||||
|
});
|
||||||
|
|
||||||
test("Observatory setup contract rejects authority escalation and response drift", async () => {
|
test("Observatory setup contract rejects authority escalation and response drift", async () => {
|
||||||
const escalated = setup();
|
const escalated = setup();
|
||||||
escalated.authority = { ...authority, commands_enabled: true };
|
escalated.authority = { ...authority, commands_enabled: true };
|
||||||
|
|||||||
@@ -0,0 +1,179 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { after, before, test } from "node:test";
|
||||||
|
|
||||||
|
import { createServer } from "vite";
|
||||||
|
|
||||||
|
let server;
|
||||||
|
let fetchObservatoryRecordedJobs;
|
||||||
|
let submitObservatoryRecordedJob;
|
||||||
|
let ObservatoryRecordedJobContractError;
|
||||||
|
|
||||||
|
const authority = {
|
||||||
|
commands_enabled: false,
|
||||||
|
actuation_allowed: false,
|
||||||
|
navigation_or_safety_accepted: false,
|
||||||
|
production_accepted: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
function job(state = "queued") {
|
||||||
|
return {
|
||||||
|
schema_version: "missioncore.observatory-recorded-job/v1",
|
||||||
|
job_id: "observatory-run-0123456789abcdef0123456789abcdef",
|
||||||
|
idempotency_key: "observatory-ui:source-a:m49-tgs:request-a",
|
||||||
|
request_sha256: "1".repeat(64),
|
||||||
|
identity_sha256: "2".repeat(64),
|
||||||
|
submission_receipt_sha256: "3".repeat(64),
|
||||||
|
source: {
|
||||||
|
session_id: "source-a",
|
||||||
|
catalog_sha256: "4".repeat(64),
|
||||||
|
bundle_sha256: "5".repeat(64),
|
||||||
|
capability_manifest_sha256: "6".repeat(64),
|
||||||
|
adapter: {
|
||||||
|
adapter_id: "ravnoves00-m49-source-pack",
|
||||||
|
version: 1,
|
||||||
|
adapter_sha256: "7".repeat(64),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
setup: {
|
||||||
|
setup_id: "m49-tgs",
|
||||||
|
definition_id: "m49-tgs",
|
||||||
|
definition_version: 1,
|
||||||
|
definition_sha256: "8".repeat(64),
|
||||||
|
},
|
||||||
|
executor: {
|
||||||
|
release_id: "m49-tgs-release",
|
||||||
|
release_sha256: "9".repeat(64),
|
||||||
|
image_sha256: "a".repeat(64),
|
||||||
|
model_release_ids: [],
|
||||||
|
learned_models: [],
|
||||||
|
model_manifest_sha256: "b".repeat(64),
|
||||||
|
resource_profile_id: "m49-tgs-cpu",
|
||||||
|
resource_profile_sha256: "c".repeat(64),
|
||||||
|
},
|
||||||
|
checkpoint_policy: {
|
||||||
|
mode: "non-checkpointable",
|
||||||
|
allowed_checkpoints: [],
|
||||||
|
last_checkpoint_id: null,
|
||||||
|
},
|
||||||
|
priority: { class: "recorded", rank: 100, server_owned: true },
|
||||||
|
state,
|
||||||
|
preemption_requested: state === "preemption-pending",
|
||||||
|
restart_from_zero: state === "paused",
|
||||||
|
preemption_receipt_sha256: null,
|
||||||
|
claim_generation: 0,
|
||||||
|
result: state === "succeeded"
|
||||||
|
? { result_id: "m49-result", sha256: "d".repeat(64) }
|
||||||
|
: null,
|
||||||
|
terminal: state === "failed"
|
||||||
|
? { code: "worker-failed", message: "Worker завершил расчёт с ошибкой." }
|
||||||
|
: null,
|
||||||
|
created_at_utc: "2026-08-31T10:00:00Z",
|
||||||
|
updated_at_utc: "2026-08-31T10:00:01Z",
|
||||||
|
authority,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
before(async () => {
|
||||||
|
server = await createServer({
|
||||||
|
appType: "custom",
|
||||||
|
logLevel: "silent",
|
||||||
|
server: { middlewareMode: true },
|
||||||
|
});
|
||||||
|
({
|
||||||
|
fetchObservatoryRecordedJobs,
|
||||||
|
submitObservatoryRecordedJob,
|
||||||
|
ObservatoryRecordedJobContractError,
|
||||||
|
} = await server.ssrLoadModule("/src/core/observatory/recordedJobs.ts"));
|
||||||
|
});
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
await server?.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Observatory reads the exact session/setup queue without polling", async () => {
|
||||||
|
const calls = [];
|
||||||
|
const jobs = await fetchObservatoryRecordedJobs("source-a", "m49-tgs", {
|
||||||
|
fetcher: async (input, init) => {
|
||||||
|
calls.push({ input: String(input), init });
|
||||||
|
return new Response(JSON.stringify({
|
||||||
|
schema_version: "missioncore.observatory-recorded-job-list/v1",
|
||||||
|
items: [job("running")],
|
||||||
|
authority,
|
||||||
|
}), { status: 200 });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.equal(calls.length, 1);
|
||||||
|
assert.equal(
|
||||||
|
calls[0].input,
|
||||||
|
"/api/v1/observatory/runs?source_session_id=source-a&setup_id=m49-tgs&limit=20",
|
||||||
|
);
|
||||||
|
assert.equal(calls[0].init.method, "GET");
|
||||||
|
assert.equal(jobs[0].state, "running");
|
||||||
|
assert.equal(jobs[0].definitionSha256, "8".repeat(64));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Observatory submits only public identities and accepts every durable state", async () => {
|
||||||
|
const states = [
|
||||||
|
"accepted",
|
||||||
|
"queued",
|
||||||
|
"claimed",
|
||||||
|
"running",
|
||||||
|
"paused",
|
||||||
|
"preemption-pending",
|
||||||
|
"succeeded",
|
||||||
|
"failed",
|
||||||
|
"reconciliation-required",
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const state of states) {
|
||||||
|
let request;
|
||||||
|
const result = await submitObservatoryRecordedJob(
|
||||||
|
"source-a",
|
||||||
|
"m49-tgs",
|
||||||
|
"observatory-ui:source-a:m49-tgs:request-a",
|
||||||
|
{
|
||||||
|
fetcher: async (input, init) => {
|
||||||
|
request = { input: String(input), init };
|
||||||
|
return new Response(JSON.stringify(job(state)), { status: 200 });
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(result.state, state);
|
||||||
|
assert.equal(request.input, "/api/v1/observatory/runs");
|
||||||
|
assert.equal(request.init.method, "POST");
|
||||||
|
assert.deepEqual(JSON.parse(request.init.body), {
|
||||||
|
schema_version: "missioncore.observatory-recorded-run-submit/v1",
|
||||||
|
idempotency_key: "observatory-ui:source-a:m49-tgs:request-a",
|
||||||
|
source_session_id: "source-a",
|
||||||
|
setup_id: "m49-tgs",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Observatory queue contract rejects authority escalation and response drift", async () => {
|
||||||
|
const escalated = job();
|
||||||
|
escalated.authority = { ...authority, commands_enabled: true };
|
||||||
|
await assert.rejects(
|
||||||
|
fetchObservatoryRecordedJobs("source-a", "m49-tgs", {
|
||||||
|
fetcher: async () => new Response(JSON.stringify({
|
||||||
|
schema_version: "missioncore.observatory-recorded-job-list/v1",
|
||||||
|
items: [escalated],
|
||||||
|
authority,
|
||||||
|
}), { status: 200 }),
|
||||||
|
}),
|
||||||
|
ObservatoryRecordedJobContractError,
|
||||||
|
);
|
||||||
|
|
||||||
|
const drifted = job();
|
||||||
|
drifted.worker_command = "forbidden";
|
||||||
|
await assert.rejects(
|
||||||
|
submitObservatoryRecordedJob(
|
||||||
|
"source-a",
|
||||||
|
"m49-tgs",
|
||||||
|
"observatory-ui:source-a:m49-tgs:request-a",
|
||||||
|
{ fetcher: async () => new Response(JSON.stringify(drifted), { status: 200 }) },
|
||||||
|
),
|
||||||
|
ObservatoryRecordedJobContractError,
|
||||||
|
);
|
||||||
|
});
|
||||||
@@ -144,11 +144,11 @@ test("Observatory mounts the one shared canonical replay only after explicit adm
|
|||||||
);
|
);
|
||||||
assert.match(
|
assert.match(
|
||||||
styles,
|
styles,
|
||||||
/\.observatory-catalog-bar h3,[\s\S]*\.observatory-session-summary h3,[\s\S]*\.observatory-setup-detail h3,[\s\S]*\.observatory-evidence h3 \{[\s\S]*font-size: var\(--nodedc-font-size-lg\);/,
|
/\.observatory-catalog-bar h3,[\s\S]*\.observatory-session-summary h3,[\s\S]*\.observatory-evidence h3 \{[\s\S]*font-size: var\(--nodedc-font-size-lg\);/,
|
||||||
);
|
);
|
||||||
assert.match(
|
assert.match(
|
||||||
styles,
|
styles,
|
||||||
/@container observatory-session \(max-width: 56rem\) \{[\s\S]*\.observatory-session-summary \{[\s\S]*grid-template-columns: minmax\(0, 1fr\);/,
|
/@container observatory-session \(max-width: 46rem\) \{[\s\S]*\.observatory-session-summary \{[\s\S]*grid-template-columns: minmax\(0, 1fr\);/,
|
||||||
);
|
);
|
||||||
assert.match(
|
assert.match(
|
||||||
styles,
|
styles,
|
||||||
@@ -206,23 +206,67 @@ test("Observatory rename and delete use admitted projection mutations and canoni
|
|||||||
assert.match(hook, /applyObservatoryCatalogMutationOverlay\(/);
|
assert.match(hook, /applyObservatoryCatalogMutationOverlay\(/);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("Observatory configurator keeps selection and preflight read-only", async () => {
|
test("Observatory keeps one compact selector axis without the obsolete setup detail", async () => {
|
||||||
const [workspace, setupDetail, setupHook] = await Promise.all([
|
const [workspace, styles, setupHook, jobsHook] = await Promise.all([
|
||||||
read("workspaces/observatory/ObservatoryWorkspace.tsx"),
|
read("workspaces/observatory/ObservatoryWorkspace.tsx"),
|
||||||
read("workspaces/observatory/ObservatorySetupDetail.tsx"),
|
read("styles/observatory.css"),
|
||||||
read("core/observatory/useObservatoryLaboratorySetups.ts"),
|
read("core/observatory/useObservatoryLaboratorySetups.ts"),
|
||||||
|
read("core/observatory/useObservatoryRecordedJobs.ts"),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
assert.match(workspace, /className="observatory-catalog-bar" padding="sm"/);
|
||||||
assert.match(workspace, /label="Выбрать сохранённую сессию"/);
|
assert.match(workspace, /label="Выбрать сохранённую сессию"/);
|
||||||
assert.match(workspace, /label="Выбрать сетап лаборатории"/);
|
assert.match(workspace, /label="Выбрать сетап лаборатории"/);
|
||||||
assert.match(workspace, /Показан последний каталог сетапов/);
|
assert.match(workspace, /Показан последний каталог сетапов/);
|
||||||
assert.match(setupDetail, /Проверить совместимость/);
|
assert.doesNotMatch(workspace, /Выбор меняет только читаемую карточку/);
|
||||||
assert.doesNotMatch(
|
assert.doesNotMatch(workspace, /ObservatorySetupDetail|observatory-setup-detail/);
|
||||||
setupDetail,
|
assert.doesNotMatch(styles, /observatory-setup-detail|observatory-setup-results/);
|
||||||
/Рассчитать лабораторию|definitionSha256|<small>\{result\.resultId\}/,
|
assert.match(workspace, /useObservatoryRecordedJobs/);
|
||||||
|
assert.match(
|
||||||
|
workspace,
|
||||||
|
/runPreflight\?\.outcome === "queueable"[\s\S]*runPreflight\.submissionAllowed/,
|
||||||
|
);
|
||||||
|
assert.match(
|
||||||
|
workspace,
|
||||||
|
/canSubmitRecordedJob \? \([\s\S]*>\s*Рассчитать\s*<\/Button>/,
|
||||||
|
);
|
||||||
|
assert.match(workspace, /accepted: \{ label: "Принят"/);
|
||||||
|
assert.match(workspace, /queued: \{ label: "Ждёт Worker"/);
|
||||||
|
assert.match(workspace, /claimed: \{ label: "Назначен Worker"/);
|
||||||
|
assert.match(workspace, /running: \{ label: "Выполняется"/);
|
||||||
|
assert.match(workspace, /paused: \{ label: "Пауза: live-поток"/);
|
||||||
|
assert.match(
|
||||||
|
workspace,
|
||||||
|
/"preemption-pending": \{[\s\S]*label: "Ждём подтверждения остановки"/,
|
||||||
|
);
|
||||||
|
assert.match(workspace, /succeeded: \{ label: "Готово"/);
|
||||||
|
assert.match(workspace, /failed: \{ label: "Ошибка расчёта"/);
|
||||||
|
assert.match(workspace, /"reconciliation-required": \{ label: "Нужна сверка"/);
|
||||||
|
assert.match(
|
||||||
|
jobsHook,
|
||||||
|
/OPEN_STATES[\s\S]*"preemption-pending",[\s\S]*"reconciliation-required"/,
|
||||||
|
);
|
||||||
|
assert.match(jobsHook, /return `observatory-ui:\$\{entropy\}`;/);
|
||||||
|
assert.doesNotMatch(jobsHook, /observatory-ui:[^`]*sourceSessionId|\.slice\(0, 160\)/);
|
||||||
|
assert.match(workspace, /recordedJobsController\.refresh\(\)/);
|
||||||
|
assert.doesNotMatch(`${workspace}\n${jobsHook}`, /setInterval|setTimeout/);
|
||||||
|
assert.match(
|
||||||
|
styles,
|
||||||
|
/\.observatory-catalog-bar__controls \{[\s\S]*min-width: 0;[\s\S]*flex: 1 1 auto;[\s\S]*flex-wrap: nowrap;[\s\S]*justify-content: flex-end;/,
|
||||||
|
);
|
||||||
|
assert.match(
|
||||||
|
styles,
|
||||||
|
/\.observatory-catalog-bar__run \{[\s\S]*display: flex;[\s\S]*align-items: center;/,
|
||||||
|
);
|
||||||
|
assert.match(
|
||||||
|
styles,
|
||||||
|
/@media \(max-width: 920px\)[\s\S]*\.observatory-catalog-bar,[\s\S]*\.observatory-catalog-bar__controls,[\s\S]*flex-direction: column;/,
|
||||||
);
|
);
|
||||||
assert.match(setupHook, /catalog\?\.sourceSessionId === sourceSessionId/);
|
assert.match(setupHook, /catalog\?\.sourceSessionId === sourceSessionId/);
|
||||||
assert.match(setupHook, /preflightRequest\.current\?\.abort\(\)/);
|
assert.match(setupHook, /preflightRequest\.current\?\.abort\(\)/);
|
||||||
assert.match(setupHook, /preflightRequest\.current !== request/);
|
assert.match(setupHook, /preflightRequest\.current !== request/);
|
||||||
assert.doesNotMatch(`${workspace}\n${setupDetail}\n${setupHook}`, /\/api\/v1\/observatory\/runs/);
|
assert.match(
|
||||||
|
setupHook,
|
||||||
|
/state !== "ready" \|\| !selectedSetup \|\| preflight\.kind !== "idle"[\s\S]*void check\(\)/,
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -3,8 +3,8 @@
|
|||||||
"setups": [
|
"setups": [
|
||||||
{
|
{
|
||||||
"setup_id": "m49-tgs-full-shadow-v1",
|
"setup_id": "m49-tgs-full-shadow-v1",
|
||||||
"display_name": "M4.9T5 · TRAVEL TGS · полный source-paced shadow",
|
"display_name": "M4.9T5 · TRAVEL TGS · CPU-only, без ML",
|
||||||
"description": "Сохранённая CPU-конфигурация RAVNOVES00: 4 489 кадров, causal rolling 1 s, полный визуальный разбор и отдельное доказательство integrated graph.",
|
"description": "Сохранённая конфигурация RAVNOVES00: TRAVEL TGS без ML-моделей, 4 489 кадров, causal rolling 1 s, полный визуальный разбор и отдельное доказательство integrated graph.",
|
||||||
"origin": "archived-definition",
|
"origin": "archived-definition",
|
||||||
"source": {
|
"source": {
|
||||||
"session_id": "20260720T065719Z_viewer_live",
|
"session_id": "20260720T065719Z_viewer_live",
|
||||||
@@ -58,8 +58,8 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"setup_id": "lab-v1-ravnoves004tree-final",
|
"setup_id": "lab-v1-ravnoves004tree-final",
|
||||||
"display_name": "LAB V1 · финальный записанный контур восприятия",
|
"display_name": "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||||
"description": "Текущий точный RAVNOVES004TREE result: 6 830 кадров EoMT + DDRNet и синхронный записанный replay без доступа к управлению.",
|
"description": "Точный RAVNOVES004TREE result: 6 830 кадров EoMT Cityscapes Large 1024 + DDRNet-39 и синхронный записанный replay; полный TGS и независимый YOLOX отсутствуют.",
|
||||||
"origin": "existing-result",
|
"origin": "existing-result",
|
||||||
"source": {
|
"source": {
|
||||||
"session_id": "20260828T130511Z_viewer_live",
|
"session_id": "20260828T130511Z_viewer_live",
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
{
|
||||||
|
"schema_version": "missioncore.observatory-m49-recorded-queue-binding/v1",
|
||||||
|
"binding_id": "m49-ravnoves00-recorded-queue-v1",
|
||||||
|
"source": {
|
||||||
|
"session_id": "20260720T065719Z_viewer_live",
|
||||||
|
"label": "RAVNOVES00",
|
||||||
|
"required_modalities": [
|
||||||
|
"point-cloud",
|
||||||
|
"trajectory",
|
||||||
|
"video"
|
||||||
|
],
|
||||||
|
"source_pack": {
|
||||||
|
"artifact_id": "e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b",
|
||||||
|
"sha256": "0685d24219d8236caf8b7f1685e93f6d6b59e7fd015a768d88a92bbe8b154944",
|
||||||
|
"byte_length": 72996000,
|
||||||
|
"media_type": "application/vnd.nodedc.lidar-source-pack+npz",
|
||||||
|
"expected_timeline_frames": 4489,
|
||||||
|
"expected_available_lidar_frames": 3928
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"setup": {
|
||||||
|
"setup_id": "m49-tgs-full-shadow-v1",
|
||||||
|
"definition_id": "m49-tgs-full-shadow",
|
||||||
|
"definition_version": 1,
|
||||||
|
"definition_sha256": "836a66639e69f7ea00de2a9111c1c6c9c3c00f1abd726f1df596aeb4e3a88ae6"
|
||||||
|
},
|
||||||
|
"source_adapter": {
|
||||||
|
"adapter_id": "ravnoves00-m49-source-pack",
|
||||||
|
"version": 1
|
||||||
|
},
|
||||||
|
"executor": {
|
||||||
|
"release_id": "m49-tgs-full-shadow-worker-release",
|
||||||
|
"artifact_sha256": "5e0ea16c7a5cc760463836718b0cd8b0006ffc4b202e5f706a20a86ef2f912ab",
|
||||||
|
"code_revision": "40c850b167dda366d8aa45d828520168affaf9fd",
|
||||||
|
"service_installed": false,
|
||||||
|
"image_set": {
|
||||||
|
"schema_version": "missioncore.observatory-executor-image-set/v1",
|
||||||
|
"images": {
|
||||||
|
"travel": "7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f",
|
||||||
|
"parity": "ceb13548617e4bd3f619766bfdff00af3fa5160946b367828da6d2233dcdcba0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"learned_models": []
|
||||||
|
},
|
||||||
|
"resource_profile": {
|
||||||
|
"schema_version": "missioncore.observatory-recorded-resource-profile/v1",
|
||||||
|
"profile_id": "worker006-cpu-single-run-v1",
|
||||||
|
"contour_id": "worker-006",
|
||||||
|
"concurrency": 1,
|
||||||
|
"checkpoint_policy": "non-checkpointable",
|
||||||
|
"restart_from_zero": true,
|
||||||
|
"staging_discard_required": true,
|
||||||
|
"resource_release_receipt_required": true
|
||||||
|
},
|
||||||
|
"authority": {
|
||||||
|
"commands_enabled": false,
|
||||||
|
"actuation_allowed": false,
|
||||||
|
"navigation_or_safety_accepted": false,
|
||||||
|
"production_accepted": false
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -275,33 +275,64 @@ execution canaries. Once they prove one automatic run-to-evidence path, the LAB
|
|||||||
canonicalization slice closes and remaining historical work stays `legacy` unless
|
canonicalization slice closes and remaining historical work stays `legacy` unless
|
||||||
an integrity or product need justifies a targeted migration.
|
an integrity or product need justifies a targeted migration.
|
||||||
|
|
||||||
## Observatory preparation and Worker dispatch
|
## Observatory durable recorded queue and Worker dispatch
|
||||||
|
|
||||||
Selecting a source and a laboratory setup in Observatory is not itself a run.
|
Selecting a source and a laboratory setup in Observatory is not itself a run.
|
||||||
Before a remote executor is installed, Mission Core may persist only an immutable
|
The public submission contains only `source_session_id`, `setup_id` and an
|
||||||
`missioncore.observatory-run-preparation/v1` receipt. That receipt binds the exact
|
idempotency key. It cannot supply commands, executable text, filesystem paths,
|
||||||
source-catalog snapshot, setup id, RunDefinition version and configuration digest
|
container images, model identities, resource limits or priority. The server
|
||||||
to one idempotency key. Its only admitted terminal state is `blocked`; it has no
|
resolves the allowlisted pair and seals all executable identity into one durable
|
||||||
`run_id`, dispatch receipt or execution authority.
|
record:
|
||||||
|
|
||||||
A real Worker run begins only after a separate durable dispatcher has atomically
|
- the current source-catalog snapshot captured at admission, plus immutable source
|
||||||
published an immutable dispatch receipt. The executor must be an additive,
|
bundle and source-capability-manifest SHA-256 identities;
|
||||||
allowlisted service owned outside the K1 acquisition path. It may accept bounded
|
- source-adapter id, version and SHA-256;
|
||||||
versioned data, but never an arbitrary command, filesystem path or PowerShell
|
- setup RunDefinition id, version and SHA-256;
|
||||||
fragment from the UI. Exact retries return the original receipt; a reused key with
|
- executor release and image SHA-256 identities;
|
||||||
different identity fails closed. Transport uncertainty produces an explicit
|
- learned-model manifest and resource-profile identities;
|
||||||
reconciliation state and never an automatic duplicate run.
|
- checkpoint policy and the server-owned priority class.
|
||||||
|
|
||||||
The first executable Observatory setup remains the exact recorded
|
An exact retry returns the existing job. Reusing an idempotency key for another
|
||||||
RAVNOVES00/M4.9T5 definition. Its full and integrated successors are different
|
identity fails closed. The ordinary lifecycle advances through `accepted`,
|
||||||
definitions and results. The current RAVNOVES004TREE LAB V1 projection remains
|
`queued`, `claimed` and `running`, then terminates as `succeeded` or `failed`.
|
||||||
replay-only until a new, independently versioned RunDefinition is reconstructed;
|
`paused`, `preemption-pending` and `reconciliation-required` are explicit safety
|
||||||
Mission Core does not invent a configuration for an already published result.
|
branches rather than hidden retries. The durable queue serializes one recorded
|
||||||
|
compute owner and does not equate a queued receipt with a Worker execution
|
||||||
|
receipt.
|
||||||
|
|
||||||
Worker telemetry is secondary observation evidence. It does not replace the
|
The only currently queueable pair is the exact `RAVNOVES00` source and
|
||||||
authoritative run ledger, dispatch receipt, result validation or common laboratory
|
`M4.9T5 · TRAVEL TGS · CPU-only, без ML` RunDefinition. It seals an explicitly
|
||||||
receipt. K1 control, Simulation/Gaussian runtimes and legacy LAB projections are
|
empty learned-model list because TGS is an algorithmic CPU pipeline, not an
|
||||||
outside this executor boundary and are not restarted or migrated by it.
|
unknown model dependency. The binding pins the exact source pack rather than a
|
||||||
|
volatile whole-catalog digest; the server captures and seals the current catalog
|
||||||
|
snapshot into each admitted job. `LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39`
|
||||||
|
remains a result-only replay: no independently versioned durable executor exists
|
||||||
|
for that published result, so Mission Core does not invent one. Every other
|
||||||
|
session/setup pair fails closed until its own source adapter, capability manifest,
|
||||||
|
RunDefinition and executor identities are admitted.
|
||||||
|
|
||||||
|
Recorded work has priority rank `100`. A future live K1 lease has rank `0` and
|
||||||
|
closes new recorded claims while it is pending or active. Cooperative executors
|
||||||
|
yield only at an allowlisted checkpoint. A non-checkpointable monolith must be
|
||||||
|
cancelled by the scheduler, discard all staging output and later restart from
|
||||||
|
zero. Cancellation uses a crash-safe two-phase protocol: Mission Core first
|
||||||
|
persists one stable cancellation intent and activates live work only after an
|
||||||
|
identity-bound receipt proves resource release and staging discard. A retry uses
|
||||||
|
the same cancellation identity. A missing callback or receipt remains durably
|
||||||
|
pending for retry; a conflicting or indeterminate receipt enters reconciliation.
|
||||||
|
Every unresolved form blocks live activation and concurrent ownership.
|
||||||
|
An explicit live terminal trigger requeues paused recorded work.
|
||||||
|
|
||||||
|
ADR 0046 records this decision. The durable queue and identity contracts are
|
||||||
|
implemented; Worker claim transport, installation of the exact M4.9T5 executor
|
||||||
|
and wiring from the real K1 lifecycle to live-lease triggers remain pending.
|
||||||
|
Therefore a submitted M4.9T5 job may honestly wait in `queued` without implying
|
||||||
|
that Worker 006 can execute it yet.
|
||||||
|
|
||||||
|
Worker telemetry remains secondary observation evidence. It does not replace the
|
||||||
|
authoritative queue ledger, result validation or common laboratory receipt. K1
|
||||||
|
acquisition and control, the legacy LAB archive, and Simulation/Gaussian runtimes
|
||||||
|
are outside this boundary and are not restarted, migrated or modified by it.
|
||||||
|
|
||||||
## Planning discipline
|
## Planning discipline
|
||||||
|
|
||||||
|
|||||||
@@ -113,11 +113,11 @@ The next bounded slice adds an explicit laboratory Setup selection beside the so
|
|||||||
Setup is a versioned, immutable compatibility contract, not a mutable bag of UI parameters. The
|
Setup is a versioned, immutable compatibility contract, not a mutable bag of UI parameters. The
|
||||||
catalog keeps two independently named historical facts:
|
catalog keeps two independently named historical facts:
|
||||||
|
|
||||||
- `M4.9T5 · TRAVEL TGS · полный source-paced shadow` is an archived RunDefinition bound exactly to
|
- `M4.9T5 · TRAVEL TGS · CPU-only, без ML` is an archived RunDefinition bound exactly to
|
||||||
`20260720T065719Z_viewer_live`. Its configuration references are content-addressed, its primary
|
`20260720T065719Z_viewer_live`. Its configuration references are content-addressed, its primary
|
||||||
visual result remains in legacy LAB, and the integrated-graph result is retained as a compute
|
visual result remains in legacy LAB, and the integrated-graph result is retained as a compute
|
||||||
successor rather than presented as a second viewer.
|
successor rather than presented as a second viewer.
|
||||||
- `LAB V1 · финальный записанный контур восприятия` is the current exact RAV004 result bound to
|
- `LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39` is the current exact RAV004 result bound to
|
||||||
`20260828T130511Z_viewer_live`. It predates the product RunDefinition contract and therefore keeps
|
`20260828T130511Z_viewer_live`. It predates the product RunDefinition contract and therefore keeps
|
||||||
`run_definition = null`; the UI identifies it as an existing result and never fabricates a config
|
`run_definition = null`; the UI identifies it as an existing result and never fabricates a config
|
||||||
digest for it.
|
digest for it.
|
||||||
|
|||||||
@@ -0,0 +1,138 @@
|
|||||||
|
# ADR 0046: Durable Observatory recorded queue and live K1 priority
|
||||||
|
|
||||||
|
Date: 2026-08-31
|
||||||
|
Status: accepted; durable queue core implemented; Worker and live-trigger wiring pending
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Observatory must let an operator apply an admitted laboratory setup to a saved
|
||||||
|
session without turning the browser into an execution client. The current compute
|
||||||
|
host is a shared, finite Worker rather than an elastic pool. Recorded experiments
|
||||||
|
may wait, but a future live K1 perception process must acquire that compute
|
||||||
|
immediately and exclusively when its real acquisition lifecycle begins.
|
||||||
|
|
||||||
|
Historical LAB results are not automatically executable definitions. In
|
||||||
|
particular, the existing LAB V1 result proves a recorded EoMT and DDRNet analysis,
|
||||||
|
but it does not retain one independently versioned, durable executor contract that
|
||||||
|
can safely be reconstructed from its display metadata. Conversely, M4.9T5 has an
|
||||||
|
exact accepted RAVNOVES00 source pack and CPU-only TRAVEL TGS release which can be
|
||||||
|
sealed without inventing learned-model dependencies.
|
||||||
|
|
||||||
|
The queue must also survive process restarts and ambiguity around preemption. A
|
||||||
|
database flag alone cannot prove that a non-checkpointable Worker process released
|
||||||
|
CPU, memory and staging storage. Starting live K1 work before physical cancellation
|
||||||
|
is confirmed would permit two owners of the same constrained resource.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Mission Core owns a bounded SQLite-backed recorded-job queue. The browser submits
|
||||||
|
only:
|
||||||
|
|
||||||
|
- `source_session_id`;
|
||||||
|
- `setup_id`;
|
||||||
|
- one idempotency key.
|
||||||
|
|
||||||
|
The request contains no command, script, path, environment variable, image,
|
||||||
|
model, resource limit or priority. The server resolves an allowlisted source/setup
|
||||||
|
pair and seals the following identities into the accepted job:
|
||||||
|
|
||||||
|
1. the current source-catalog snapshot SHA-256 captured at admission;
|
||||||
|
2. immutable source-bundle SHA-256;
|
||||||
|
3. source-capability-manifest SHA-256;
|
||||||
|
4. source-adapter id, version and SHA-256;
|
||||||
|
5. setup RunDefinition id, version and SHA-256;
|
||||||
|
6. executor release and container-image SHA-256;
|
||||||
|
7. learned-model release list and model-manifest SHA-256;
|
||||||
|
8. resource-profile SHA-256 and checkpoint policy.
|
||||||
|
|
||||||
|
The queue owns idempotency. An exact retry returns the original job and receipt.
|
||||||
|
The same key with changed identity is a conflict. Jobs advance through `accepted`,
|
||||||
|
`queued`, `claimed` and `running`, then terminate as `succeeded` or `failed`.
|
||||||
|
`paused`, `preemption-pending` and `reconciliation-required` expose safety-relevant
|
||||||
|
branches. Mission Core admits only one active recorded compute owner.
|
||||||
|
|
||||||
|
The queue is an orchestration and evidence boundary, not a Worker transport. A
|
||||||
|
queued job proves durable admission, not remote execution or eventual success.
|
||||||
|
Worker claims, terminal result publication and telemetry must later use a separate
|
||||||
|
authenticated, versioned service boundary; no arbitrary PowerShell or filesystem
|
||||||
|
payload is added to this contract.
|
||||||
|
|
||||||
|
## Current setup admission
|
||||||
|
|
||||||
|
The current executable matrix contains one exact pair:
|
||||||
|
|
||||||
|
- source: `RAVNOVES00`;
|
||||||
|
- setup: `M4.9T5 · TRAVEL TGS · CPU-only, без ML`;
|
||||||
|
- learned-model releases: `[]`;
|
||||||
|
- checkpoint policy: `non-checkpointable`.
|
||||||
|
|
||||||
|
The empty model list is affirmative provenance: this TGS configuration is a
|
||||||
|
CPU-only algorithmic pipeline. It is not a placeholder for unidentified weights.
|
||||||
|
Admission still reads and seals the current source-catalog snapshot and verifies
|
||||||
|
the pinned source bundle and capability manifest before the job is accepted. The
|
||||||
|
binding deliberately does not pin a digest of the whole mutable catalog: the
|
||||||
|
exact compute input is the immutable source pack, while the catalog snapshot is
|
||||||
|
job-specific provenance.
|
||||||
|
|
||||||
|
`LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39` remains result-only. The
|
||||||
|
published evidence stays viewable, but the setup cannot be queued until a new
|
||||||
|
independently versioned source adapter, RunDefinition and executable release are
|
||||||
|
created and accepted. The same fail-closed rule applies to every other
|
||||||
|
session/setup combination. The product may grow toward any admitted setup over
|
||||||
|
any compatible recorded source, but compatibility is evidence, not an assumption.
|
||||||
|
|
||||||
|
## Priority and preemption protocol
|
||||||
|
|
||||||
|
Priority is server-owned:
|
||||||
|
|
||||||
|
- live K1 lease: rank `0`;
|
||||||
|
- recorded replay job: rank `100`.
|
||||||
|
|
||||||
|
A pending or active live lease blocks new recorded claims. A running cooperative
|
||||||
|
job yields at an allowlisted checkpoint and moves to `paused` before live
|
||||||
|
activation. A non-checkpointable job cannot pretend to pause. Its transition is:
|
||||||
|
|
||||||
|
1. persist a stable cancellation intent bound to the job claim, executor release,
|
||||||
|
image, resource profile and live trigger;
|
||||||
|
2. invoke the scheduler-owned physical cancellation boundary;
|
||||||
|
3. persist an identity-bound cancellation receipt proving resource release,
|
||||||
|
staging discard and restart-from-zero;
|
||||||
|
4. only then activate the live K1 lease.
|
||||||
|
|
||||||
|
The cancellation identity is reused after a crash or retry. A missing callback or
|
||||||
|
receipt leaves the durable intent pending for retry; a conflicting or
|
||||||
|
indeterminate receipt moves the record to reconciliation. Both forms block live
|
||||||
|
activation instead of automatically launching duplicate work. When an
|
||||||
|
explicit live terminal trigger completes, fails or cancels the lease, paused
|
||||||
|
recorded jobs return to the queue. A non-checkpointable job starts again from
|
||||||
|
zero; partial staging never becomes evidence.
|
||||||
|
|
||||||
|
## Acceptance boundary
|
||||||
|
|
||||||
|
This decision accepts the durable queue core and its identity, lifecycle,
|
||||||
|
idempotency and preemption contracts. It does not claim that the complete compute
|
||||||
|
loop is deployed. The following remain implementation gates:
|
||||||
|
|
||||||
|
- authenticated Worker claim and result transport;
|
||||||
|
- installation of the exact M4.9T5 executor on the Worker;
|
||||||
|
- binding real K1 acquisition start/terminal events to live-lease triggers;
|
||||||
|
- physical acceptance that cancellation releases the required Worker resources
|
||||||
|
and that recorded work resumes only after the live lease terminates.
|
||||||
|
|
||||||
|
Until those gates close, an M4.9T5 submission may remain honestly `queued`. LAB V1
|
||||||
|
cannot be submitted, and no UI state may present it as executable.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- The UI stays a bounded selector and submitter; executable authority remains on
|
||||||
|
the server.
|
||||||
|
- Every accepted job is reproducible from sealed source, setup, executor, model
|
||||||
|
and resource identities rather than mutable host paths.
|
||||||
|
- Live K1 can receive hard priority without silently losing or concurrently
|
||||||
|
running recorded work.
|
||||||
|
- Non-checkpointable replay pays the deliberate cost of discard and
|
||||||
|
restart-from-zero after preemption.
|
||||||
|
- K1 acquisition/control principles, the legacy LAB archive and
|
||||||
|
Simulation/Gaussian remain unchanged.
|
||||||
|
- Adding a setup or source requires an admitted adapter and RunDefinition; a
|
||||||
|
display name or historical result is insufficient.
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -42,6 +42,15 @@ from k1link.observatory import (
|
|||||||
ObservatoryRunPreparationLedger,
|
ObservatoryRunPreparationLedger,
|
||||||
load_observatory_run_preparation_ledger,
|
load_observatory_run_preparation_ledger,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.m49_queue_binding import (
|
||||||
|
M49QueueBindingConfig,
|
||||||
|
M49QueueBindingError,
|
||||||
|
M49RecordedQueueBindingService,
|
||||||
|
)
|
||||||
|
from k1link.observatory.recorded_jobs import (
|
||||||
|
ObservatoryRecordedJobQueue,
|
||||||
|
ObservatoryRecordedQueueError,
|
||||||
|
)
|
||||||
from k1link.sessions import (
|
from k1link.sessions import (
|
||||||
MaterializedRecording,
|
MaterializedRecording,
|
||||||
RecordedCameraFrameService,
|
RecordedCameraFrameService,
|
||||||
@@ -242,6 +251,34 @@ OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR: str | None
|
|||||||
OBSERVATORY_RUN_PREPARATION_LEDGER,
|
OBSERVATORY_RUN_PREPARATION_LEDGER,
|
||||||
OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR,
|
OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR,
|
||||||
) = load_observatory_run_preparation_ledger(session_store.data_dir)
|
) = load_observatory_run_preparation_ledger(session_store.data_dir)
|
||||||
|
OBSERVATORY_RECORDED_BINDING_SERVICE: M49RecordedQueueBindingService | None
|
||||||
|
OBSERVATORY_RECORDED_JOB_QUEUE: ObservatoryRecordedJobQueue | None
|
||||||
|
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR: str | None
|
||||||
|
try:
|
||||||
|
if OBSERVATORY_LABORATORY_SETUP_REGISTRY is None:
|
||||||
|
raise M49QueueBindingError("observatory setup registry is unavailable")
|
||||||
|
OBSERVATORY_RECORDED_BINDING_SERVICE = M49RecordedQueueBindingService(
|
||||||
|
data_dir=session_store.data_dir,
|
||||||
|
session_store=session_store,
|
||||||
|
setup_registry=OBSERVATORY_LABORATORY_SETUP_REGISTRY,
|
||||||
|
config=M49QueueBindingConfig.from_file(
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "config"
|
||||||
|
/ "observatory-m49-recorded-queue-binding.json"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
OBSERVATORY_RECORDED_JOB_QUEUE = ObservatoryRecordedJobQueue(
|
||||||
|
session_store.data_dir,
|
||||||
|
definitions=OBSERVATORY_RECORDED_BINDING_SERVICE.definitions,
|
||||||
|
)
|
||||||
|
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = None
|
||||||
|
except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError) as exc:
|
||||||
|
# Recorded execution remains an optional observation-only slice. A drifted
|
||||||
|
# seal or queue must fail closed without preventing K1, Simulation or legacy
|
||||||
|
# LAB from starting.
|
||||||
|
OBSERVATORY_RECORDED_BINDING_SERVICE = None
|
||||||
|
OBSERVATORY_RECORDED_JOB_QUEUE = None
|
||||||
|
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = str(exc)
|
||||||
simulation_project_store = SimulationProjectStore(session_store.data_dir)
|
simulation_project_store = SimulationProjectStore(session_store.data_dir)
|
||||||
simulation_project_service = SimulationProjectService(simulation_project_store)
|
simulation_project_service = SimulationProjectService(simulation_project_store)
|
||||||
session_artifact_gateway = configured_artifact_gateway(session_store.data_dir)
|
session_artifact_gateway = configured_artifact_gateway(session_store.data_dir)
|
||||||
@@ -725,6 +762,9 @@ app.include_router(
|
|||||||
setup_registry_error=OBSERVATORY_LABORATORY_SETUP_REGISTRY_ERROR,
|
setup_registry_error=OBSERVATORY_LABORATORY_SETUP_REGISTRY_ERROR,
|
||||||
run_preparation_ledger=OBSERVATORY_RUN_PREPARATION_LEDGER,
|
run_preparation_ledger=OBSERVATORY_RUN_PREPARATION_LEDGER,
|
||||||
run_preparation_ledger_error=OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR,
|
run_preparation_ledger_error=OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR,
|
||||||
|
recorded_binding_service=OBSERVATORY_RECORDED_BINDING_SERVICE,
|
||||||
|
recorded_job_queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||||
|
recorded_job_queue_error=OBSERVATORY_RECORDED_JOB_QUEUE_ERROR,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
app.include_router(
|
app.include_router(
|
||||||
|
|||||||
@@ -17,6 +17,18 @@ from k1link.observatory import (
|
|||||||
is_admitted_observatory_recorded_result,
|
is_admitted_observatory_recorded_result,
|
||||||
observatory_run_preparation_request_sha256,
|
observatory_run_preparation_request_sha256,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.m49_queue_binding import (
|
||||||
|
M49QueueBindingError,
|
||||||
|
M49QueueBindingIntegrityError,
|
||||||
|
M49RecordedQueueBindingService,
|
||||||
|
)
|
||||||
|
from k1link.observatory.recorded_jobs import (
|
||||||
|
ObservatoryRecordedJobQueue,
|
||||||
|
ObservatoryRecordedQueueCapacityError,
|
||||||
|
ObservatoryRecordedQueueConflictError,
|
||||||
|
ObservatoryRecordedQueueError,
|
||||||
|
ObservatoryRecordedQueueNotFoundError,
|
||||||
|
)
|
||||||
from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore
|
from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore
|
||||||
from k1link.sessions.models import SessionSummary
|
from k1link.sessions.models import SessionSummary
|
||||||
|
|
||||||
@@ -32,6 +44,19 @@ OBSERVATORY_RUN_PREFLIGHT_REQUEST_SCHEMA: Literal[
|
|||||||
OBSERVATORY_RUN_PREFLIGHT_SCHEMA: Literal[
|
OBSERVATORY_RUN_PREFLIGHT_SCHEMA: Literal[
|
||||||
"missioncore.observatory-run-preflight/v1"
|
"missioncore.observatory-run-preflight/v1"
|
||||||
] = "missioncore.observatory-run-preflight/v1"
|
] = "missioncore.observatory-run-preflight/v1"
|
||||||
|
OBSERVATORY_RECORDED_RUN_SUBMIT_SCHEMA: Literal[
|
||||||
|
"missioncore.observatory-recorded-run-submit/v1"
|
||||||
|
] = "missioncore.observatory-recorded-run-submit/v1"
|
||||||
|
OBSERVATORY_RECORDED_JOB_LIST_SCHEMA: Literal[
|
||||||
|
"missioncore.observatory-recorded-job-list/v1"
|
||||||
|
] = "missioncore.observatory-recorded-job-list/v1"
|
||||||
|
|
||||||
|
_OBSERVATION_ONLY_AUTHORITY: dict[str, bool] = {
|
||||||
|
"commands_enabled": False,
|
||||||
|
"actuation_allowed": False,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
"production_accepted": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class _StrictApiModel(BaseModel):
|
class _StrictApiModel(BaseModel):
|
||||||
@@ -88,6 +113,25 @@ class ObservatoryRunPreparationRequest(_StrictApiModel):
|
|||||||
definition_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
definition_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||||
|
|
||||||
|
|
||||||
|
class ObservatoryRecordedRunSubmitRequest(_StrictApiModel):
|
||||||
|
schema_version: Literal["missioncore.observatory-recorded-run-submit/v1"]
|
||||||
|
idempotency_key: str = Field(
|
||||||
|
min_length=1,
|
||||||
|
max_length=160,
|
||||||
|
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$",
|
||||||
|
)
|
||||||
|
source_session_id: str = Field(
|
||||||
|
min_length=1,
|
||||||
|
max_length=128,
|
||||||
|
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||||
|
)
|
||||||
|
setup_id: str = Field(
|
||||||
|
min_length=3,
|
||||||
|
max_length=96,
|
||||||
|
pattern=r"^[a-z][a-z0-9-]{2,95}$",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def build_observatory_router(
|
def build_observatory_router(
|
||||||
store: SessionStore,
|
store: SessionStore,
|
||||||
*,
|
*,
|
||||||
@@ -95,6 +139,9 @@ def build_observatory_router(
|
|||||||
setup_registry_error: str | None = None,
|
setup_registry_error: str | None = None,
|
||||||
run_preparation_ledger: ObservatoryRunPreparationLedger | None = None,
|
run_preparation_ledger: ObservatoryRunPreparationLedger | None = None,
|
||||||
run_preparation_ledger_error: str | None = None,
|
run_preparation_ledger_error: str | None = None,
|
||||||
|
recorded_binding_service: M49RecordedQueueBindingService | None = None,
|
||||||
|
recorded_job_queue: ObservatoryRecordedJobQueue | None = None,
|
||||||
|
recorded_job_queue_error: str | None = None,
|
||||||
) -> APIRouter:
|
) -> APIRouter:
|
||||||
"""Build bounded catalog-only mutations for typed Observatory projections."""
|
"""Build bounded catalog-only mutations for typed Observatory projections."""
|
||||||
|
|
||||||
@@ -224,6 +271,34 @@ def build_observatory_router(
|
|||||||
assert isinstance(executor, dict)
|
assert isinstance(executor, dict)
|
||||||
compatible = compatibility.get("compatible") is True
|
compatible = compatibility.get("compatible") is True
|
||||||
existing = preflight.get("outcome") == "existing"
|
existing = preflight.get("outcome") == "existing"
|
||||||
|
queue_binding_ready = False
|
||||||
|
queue_binding_reason: str | None = None
|
||||||
|
binding_service = recorded_binding_service
|
||||||
|
exact_queue_setup = (
|
||||||
|
binding_service is not None
|
||||||
|
and request.setup_id
|
||||||
|
== binding_service.config.setup.setup_id
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
compatible
|
||||||
|
and not existing
|
||||||
|
and binding_service is not None
|
||||||
|
and exact_queue_setup
|
||||||
|
and isinstance(expected_digest, str)
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
binding_service.check(
|
||||||
|
source_session_id=request.source_session_id,
|
||||||
|
setup_id=request.setup_id,
|
||||||
|
definition_sha256=expected_digest,
|
||||||
|
)
|
||||||
|
queue_binding_ready = True
|
||||||
|
except M49QueueBindingError:
|
||||||
|
queue_binding_reason = (
|
||||||
|
"Точная привязка источника и исполняемого релиза не прошла "
|
||||||
|
"проверку целостности."
|
||||||
|
)
|
||||||
|
queueable = queue_binding_ready and recorded_job_queue is not None
|
||||||
checks: list[dict[str, Any]] = [
|
checks: list[dict[str, Any]] = [
|
||||||
{
|
{
|
||||||
"check_id": "source-compatibility",
|
"check_id": "source-compatibility",
|
||||||
@@ -247,26 +322,66 @@ def build_observatory_router(
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"check_id": "executor",
|
"check_id": "executor",
|
||||||
"outcome": "not-applicable" if existing else "fail",
|
"outcome": (
|
||||||
|
"not-applicable"
|
||||||
|
if existing
|
||||||
|
else "pass"
|
||||||
|
if queue_binding_ready
|
||||||
|
else "fail"
|
||||||
|
),
|
||||||
"reason_code": (
|
"reason_code": (
|
||||||
"existing-result-does-not-require-executor"
|
"existing-result-does-not-require-executor"
|
||||||
if existing
|
if existing
|
||||||
|
else "executor-release-sealed"
|
||||||
|
if queue_binding_ready
|
||||||
else str(executor.get("reason_code"))
|
else str(executor.get("reason_code"))
|
||||||
),
|
),
|
||||||
"message": (
|
"message": (
|
||||||
"Готовый результат открывается без повторного запуска Worker."
|
"Готовый результат открывается без повторного запуска Worker."
|
||||||
if existing
|
if existing
|
||||||
|
else (
|
||||||
|
"Исполняемый релиз и его ресурсы запечатаны; отдельный "
|
||||||
|
"Worker service заберёт расчёт из очереди после установки."
|
||||||
|
)
|
||||||
|
if queue_binding_ready
|
||||||
else str(executor.get("reason"))
|
else str(executor.get("reason"))
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"check_id": "durable-queue",
|
||||||
|
"outcome": (
|
||||||
|
"not-applicable"
|
||||||
|
if existing
|
||||||
|
else "pass"
|
||||||
|
if queueable
|
||||||
|
else "fail"
|
||||||
|
),
|
||||||
|
"reason_code": (
|
||||||
|
"existing-result-does-not-require-queue"
|
||||||
|
if existing
|
||||||
|
else "durable-queue-ready"
|
||||||
|
if queueable
|
||||||
|
else "durable-queue-unavailable"
|
||||||
|
),
|
||||||
|
"message": (
|
||||||
|
"Готовый результат не требует постановки в очередь."
|
||||||
|
if existing
|
||||||
|
else "Durable-очередь готова принять расчёт."
|
||||||
|
if queueable
|
||||||
|
else queue_binding_reason
|
||||||
|
or "Этот источник и сетап пока нельзя поставить в очередь."
|
||||||
|
),
|
||||||
|
},
|
||||||
]
|
]
|
||||||
return {
|
return {
|
||||||
"schema_version": OBSERVATORY_RUN_PREFLIGHT_SCHEMA,
|
"schema_version": OBSERVATORY_RUN_PREFLIGHT_SCHEMA,
|
||||||
"source_session_id": request.source_session_id,
|
"source_session_id": request.source_session_id,
|
||||||
"setup_id": request.setup_id,
|
"setup_id": request.setup_id,
|
||||||
"definition_sha256": expected_digest,
|
"definition_sha256": expected_digest,
|
||||||
"outcome": "existing" if existing else "blocked",
|
"outcome": (
|
||||||
"submission_allowed": False,
|
"existing" if existing else "queueable" if queueable else "blocked"
|
||||||
|
),
|
||||||
|
"submission_allowed": queueable,
|
||||||
"checks": checks,
|
"checks": checks,
|
||||||
"existing_result_ids": preflight.get("existing_result_ids", []),
|
"existing_result_ids": preflight.get("existing_result_ids", []),
|
||||||
"executor": executor,
|
"executor": executor,
|
||||||
@@ -508,6 +623,236 @@ def build_observatory_router(
|
|||||||
detail="Подготовка расчётов Обсерватории недоступна.",
|
detail="Подготовка расчётов Обсерватории недоступна.",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if (
|
||||||
|
setup_registry is not None
|
||||||
|
and recorded_binding_service is not None
|
||||||
|
and recorded_job_queue is not None
|
||||||
|
):
|
||||||
|
|
||||||
|
@router.post("/api/v1/observatory/runs", status_code=202)
|
||||||
|
def submit_observatory_recorded_run(
|
||||||
|
request: ObservatoryRecordedRunSubmitRequest,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
try:
|
||||||
|
existing_job = recorded_job_queue.get_by_idempotency_key(
|
||||||
|
request.idempotency_key
|
||||||
|
)
|
||||||
|
except ObservatoryRecordedQueueNotFoundError:
|
||||||
|
existing_job = None
|
||||||
|
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Durable-очередь расчётов недоступна.",
|
||||||
|
) from exc
|
||||||
|
if existing_job is not None:
|
||||||
|
if (
|
||||||
|
existing_job.source_session_id != request.source_session_id
|
||||||
|
or existing_job.setup_id != request.setup_id
|
||||||
|
):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Ключ идемпотентности уже связан с другим расчётом.",
|
||||||
|
)
|
||||||
|
return existing_job.as_dict()
|
||||||
|
|
||||||
|
source = source_summary(request.source_session_id)
|
||||||
|
try:
|
||||||
|
setup_registry.setup(request.setup_id)
|
||||||
|
except KeyError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=404,
|
||||||
|
detail="Сетап лаборатории не найден.",
|
||||||
|
) from exc
|
||||||
|
catalog = setup_registry.catalog(
|
||||||
|
source,
|
||||||
|
available_observatory_result_ids=available_observatory_results(
|
||||||
|
request.source_session_id
|
||||||
|
),
|
||||||
|
)
|
||||||
|
setups = catalog.get("setups")
|
||||||
|
if not isinstance(setups, list):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Каталог сетапов Обсерватории нарушил контракт.",
|
||||||
|
)
|
||||||
|
projected = next(
|
||||||
|
(
|
||||||
|
item
|
||||||
|
for item in setups
|
||||||
|
if isinstance(item, dict) and item.get("setup_id") == request.setup_id
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if projected is None:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Каталог сетапов Обсерватории нарушил контракт.",
|
||||||
|
)
|
||||||
|
compatibility = projected.get("compatibility")
|
||||||
|
preflight = projected.get("preflight")
|
||||||
|
definition = projected.get("run_definition")
|
||||||
|
if not isinstance(compatibility, dict) or not isinstance(preflight, dict):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Каталог сетапов Обсерватории нарушил контракт.",
|
||||||
|
)
|
||||||
|
if compatibility.get("compatible") is not True:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Исходная сессия несовместима с выбранным сетапом.",
|
||||||
|
)
|
||||||
|
if preflight.get("outcome") == "existing":
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Точный результат уже существует; новый расчёт не создан.",
|
||||||
|
)
|
||||||
|
if not isinstance(definition, dict):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Для сетапа нет воспроизводимой RunDefinition.",
|
||||||
|
)
|
||||||
|
definition_sha256 = definition.get("definition_sha256")
|
||||||
|
if not isinstance(definition_sha256, str):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Каталог сетапов Обсерватории нарушил контракт.",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
admission = recorded_binding_service.admit(
|
||||||
|
source_session_id=request.source_session_id,
|
||||||
|
setup_id=request.setup_id,
|
||||||
|
definition_sha256=definition_sha256,
|
||||||
|
)
|
||||||
|
job, _created = recorded_job_queue.submit(
|
||||||
|
admission.intent(idempotency_key=request.idempotency_key),
|
||||||
|
enqueue=True,
|
||||||
|
)
|
||||||
|
return job.as_dict()
|
||||||
|
except M49QueueBindingIntegrityError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail=(
|
||||||
|
"Точная привязка источника и исполняемого релиза не прошла "
|
||||||
|
"проверку целостности."
|
||||||
|
),
|
||||||
|
) from exc
|
||||||
|
except ObservatoryRecordedQueueConflictError as exc:
|
||||||
|
try:
|
||||||
|
existing_job = recorded_job_queue.get_by_idempotency_key(
|
||||||
|
request.idempotency_key
|
||||||
|
)
|
||||||
|
except ObservatoryRecordedQueueNotFoundError:
|
||||||
|
existing_job = None
|
||||||
|
except (ObservatoryRecordedQueueError, ValueError) as lookup_exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Durable-очередь расчётов недоступна.",
|
||||||
|
) from lookup_exc
|
||||||
|
if (
|
||||||
|
existing_job is not None
|
||||||
|
and existing_job.source_session_id == request.source_session_id
|
||||||
|
and existing_job.setup_id == request.setup_id
|
||||||
|
):
|
||||||
|
return existing_job.as_dict()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Ключ идемпотентности уже связан с другим расчётом.",
|
||||||
|
) from exc
|
||||||
|
except ObservatoryRecordedQueueCapacityError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Квота durable-очереди расчётов исчерпана.",
|
||||||
|
) from exc
|
||||||
|
except (M49QueueBindingError, ObservatoryRecordedQueueError, ValueError) as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Durable-очередь расчётов недоступна.",
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
@router.get("/api/v1/observatory/runs")
|
||||||
|
def list_observatory_recorded_runs(
|
||||||
|
source_session_id: str = Query(
|
||||||
|
min_length=1,
|
||||||
|
max_length=128,
|
||||||
|
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||||
|
),
|
||||||
|
setup_id: str = Query(
|
||||||
|
min_length=3,
|
||||||
|
max_length=96,
|
||||||
|
pattern=r"^[a-z][a-z0-9-]{2,95}$",
|
||||||
|
),
|
||||||
|
limit: int = Query(default=20, ge=1, le=100),
|
||||||
|
) -> dict[str, object]:
|
||||||
|
try:
|
||||||
|
jobs = recorded_job_queue.list_jobs(
|
||||||
|
source_session_id=source_session_id,
|
||||||
|
setup_id=setup_id,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Durable-очередь расчётов недоступна.",
|
||||||
|
) from exc
|
||||||
|
return {
|
||||||
|
"schema_version": OBSERVATORY_RECORDED_JOB_LIST_SCHEMA,
|
||||||
|
"items": [job.as_dict() for job in jobs],
|
||||||
|
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
||||||
|
}
|
||||||
|
|
||||||
|
@router.get("/api/v1/observatory/runs/{job_id}")
|
||||||
|
def get_observatory_recorded_run(
|
||||||
|
job_id: str = ApiPath(
|
||||||
|
min_length=48,
|
||||||
|
max_length=48,
|
||||||
|
pattern=r"^observatory-run-[a-f0-9]{32}$",
|
||||||
|
),
|
||||||
|
) -> dict[str, object]:
|
||||||
|
try:
|
||||||
|
return recorded_job_queue.get(job_id).as_dict()
|
||||||
|
except ObservatoryRecordedQueueNotFoundError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=404,
|
||||||
|
detail="Расчёт Обсерватории не найден.",
|
||||||
|
) from exc
|
||||||
|
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Durable-очередь расчётов недоступна.",
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
elif recorded_job_queue_error is not None:
|
||||||
|
|
||||||
|
@router.post("/api/v1/observatory/runs")
|
||||||
|
def unavailable_observatory_recorded_run(
|
||||||
|
request: ObservatoryRecordedRunSubmitRequest,
|
||||||
|
) -> None:
|
||||||
|
del request
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Durable-очередь расчётов недоступна.",
|
||||||
|
)
|
||||||
|
|
||||||
|
@router.get("/api/v1/observatory/runs")
|
||||||
|
def unavailable_observatory_recorded_runs(
|
||||||
|
source_session_id: str = Query(
|
||||||
|
min_length=1,
|
||||||
|
max_length=128,
|
||||||
|
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||||
|
),
|
||||||
|
setup_id: str = Query(
|
||||||
|
min_length=3,
|
||||||
|
max_length=96,
|
||||||
|
pattern=r"^[a-z][a-z0-9-]{2,95}$",
|
||||||
|
),
|
||||||
|
limit: int = Query(default=20, ge=1, le=100),
|
||||||
|
) -> None:
|
||||||
|
del source_session_id, setup_id, limit
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Durable-очередь расчётов недоступна.",
|
||||||
|
)
|
||||||
|
|
||||||
@router.patch(
|
@router.patch(
|
||||||
"/api/v1/observatory/lab-projections/{session_id}",
|
"/api/v1/observatory/lab-projections/{session_id}",
|
||||||
response_model=ObservatoryProjectionDocument,
|
response_model=ObservatoryProjectionDocument,
|
||||||
|
|||||||
@@ -0,0 +1,555 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import cast
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import k1link.observatory.m49_queue_binding as binding_module
|
||||||
|
from k1link.artifact_gateway import (
|
||||||
|
ArtifactMember,
|
||||||
|
CentralArtifactStore,
|
||||||
|
LocalArtifactCache,
|
||||||
|
)
|
||||||
|
from k1link.observatory.m49_queue_binding import (
|
||||||
|
OBSERVATORY_EXECUTOR_BINDING_SCHEMA,
|
||||||
|
OBSERVATORY_SOURCE_BUNDLE_SCHEMA,
|
||||||
|
OBSERVATORY_SOURCE_CAPABILITY_SCHEMA,
|
||||||
|
SOURCE_DOCUMENT_STORE_DIRECTORY,
|
||||||
|
M49QueueBindingConfig,
|
||||||
|
M49QueueBindingIntegrityError,
|
||||||
|
M49RecordedQueueBindingService,
|
||||||
|
M49SourcePackIdentity,
|
||||||
|
M49SourcePackVerifier,
|
||||||
|
_verify_source_pack_from_cache,
|
||||||
|
)
|
||||||
|
from k1link.observatory.setups import LaboratorySetupRegistry
|
||||||
|
from k1link.sessions.models import (
|
||||||
|
SessionArtifact,
|
||||||
|
SessionDetail,
|
||||||
|
SessionSource,
|
||||||
|
SessionSummary,
|
||||||
|
)
|
||||||
|
from k1link.sessions.store import SessionStore
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
BINDING_PATH = (
|
||||||
|
REPOSITORY_ROOT / "config" / "observatory-m49-recorded-queue-binding.json"
|
||||||
|
)
|
||||||
|
SETUP_REGISTRY_PATH = (
|
||||||
|
REPOSITORY_ROOT / "config" / "observatory-laboratory-setups.json"
|
||||||
|
)
|
||||||
|
SOURCE_SESSION_ID = "20260720T065719Z_viewer_live"
|
||||||
|
SOURCE_LABEL = "RAVNOVES00"
|
||||||
|
SETUP_ID = "m49-tgs-full-shadow-v1"
|
||||||
|
DEFINITION_SHA256 = (
|
||||||
|
"836a66639e69f7ea00de2a9111c1c6c9c3c00f1abd726f1df596aeb4e3a88ae6"
|
||||||
|
)
|
||||||
|
SOURCE_PACK_SHA256 = (
|
||||||
|
"0685d24219d8236caf8b7f1685e93f6d6b59e7fd015a768d88a92bbe8b154944"
|
||||||
|
)
|
||||||
|
EXECUTOR_ARTIFACT_SHA256 = (
|
||||||
|
"5e0ea16c7a5cc760463836718b0cd8b0006ffc4b202e5f706a20a86ef2f912ab"
|
||||||
|
)
|
||||||
|
CODE_REVISION = "40c850b167dda366d8aa45d828520168affaf9fd"
|
||||||
|
TRAVEL_IMAGE_SHA256 = (
|
||||||
|
"7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f"
|
||||||
|
)
|
||||||
|
PARITY_IMAGE_SHA256 = (
|
||||||
|
"ceb13548617e4bd3f619766bfdff00af3fa5160946b367828da6d2233dcdcba0"
|
||||||
|
)
|
||||||
|
SOURCE_CATALOG_SHA256 = (
|
||||||
|
"24207af81b67de515ba9f1b899577bd877a00d058c3472269508980541b1e185"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _SessionStore:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
data_dir: Path,
|
||||||
|
detail: SessionDetail,
|
||||||
|
catalog_sha256: str,
|
||||||
|
) -> None:
|
||||||
|
self.data_dir = data_dir.resolve()
|
||||||
|
self.detail = detail
|
||||||
|
self.catalog_sha256 = catalog_sha256
|
||||||
|
|
||||||
|
def get_session_with_catalog_snapshot(
|
||||||
|
self, session_id: str
|
||||||
|
) -> tuple[SessionDetail, str]:
|
||||||
|
if session_id != self.detail.summary.session_id:
|
||||||
|
raise KeyError(session_id)
|
||||||
|
return self.detail, self.catalog_sha256
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_sha256(value: object) -> str:
|
||||||
|
payload = json.dumps(
|
||||||
|
value,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
allow_nan=False,
|
||||||
|
).encode()
|
||||||
|
return hashlib.sha256(payload).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def test_first_immutable_contract_store_syncs_directory_chain_bottom_up(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
payload = b'{"schema_version":"test/v1"}'
|
||||||
|
digest = hashlib.sha256(payload).hexdigest()
|
||||||
|
root = tmp_path / "observatory-source-contracts"
|
||||||
|
synced: list[Path] = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
binding_module,
|
||||||
|
"_fsync_directory",
|
||||||
|
lambda path: synced.append(path),
|
||||||
|
)
|
||||||
|
|
||||||
|
binding_module._write_immutable_document(root, digest, payload)
|
||||||
|
|
||||||
|
assert synced == [
|
||||||
|
root / "objects" / "sha256" / digest[:2],
|
||||||
|
root / "objects" / "sha256",
|
||||||
|
root / "objects",
|
||||||
|
root,
|
||||||
|
tmp_path,
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_existing_immutable_contract_replays_directory_chain_sync(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
payload = b'{"schema_version":"test/v1"}'
|
||||||
|
digest = hashlib.sha256(payload).hexdigest()
|
||||||
|
root = tmp_path / "observatory-source-contracts"
|
||||||
|
binding_module._write_immutable_document(root, digest, payload)
|
||||||
|
synced: list[Path] = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
binding_module,
|
||||||
|
"_fsync_directory",
|
||||||
|
lambda path: synced.append(path),
|
||||||
|
)
|
||||||
|
|
||||||
|
binding_module._write_immutable_document(root, digest, payload)
|
||||||
|
|
||||||
|
assert synced == [
|
||||||
|
root / "objects" / "sha256" / digest[:2],
|
||||||
|
root / "objects" / "sha256",
|
||||||
|
root / "objects",
|
||||||
|
root,
|
||||||
|
tmp_path,
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_immutable_contract_store_rejects_intermediate_symlink(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
payload = b'{"schema_version":"test/v1"}'
|
||||||
|
digest = hashlib.sha256(payload).hexdigest()
|
||||||
|
root = tmp_path / "observatory-source-contracts"
|
||||||
|
outside = tmp_path / "outside"
|
||||||
|
root.mkdir()
|
||||||
|
outside.mkdir()
|
||||||
|
(root / "objects").symlink_to(outside, target_is_directory=True)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
M49QueueBindingIntegrityError,
|
||||||
|
match="store directory is invalid",
|
||||||
|
):
|
||||||
|
binding_module._write_immutable_document(root, digest, payload)
|
||||||
|
|
||||||
|
assert list(outside.iterdir()) == []
|
||||||
|
|
||||||
|
|
||||||
|
def _detail() -> SessionDetail:
|
||||||
|
summary = SessionSummary(
|
||||||
|
session_id=SOURCE_SESSION_ID,
|
||||||
|
display_name=SOURCE_LABEL,
|
||||||
|
status="ready",
|
||||||
|
started_at_utc="2026-07-20T06:57:20.888Z",
|
||||||
|
completed_at_utc="2026-07-20T07:06:16.599Z",
|
||||||
|
duration_seconds=535.717620042,
|
||||||
|
modalities=("point-cloud", "trajectory", "video"),
|
||||||
|
source_count=3,
|
||||||
|
total_bytes=100,
|
||||||
|
replayable=True,
|
||||||
|
origin="xgrids-k1.viewer-live.evidence",
|
||||||
|
)
|
||||||
|
sources = (
|
||||||
|
SessionSource(
|
||||||
|
source_id="sensor.camera.right",
|
||||||
|
semantic_channel_id="camera.video.recorded",
|
||||||
|
modality="video",
|
||||||
|
status="recorded",
|
||||||
|
seekable=True,
|
||||||
|
artifact_id="recorded-video",
|
||||||
|
),
|
||||||
|
SessionSource(
|
||||||
|
source_id="sensor.lidar.primary",
|
||||||
|
semantic_channel_id="spatial.point-cloud.recorded",
|
||||||
|
modality="point-cloud",
|
||||||
|
status="recorded",
|
||||||
|
seekable=True,
|
||||||
|
artifact_id="raw-primary",
|
||||||
|
),
|
||||||
|
SessionSource(
|
||||||
|
source_id="spatial.trajectory",
|
||||||
|
semantic_channel_id="spatial.pose.recorded",
|
||||||
|
modality="trajectory",
|
||||||
|
status="recorded",
|
||||||
|
seekable=True,
|
||||||
|
artifact_id="raw-primary",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
artifacts = (
|
||||||
|
SessionArtifact(
|
||||||
|
artifact_id="raw-primary",
|
||||||
|
kind="raw-transport",
|
||||||
|
media_type="application/x-nodedc-k1mqtt",
|
||||||
|
byte_length=64,
|
||||||
|
sha256="d" * 64,
|
||||||
|
integrity_status="verified",
|
||||||
|
),
|
||||||
|
SessionArtifact(
|
||||||
|
artifact_id="recorded-video",
|
||||||
|
kind="recorded-video",
|
||||||
|
media_type="video/mp4",
|
||||||
|
byte_length=36,
|
||||||
|
sha256=None,
|
||||||
|
integrity_status="validated-structure",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return SessionDetail(
|
||||||
|
summary=summary,
|
||||||
|
sources=sources,
|
||||||
|
artifacts=artifacts,
|
||||||
|
plugin_id="xgrids-k1",
|
||||||
|
archive_id="viewer-live",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _registry() -> LaboratorySetupRegistry:
|
||||||
|
return LaboratorySetupRegistry.from_file(
|
||||||
|
SETUP_REGISTRY_PATH,
|
||||||
|
repository_root=REPOSITORY_ROOT,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _artifact_cache(
|
||||||
|
data_dir: Path,
|
||||||
|
tmp_path: Path,
|
||||||
|
payload: bytes,
|
||||||
|
identity: M49SourcePackIdentity,
|
||||||
|
) -> LocalArtifactCache:
|
||||||
|
source = tmp_path / "source-pack.npz"
|
||||||
|
source.write_bytes(payload)
|
||||||
|
central = CentralArtifactStore(tmp_path / "central", create=True)
|
||||||
|
published = central.publish_file(source)
|
||||||
|
assert published.sha256 == identity.sha256
|
||||||
|
cache = LocalArtifactCache(
|
||||||
|
data_dir / "artifact-cache",
|
||||||
|
max_bytes=1024 * 1024,
|
||||||
|
free_space_reserve_bytes=0,
|
||||||
|
)
|
||||||
|
member = ArtifactMember(
|
||||||
|
role="lidar-source-pack",
|
||||||
|
media_type=identity.media_type,
|
||||||
|
sha256=published.sha256,
|
||||||
|
byte_length=published.byte_length,
|
||||||
|
)
|
||||||
|
cache.fetch(central, member, pin_id=f"session:{SOURCE_SESSION_ID}")
|
||||||
|
return cache
|
||||||
|
|
||||||
|
|
||||||
|
def _accept_source_pack(
|
||||||
|
_cache: LocalArtifactCache,
|
||||||
|
identity: M49SourcePackIdentity,
|
||||||
|
) -> None:
|
||||||
|
assert identity.sha256 == SOURCE_PACK_SHA256
|
||||||
|
assert identity.byte_length == 72_996_000
|
||||||
|
|
||||||
|
|
||||||
|
def _service(
|
||||||
|
tmp_path: Path,
|
||||||
|
*,
|
||||||
|
catalog_sha256: str = SOURCE_CATALOG_SHA256,
|
||||||
|
setup_registry: LaboratorySetupRegistry | None = None,
|
||||||
|
source_pack_verifier: M49SourcePackVerifier = _accept_source_pack,
|
||||||
|
) -> tuple[M49RecordedQueueBindingService, LocalArtifactCache]:
|
||||||
|
data_dir = tmp_path / "data"
|
||||||
|
data_dir.mkdir()
|
||||||
|
config = M49QueueBindingConfig.from_file(BINDING_PATH)
|
||||||
|
cache = LocalArtifactCache(
|
||||||
|
data_dir / "artifact-cache",
|
||||||
|
max_bytes=1024 * 1024,
|
||||||
|
free_space_reserve_bytes=0,
|
||||||
|
)
|
||||||
|
store = _SessionStore(data_dir, _detail(), catalog_sha256)
|
||||||
|
service = M49RecordedQueueBindingService(
|
||||||
|
data_dir=data_dir,
|
||||||
|
session_store=cast(SessionStore, store),
|
||||||
|
setup_registry=setup_registry or _registry(),
|
||||||
|
config=config,
|
||||||
|
artifact_cache=cache,
|
||||||
|
source_pack_verifier=source_pack_verifier,
|
||||||
|
)
|
||||||
|
return service, cache
|
||||||
|
|
||||||
|
|
||||||
|
def test_bundled_binding_freezes_the_accepted_exact_m49_identity() -> None:
|
||||||
|
config = M49QueueBindingConfig.from_file(BINDING_PATH)
|
||||||
|
definition = config.recorded_definition()
|
||||||
|
image_set = config.image_set_document()
|
||||||
|
resource = config.resource_profile.document()
|
||||||
|
|
||||||
|
assert config.source.session_id == SOURCE_SESSION_ID
|
||||||
|
assert config.source.label == SOURCE_LABEL
|
||||||
|
assert config.source.source_pack.sha256 == SOURCE_PACK_SHA256
|
||||||
|
assert config.source.source_pack.byte_length == 72_996_000
|
||||||
|
assert config.setup.setup_id == SETUP_ID
|
||||||
|
assert config.setup.definition_sha256 == DEFINITION_SHA256
|
||||||
|
assert config.executor.artifact_sha256 == EXECUTOR_ARTIFACT_SHA256
|
||||||
|
assert config.executor.code_revision == CODE_REVISION
|
||||||
|
assert config.executor.service_installed is False
|
||||||
|
assert image_set["images"] == [
|
||||||
|
{"role": "parity", "sha256": PARITY_IMAGE_SHA256},
|
||||||
|
{"role": "travel", "sha256": TRAVEL_IMAGE_SHA256},
|
||||||
|
]
|
||||||
|
assert definition.executor_image_sha256 == _canonical_sha256(image_set)
|
||||||
|
assert definition.model_release_ids == ()
|
||||||
|
assert definition.learned_models == ()
|
||||||
|
assert definition.checkpoint_policy == "non-checkpointable"
|
||||||
|
assert definition.allowed_checkpoints == ()
|
||||||
|
assert resource["restart_from_zero"] is True
|
||||||
|
assert resource["staging_discard_required"] is True
|
||||||
|
assert resource["resource_release_receipt_required"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_is_read_only_and_admit_seals_path_free_source_contracts(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
service, cache = _service(tmp_path)
|
||||||
|
document_root = service.data_dir / SOURCE_DOCUMENT_STORE_DIRECTORY
|
||||||
|
cache_database_mtime_ns = cache.database_path.stat().st_mtime_ns
|
||||||
|
|
||||||
|
checked = service.check(
|
||||||
|
source_session_id=SOURCE_SESSION_ID,
|
||||||
|
setup_id=SETUP_ID,
|
||||||
|
definition_sha256=DEFINITION_SHA256,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not document_root.exists()
|
||||||
|
assert cache.database_path.stat().st_mtime_ns == cache_database_mtime_ns
|
||||||
|
assert checked.source_bundle.schema_version == OBSERVATORY_SOURCE_BUNDLE_SCHEMA
|
||||||
|
assert (
|
||||||
|
checked.capability_manifest.schema_version
|
||||||
|
== OBSERVATORY_SOURCE_CAPABILITY_SCHEMA
|
||||||
|
)
|
||||||
|
assert checked.executor_binding.schema_version == OBSERVATORY_EXECUTOR_BINDING_SCHEMA
|
||||||
|
assert checked.source_catalog_sha256 == SOURCE_CATALOG_SHA256
|
||||||
|
assert checked.definition.definition_sha256 == DEFINITION_SHA256
|
||||||
|
assert checked.definition.model_release_ids == ()
|
||||||
|
assert checked.definition.checkpoint_policy == "non-checkpointable"
|
||||||
|
assert checked.source_bundle.document()["members"] == [
|
||||||
|
{
|
||||||
|
"role": "lidar-source-pack",
|
||||||
|
"artifact_id": service.config.source.source_pack.artifact_id,
|
||||||
|
"media_type": service.config.source.source_pack.media_type,
|
||||||
|
"sha256": service.config.source.source_pack.sha256,
|
||||||
|
"byte_length": service.config.source.source_pack.byte_length,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
assert "path" not in json.dumps(checked.as_dict()).lower()
|
||||||
|
|
||||||
|
admitted = service.admit(
|
||||||
|
source_session_id=SOURCE_SESSION_ID,
|
||||||
|
setup_id=SETUP_ID,
|
||||||
|
definition_sha256=DEFINITION_SHA256,
|
||||||
|
)
|
||||||
|
repeated = service.admit(
|
||||||
|
source_session_id=SOURCE_SESSION_ID,
|
||||||
|
setup_id=SETUP_ID,
|
||||||
|
definition_sha256=DEFINITION_SHA256,
|
||||||
|
)
|
||||||
|
sealed_files = tuple(document_root.rglob("*.json"))
|
||||||
|
|
||||||
|
assert admitted == checked
|
||||||
|
assert repeated == admitted
|
||||||
|
assert len(sealed_files) == 3
|
||||||
|
assert {path.stem for path in sealed_files} == {
|
||||||
|
admitted.source_bundle.sha256,
|
||||||
|
admitted.capability_manifest.sha256,
|
||||||
|
admitted.executor_binding.sha256,
|
||||||
|
}
|
||||||
|
assert all(path.read_bytes() in {
|
||||||
|
admitted.source_bundle.payload,
|
||||||
|
admitted.capability_manifest.payload,
|
||||||
|
admitted.executor_binding.payload,
|
||||||
|
} for path in sealed_files)
|
||||||
|
assert service.definitions.resolve(SETUP_ID, DEFINITION_SHA256) == (
|
||||||
|
admitted.definition
|
||||||
|
)
|
||||||
|
intent = admitted.intent(idempotency_key="m49-recorded-submit-001")
|
||||||
|
assert intent.source_session_id == SOURCE_SESSION_ID
|
||||||
|
assert intent.source_catalog_sha256 == SOURCE_CATALOG_SHA256
|
||||||
|
assert intent.source_bundle_sha256 == admitted.source_bundle.sha256
|
||||||
|
assert intent.source_capability_manifest_sha256 == (
|
||||||
|
admitted.capability_manifest.sha256
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("source_session_id", "setup_id", "definition_sha256", "message"),
|
||||||
|
[
|
||||||
|
("other-session", SETUP_ID, DEFINITION_SHA256, "exact source session"),
|
||||||
|
(SOURCE_SESSION_ID, "other-setup", DEFINITION_SHA256, "exact setup"),
|
||||||
|
(SOURCE_SESSION_ID, SETUP_ID, "f" * 64, "definition digest"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_binding_rejects_every_other_session_setup_or_definition(
|
||||||
|
tmp_path: Path,
|
||||||
|
source_session_id: str,
|
||||||
|
setup_id: str,
|
||||||
|
definition_sha256: str,
|
||||||
|
message: str,
|
||||||
|
) -> None:
|
||||||
|
service, _cache = _service(tmp_path)
|
||||||
|
|
||||||
|
with pytest.raises(M49QueueBindingIntegrityError, match=message):
|
||||||
|
service.check(
|
||||||
|
source_session_id=source_session_id,
|
||||||
|
setup_id=setup_id,
|
||||||
|
definition_sha256=definition_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_binding_captures_the_current_session_catalog_snapshot(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
service, _cache = _service(tmp_path, catalog_sha256="e" * 64)
|
||||||
|
|
||||||
|
admission = service.check(
|
||||||
|
source_session_id=SOURCE_SESSION_ID,
|
||||||
|
setup_id=SETUP_ID,
|
||||||
|
definition_sha256=DEFINITION_SHA256,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert admission.source_catalog_sha256 == "e" * 64
|
||||||
|
assert admission.source_bundle.document()["catalog_sha256"] == "e" * 64
|
||||||
|
|
||||||
|
|
||||||
|
def test_binding_fails_closed_when_the_cached_source_pack_changes(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
payload = b"sealed source pack fixture\n"
|
||||||
|
identity = M49SourcePackIdentity(
|
||||||
|
artifact_id="fixture-source-pack",
|
||||||
|
sha256=hashlib.sha256(payload).hexdigest(),
|
||||||
|
byte_length=len(payload),
|
||||||
|
media_type="application/vnd.nodedc.lidar-source-pack+npz",
|
||||||
|
expected_timeline_frames=2,
|
||||||
|
expected_available_lidar_frames=1,
|
||||||
|
)
|
||||||
|
data_dir = tmp_path / "data"
|
||||||
|
data_dir.mkdir()
|
||||||
|
cache = _artifact_cache(data_dir, tmp_path, payload, identity)
|
||||||
|
_verify_source_pack_from_cache(cache, identity)
|
||||||
|
cache.object_path(identity.sha256).write_bytes(b"x" * len(payload))
|
||||||
|
|
||||||
|
with pytest.raises(M49QueueBindingIntegrityError, match="artifact-cache metadata"):
|
||||||
|
_verify_source_pack_from_cache(cache, identity)
|
||||||
|
|
||||||
|
|
||||||
|
def test_binding_fails_closed_when_the_setup_registry_definition_changes(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
document = json.loads(SETUP_REGISTRY_PATH.read_text(encoding="utf-8"))
|
||||||
|
document["setups"][0]["run_definition"]["definition_id"] = (
|
||||||
|
"m49-tgs-full-shadow-drift"
|
||||||
|
)
|
||||||
|
path = tmp_path / "setup-registry.json"
|
||||||
|
path.write_text(json.dumps(document), encoding="utf-8")
|
||||||
|
drifted_registry = LaboratorySetupRegistry.from_file(
|
||||||
|
path,
|
||||||
|
repository_root=REPOSITORY_ROOT,
|
||||||
|
)
|
||||||
|
service, _cache = _service(tmp_path, setup_registry=drifted_registry)
|
||||||
|
|
||||||
|
with pytest.raises(M49QueueBindingIntegrityError, match="setup definition changed"):
|
||||||
|
service.check(
|
||||||
|
source_session_id=SOURCE_SESSION_ID,
|
||||||
|
setup_id=SETUP_ID,
|
||||||
|
definition_sha256=DEFINITION_SHA256,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"mutation",
|
||||||
|
["unknown-key", "installed", "learned-model", "source-session"],
|
||||||
|
)
|
||||||
|
def test_binding_loader_rejects_authority_or_contract_expansion(
|
||||||
|
tmp_path: Path,
|
||||||
|
mutation: str,
|
||||||
|
) -> None:
|
||||||
|
document = json.loads(BINDING_PATH.read_text(encoding="utf-8"))
|
||||||
|
if mutation == "unknown-key":
|
||||||
|
document["command"] = "run arbitrary payload"
|
||||||
|
elif mutation == "installed":
|
||||||
|
document["executor"]["service_installed"] = True
|
||||||
|
elif mutation == "learned-model":
|
||||||
|
document["executor"]["learned_models"] = ["invented-model"]
|
||||||
|
else:
|
||||||
|
document["source"]["session_id"] = "different-source"
|
||||||
|
path = tmp_path / "invalid-binding.json"
|
||||||
|
path.write_text(json.dumps(document), encoding="utf-8")
|
||||||
|
|
||||||
|
with pytest.raises(M49QueueBindingIntegrityError):
|
||||||
|
M49QueueBindingConfig.from_file(path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_binding_requires_the_session_store_data_dir_artifact_cache(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
service, cache = _service(tmp_path)
|
||||||
|
config = service.config
|
||||||
|
foreign = LocalArtifactCache(
|
||||||
|
tmp_path / "foreign-cache",
|
||||||
|
max_bytes=1024 * 1024,
|
||||||
|
free_space_reserve_bytes=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(M49QueueBindingIntegrityError, match="data-dir artifact cache"):
|
||||||
|
M49RecordedQueueBindingService(
|
||||||
|
data_dir=service.data_dir,
|
||||||
|
session_store=service._session_store, # noqa: SLF001 - exact boundary test
|
||||||
|
setup_registry=_registry(),
|
||||||
|
config=config,
|
||||||
|
artifact_cache=foreign,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert cache.root == service.data_dir / "artifact-cache"
|
||||||
|
|
||||||
|
|
||||||
|
def test_binding_wraps_unavailable_default_artifact_cache(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
data_dir = tmp_path / "data"
|
||||||
|
data_dir.mkdir()
|
||||||
|
(data_dir / "artifact-cache").write_text("not a directory", encoding="utf-8")
|
||||||
|
store = _SessionStore(data_dir, _detail(), SOURCE_CATALOG_SHA256)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
M49QueueBindingIntegrityError,
|
||||||
|
match="artifact cache is unavailable",
|
||||||
|
):
|
||||||
|
M49RecordedQueueBindingService(
|
||||||
|
data_dir=data_dir,
|
||||||
|
session_store=cast(SessionStore, store),
|
||||||
|
setup_registry=_registry(),
|
||||||
|
config=M49QueueBindingConfig.from_file(BINDING_PATH),
|
||||||
|
)
|
||||||
@@ -0,0 +1,728 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlite3
|
||||||
|
from dataclasses import replace
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from k1link.observatory.recorded_jobs import (
|
||||||
|
LIVE_K1_PRIORITY_RANK,
|
||||||
|
RECORDED_JOB_DATABASE_NAME,
|
||||||
|
RECORDED_PRIORITY_RANK,
|
||||||
|
ObservatoryLiveLeaseIntent,
|
||||||
|
ObservatoryNonCheckpointableCancellationReceipt,
|
||||||
|
ObservatoryRecordedJobIntent,
|
||||||
|
ObservatoryRecordedJobQueue,
|
||||||
|
ObservatoryRecordedPreemptionError,
|
||||||
|
ObservatoryRecordedQueueBusyError,
|
||||||
|
ObservatoryRecordedQueueConflictError,
|
||||||
|
ObservatoryRecordedQueueIntegrityError,
|
||||||
|
ObservatoryRecordedQueueStaleClaimError,
|
||||||
|
RecordedRunDefinition,
|
||||||
|
RecordedRunDefinitionRegistry,
|
||||||
|
)
|
||||||
|
|
||||||
|
NOW = "2026-08-30T21:00:00.000Z"
|
||||||
|
DEFINITION_SHA = "a" * 64
|
||||||
|
NON_CHECKPOINTABLE_DEFINITION_SHA = "b" * 64
|
||||||
|
ADAPTER_SHA = "c" * 64
|
||||||
|
CATALOG_SHA = "d" * 64
|
||||||
|
RESULT_SHA = "e" * 64
|
||||||
|
LIVE_EPOCH_SHA = "f" * 64
|
||||||
|
SOURCE_BUNDLE_SHA = "1" * 64
|
||||||
|
SOURCE_CAPABILITIES_SHA = "2" * 64
|
||||||
|
EXECUTOR_RELEASE_SHA = "3" * 64
|
||||||
|
EXECUTOR_IMAGE_SHA = "4" * 64
|
||||||
|
MODEL_MANIFEST_SHA = "5" * 64
|
||||||
|
RESOURCE_PROFILE_SHA = "6" * 64
|
||||||
|
|
||||||
|
|
||||||
|
def _definitions() -> RecordedRunDefinitionRegistry:
|
||||||
|
return RecordedRunDefinitionRegistry(
|
||||||
|
(
|
||||||
|
RecordedRunDefinition(
|
||||||
|
setup_id="travel-tgs-eomt-v1",
|
||||||
|
definition_id="travel-tgs-eomt",
|
||||||
|
definition_version=1,
|
||||||
|
definition_sha256=DEFINITION_SHA,
|
||||||
|
source_adapter_id="sealed-session-bundle",
|
||||||
|
source_adapter_version=1,
|
||||||
|
source_adapter_sha256=ADAPTER_SHA,
|
||||||
|
executor_release_id="travel-tgs-executor-v1",
|
||||||
|
executor_release_sha256=EXECUTOR_RELEASE_SHA,
|
||||||
|
executor_image_sha256=EXECUTOR_IMAGE_SHA,
|
||||||
|
model_release_ids=("eomt-v1", "travel-tgs-v1"),
|
||||||
|
model_manifest_sha256=MODEL_MANIFEST_SHA,
|
||||||
|
resource_profile_id="worker006-single-gpu-v1",
|
||||||
|
resource_profile_sha256=RESOURCE_PROFILE_SHA,
|
||||||
|
checkpoint_policy="cooperative",
|
||||||
|
allowed_checkpoints=(
|
||||||
|
"source-bundle-ready",
|
||||||
|
"model-batch-finished",
|
||||||
|
"evidence-sealed",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
RecordedRunDefinition(
|
||||||
|
setup_id="legacy-monolith-v1",
|
||||||
|
definition_id="legacy-monolith",
|
||||||
|
definition_version=1,
|
||||||
|
definition_sha256=NON_CHECKPOINTABLE_DEFINITION_SHA,
|
||||||
|
source_adapter_id="sealed-session-bundle",
|
||||||
|
source_adapter_version=1,
|
||||||
|
source_adapter_sha256=ADAPTER_SHA,
|
||||||
|
executor_release_id="legacy-monolith-executor-v1",
|
||||||
|
executor_release_sha256=EXECUTOR_RELEASE_SHA,
|
||||||
|
executor_image_sha256=EXECUTOR_IMAGE_SHA,
|
||||||
|
model_release_ids=("legacy-monolith-v1",),
|
||||||
|
model_manifest_sha256=MODEL_MANIFEST_SHA,
|
||||||
|
resource_profile_id="worker006-single-gpu-v1",
|
||||||
|
resource_profile_sha256=RESOURCE_PROFILE_SHA,
|
||||||
|
checkpoint_policy="non-checkpointable",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _queue(
|
||||||
|
tmp_path: Path,
|
||||||
|
*,
|
||||||
|
with_non_checkpointable_preemptor: bool = False,
|
||||||
|
) -> ObservatoryRecordedJobQueue:
|
||||||
|
preemptor = None
|
||||||
|
if with_non_checkpointable_preemptor:
|
||||||
|
def preemptor(request):
|
||||||
|
return ObservatoryNonCheckpointableCancellationReceipt.sealed(
|
||||||
|
request,
|
||||||
|
cancellation_id=f"cancel-{request.cancellation_request_id}",
|
||||||
|
)
|
||||||
|
|
||||||
|
return ObservatoryRecordedJobQueue(
|
||||||
|
tmp_path,
|
||||||
|
definitions=_definitions(),
|
||||||
|
clock=lambda: NOW,
|
||||||
|
non_checkpointable_preemptor=preemptor,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _intent(
|
||||||
|
*,
|
||||||
|
idempotency_key: str = "recorded-request-001",
|
||||||
|
source_session_id: str = "20260828T130511Z_viewer_live",
|
||||||
|
source_catalog_sha256: str = CATALOG_SHA,
|
||||||
|
setup_id: str = "travel-tgs-eomt-v1",
|
||||||
|
definition_sha256: str = DEFINITION_SHA,
|
||||||
|
) -> ObservatoryRecordedJobIntent:
|
||||||
|
return ObservatoryRecordedJobIntent(
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
source_session_id=source_session_id,
|
||||||
|
source_catalog_sha256=source_catalog_sha256,
|
||||||
|
source_bundle_sha256=SOURCE_BUNDLE_SHA,
|
||||||
|
source_capability_manifest_sha256=SOURCE_CAPABILITIES_SHA,
|
||||||
|
setup_id=setup_id,
|
||||||
|
definition_sha256=definition_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _live_intent(
|
||||||
|
*,
|
||||||
|
trigger_id: str = "k1-live-start-001",
|
||||||
|
live_session_id: str = "20260830T210000Z_k1_live",
|
||||||
|
acquisition_epoch_sha256: str = LIVE_EPOCH_SHA,
|
||||||
|
) -> ObservatoryLiveLeaseIntent:
|
||||||
|
return ObservatoryLiveLeaseIntent(
|
||||||
|
trigger_id=trigger_id,
|
||||||
|
live_session_id=live_session_id,
|
||||||
|
acquisition_epoch_sha256=acquisition_epoch_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _running_job(
|
||||||
|
queue: ObservatoryRecordedJobQueue,
|
||||||
|
*,
|
||||||
|
intent: ObservatoryRecordedJobIntent | None = None,
|
||||||
|
claim_request_id: str = "worker-claim-001",
|
||||||
|
):
|
||||||
|
job, created = queue.submit(intent or _intent())
|
||||||
|
assert created is True
|
||||||
|
queue.enqueue(job.job_id)
|
||||||
|
claim = queue.claim_next(
|
||||||
|
claimant_id="recorded-worker", claim_request_id=claim_request_id
|
||||||
|
)
|
||||||
|
assert claim is not None
|
||||||
|
running = queue.start(job.job_id, claim_token=claim.claim_token)
|
||||||
|
assert running.state == "running"
|
||||||
|
return running, claim
|
||||||
|
|
||||||
|
|
||||||
|
def test_submission_is_durable_exactly_idempotent_and_path_free(tmp_path: Path) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
first, first_created = queue.submit(_intent())
|
||||||
|
second, second_created = queue.submit(_intent())
|
||||||
|
restored = ObservatoryRecordedJobQueue(
|
||||||
|
tmp_path,
|
||||||
|
definitions=_definitions(),
|
||||||
|
clock=lambda: NOW,
|
||||||
|
).get(first.job_id)
|
||||||
|
|
||||||
|
assert first_created is True
|
||||||
|
assert second_created is False
|
||||||
|
assert second == first
|
||||||
|
assert restored == first
|
||||||
|
assert first.state == "accepted"
|
||||||
|
assert first.priority_class == "recorded"
|
||||||
|
assert first.priority_rank == RECORDED_PRIORITY_RANK
|
||||||
|
assert first.source_adapter_id == "sealed-session-bundle"
|
||||||
|
assert first.checkpoint_policy == "cooperative"
|
||||||
|
projection = first.as_dict()
|
||||||
|
assert projection["priority"] == {
|
||||||
|
"class": "recorded",
|
||||||
|
"rank": RECORDED_PRIORITY_RANK,
|
||||||
|
"server_owned": True,
|
||||||
|
}
|
||||||
|
assert "path" not in str(projection).lower()
|
||||||
|
assert not hasattr(_intent(), "command")
|
||||||
|
assert not hasattr(_intent(), "image")
|
||||||
|
assert projection["source"]["bundle_sha256"] == SOURCE_BUNDLE_SHA
|
||||||
|
assert projection["source"]["capability_manifest_sha256"] == (
|
||||||
|
SOURCE_CAPABILITIES_SHA
|
||||||
|
)
|
||||||
|
assert projection["executor"]["release_sha256"] == EXECUTOR_RELEASE_SHA
|
||||||
|
assert projection["executor"]["image_sha256"] == EXECUTOR_IMAGE_SHA
|
||||||
|
assert projection["executor"]["model_manifest_sha256"] == MODEL_MANIFEST_SHA
|
||||||
|
assert projection["executor"]["resource_profile_sha256"] == (
|
||||||
|
RESOURCE_PROFILE_SHA
|
||||||
|
)
|
||||||
|
assert queue.database_path.name == RECORDED_JOB_DATABASE_NAME
|
||||||
|
assert queue.database_path.stat().st_mode & 0o777 == 0o600
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueConflictError, match="idempotency"):
|
||||||
|
queue.submit(replace(_intent(), source_session_id="another-session"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_submission_and_enqueue_can_commit_as_one_idempotent_transaction(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
|
||||||
|
first, created = queue.submit(_intent(), enqueue=True)
|
||||||
|
repeated, repeated_created = queue.submit(_intent(), enqueue=True)
|
||||||
|
|
||||||
|
assert created is True
|
||||||
|
assert repeated_created is False
|
||||||
|
assert first.state == "queued"
|
||||||
|
assert repeated == first
|
||||||
|
|
||||||
|
|
||||||
|
def test_session_setup_identity_changes_for_each_exact_combination(tmp_path: Path) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
first, _ = queue.submit(_intent())
|
||||||
|
second, _ = queue.submit(
|
||||||
|
_intent(
|
||||||
|
idempotency_key="recorded-request-002",
|
||||||
|
source_session_id="another-session",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
third, _ = queue.submit(
|
||||||
|
_intent(
|
||||||
|
idempotency_key="recorded-request-003",
|
||||||
|
setup_id="legacy-monolith-v1",
|
||||||
|
definition_sha256=NON_CHECKPOINTABLE_DEFINITION_SHA,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len({first.identity_sha256, second.identity_sha256, third.identity_sha256}) == 3
|
||||||
|
assert second.source_adapter_sha256 == first.source_adapter_sha256
|
||||||
|
assert third.checkpoint_policy == "non-checkpointable"
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueConflictError, match="allowlisted"):
|
||||||
|
queue.submit(
|
||||||
|
_intent(
|
||||||
|
idempotency_key="recorded-request-004",
|
||||||
|
definition_sha256="0" * 64,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_normal_recorded_job_lifecycle_and_terminal_idempotency(tmp_path: Path) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
running, claim = _running_job(queue)
|
||||||
|
|
||||||
|
checkpointed = queue.checkpoint(
|
||||||
|
running.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
checkpoint_id="model-batch-finished",
|
||||||
|
)
|
||||||
|
assert checkpointed.state == "running"
|
||||||
|
assert checkpointed.last_checkpoint_id == "model-batch-finished"
|
||||||
|
|
||||||
|
succeeded = queue.succeed(
|
||||||
|
running.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
result_id="recorded-result-001",
|
||||||
|
result_sha256=RESULT_SHA,
|
||||||
|
)
|
||||||
|
replayed = queue.succeed(
|
||||||
|
running.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
result_id="recorded-result-001",
|
||||||
|
result_sha256=RESULT_SHA,
|
||||||
|
)
|
||||||
|
assert succeeded.state == "succeeded"
|
||||||
|
assert replayed == succeeded
|
||||||
|
assert succeeded.result_sha256 == RESULT_SHA
|
||||||
|
assert succeeded.active_claim_token is None
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueConflictError, match="terminal"):
|
||||||
|
queue.fail(
|
||||||
|
running.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
error_code="worker-failed",
|
||||||
|
message="Synthetic failure.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_success_requires_running_and_cannot_publish_during_preemption(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
job, _ = queue.submit(_intent())
|
||||||
|
queue.enqueue(job.job_id)
|
||||||
|
claim = queue.claim_next(
|
||||||
|
claimant_id="recorded-worker",
|
||||||
|
claim_request_id="worker-claim-001",
|
||||||
|
)
|
||||||
|
assert claim is not None
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueConflictError, match="claimed"):
|
||||||
|
queue.succeed(
|
||||||
|
job.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
result_id="recorded-result-001",
|
||||||
|
result_sha256=RESULT_SHA,
|
||||||
|
)
|
||||||
|
|
||||||
|
queue.start(job.job_id, claim_token=claim.claim_token)
|
||||||
|
queue.request_live(_live_intent())
|
||||||
|
assert queue.get(job.job_id).preemption_requested is True
|
||||||
|
with pytest.raises(
|
||||||
|
ObservatoryRecordedQueueConflictError,
|
||||||
|
match="preemption",
|
||||||
|
):
|
||||||
|
queue.succeed(
|
||||||
|
job.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
result_id="recorded-result-001",
|
||||||
|
result_sha256=RESULT_SHA,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_claim_is_exactly_idempotent_including_empty_result(tmp_path: Path) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
|
||||||
|
empty = queue.claim_next(
|
||||||
|
claimant_id="recorded-worker", claim_request_id="empty-poll-001"
|
||||||
|
)
|
||||||
|
assert empty is None
|
||||||
|
job, _ = queue.submit(_intent())
|
||||||
|
queue.enqueue(job.job_id)
|
||||||
|
assert (
|
||||||
|
queue.claim_next(
|
||||||
|
claimant_id="recorded-worker", claim_request_id="empty-poll-001"
|
||||||
|
)
|
||||||
|
is None
|
||||||
|
)
|
||||||
|
|
||||||
|
claim = queue.claim_next(
|
||||||
|
claimant_id="recorded-worker", claim_request_id="non-empty-poll-001"
|
||||||
|
)
|
||||||
|
retry = queue.claim_next(
|
||||||
|
claimant_id="recorded-worker", claim_request_id="non-empty-poll-001"
|
||||||
|
)
|
||||||
|
assert claim is not None
|
||||||
|
assert retry is not None
|
||||||
|
assert retry.claim_token == claim.claim_token
|
||||||
|
assert retry.job.job_id == job.job_id
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueConflictError, match="claim"):
|
||||||
|
queue.claim_next(
|
||||||
|
claimant_id="another-worker", claim_request_id="non-empty-poll-001"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_single_worker_resource_has_only_one_recorded_owner(tmp_path: Path) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
first, _ = queue.submit(_intent())
|
||||||
|
second, _ = queue.submit(
|
||||||
|
_intent(
|
||||||
|
idempotency_key="recorded-request-002",
|
||||||
|
source_session_id="another-session",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
queue.enqueue(first.job_id)
|
||||||
|
queue.enqueue(second.job_id)
|
||||||
|
first_claim = queue.claim_next(
|
||||||
|
claimant_id="recorded-worker", claim_request_id="owner-poll-001"
|
||||||
|
)
|
||||||
|
assert first_claim is not None
|
||||||
|
assert (
|
||||||
|
queue.claim_next(
|
||||||
|
claimant_id="recorded-worker", claim_request_id="owner-poll-002"
|
||||||
|
)
|
||||||
|
is None
|
||||||
|
)
|
||||||
|
|
||||||
|
completed_job_id = first_claim.job.job_id
|
||||||
|
queue.fail(
|
||||||
|
completed_job_id,
|
||||||
|
claim_token=first_claim.claim_token,
|
||||||
|
error_code="synthetic-failure",
|
||||||
|
message="Release the synthetic single-Worker ownership.",
|
||||||
|
)
|
||||||
|
second_claim = queue.claim_next(
|
||||||
|
claimant_id="recorded-worker", claim_request_id="owner-poll-003"
|
||||||
|
)
|
||||||
|
assert second_claim is not None
|
||||||
|
assert second_claim.job.job_id in {first.job_id, second.job_id} - {completed_job_id}
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_lease_cooperatively_pauses_and_resumes_recorded_job(tmp_path: Path) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
running, claim = _running_job(queue)
|
||||||
|
|
||||||
|
lease, created = queue.request_live(_live_intent())
|
||||||
|
exact_retry, retry_created = queue.request_live(_live_intent())
|
||||||
|
requested_job = queue.get(running.job_id)
|
||||||
|
assert created is True
|
||||||
|
assert retry_created is False
|
||||||
|
assert exact_retry == lease
|
||||||
|
assert lease.state == "pending"
|
||||||
|
assert lease.priority_rank == LIVE_K1_PRIORITY_RANK
|
||||||
|
assert requested_job.state == "running"
|
||||||
|
assert requested_job.preemption_requested is True
|
||||||
|
assert queue.admission_gate().blocked is True
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueBusyError, match="safe preemption"):
|
||||||
|
queue.activate_live(lease.lease_id)
|
||||||
|
|
||||||
|
paused = queue.checkpoint(
|
||||||
|
running.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
checkpoint_id="model-batch-finished",
|
||||||
|
)
|
||||||
|
assert paused.state == "paused"
|
||||||
|
active = queue.activate_live(lease.lease_id)
|
||||||
|
assert active.state == "active"
|
||||||
|
|
||||||
|
another, _ = queue.submit(
|
||||||
|
_intent(
|
||||||
|
idempotency_key="recorded-request-002",
|
||||||
|
source_session_id="another-session",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
queue.enqueue(another.job_id)
|
||||||
|
assert (
|
||||||
|
queue.claim_next(
|
||||||
|
claimant_id="recorded-worker", claim_request_id="blocked-poll-001"
|
||||||
|
)
|
||||||
|
is None
|
||||||
|
)
|
||||||
|
|
||||||
|
completed = queue.finish_live(
|
||||||
|
lease.lease_id,
|
||||||
|
terminal_trigger_id="k1-live-finish-001",
|
||||||
|
outcome="completed",
|
||||||
|
)
|
||||||
|
completed_retry = queue.finish_live(
|
||||||
|
lease.lease_id,
|
||||||
|
terminal_trigger_id="k1-live-finish-001",
|
||||||
|
outcome="completed",
|
||||||
|
)
|
||||||
|
assert completed.state == "completed"
|
||||||
|
assert completed_retry == completed
|
||||||
|
assert queue.admission_gate().blocked is False
|
||||||
|
resumed = queue.get(running.job_id)
|
||||||
|
assert resumed.state == "queued"
|
||||||
|
assert resumed.preemption_requested is False
|
||||||
|
assert resumed.active_claim_token is None
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueConflictError, match="terminal"):
|
||||||
|
queue.finish_live(
|
||||||
|
lease.lease_id,
|
||||||
|
terminal_trigger_id="different-terminal-trigger",
|
||||||
|
outcome="failed",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cancelled_pending_live_lease_releases_cooperative_preemption_request(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
running, claim = _running_job(queue)
|
||||||
|
lease, _ = queue.request_live(_live_intent())
|
||||||
|
assert queue.get(running.job_id).preemption_requested is True
|
||||||
|
|
||||||
|
cancelled = queue.finish_live(
|
||||||
|
lease.lease_id,
|
||||||
|
terminal_trigger_id="k1-live-cancel-001",
|
||||||
|
outcome="cancelled",
|
||||||
|
)
|
||||||
|
released = queue.get(running.job_id)
|
||||||
|
|
||||||
|
assert cancelled.state == "cancelled"
|
||||||
|
assert released.state == "running"
|
||||||
|
assert released.preemption_requested is False
|
||||||
|
assert released.active_claim_token == claim.claim_token
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_request_pauses_claimed_job_before_execution(tmp_path: Path) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
job, _ = queue.submit(_intent())
|
||||||
|
queue.enqueue(job.job_id)
|
||||||
|
claim = queue.claim_next(
|
||||||
|
claimant_id="recorded-worker", claim_request_id="worker-claim-001"
|
||||||
|
)
|
||||||
|
assert claim is not None
|
||||||
|
|
||||||
|
lease, _ = queue.request_live(_live_intent())
|
||||||
|
paused = queue.get(job.job_id)
|
||||||
|
assert paused.state == "paused"
|
||||||
|
assert queue.start(job.job_id, claim_token=claim.claim_token).state == "paused"
|
||||||
|
assert queue.activate_live(lease.lease_id).state == "active"
|
||||||
|
queue.finish_live(
|
||||||
|
lease.lease_id,
|
||||||
|
terminal_trigger_id="k1-live-finish-001",
|
||||||
|
outcome="completed",
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"):
|
||||||
|
queue.start(job.job_id, claim_token=claim.claim_token)
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_checkpointable_job_is_cancelled_and_restarts_from_zero_for_live(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
queue = _queue(tmp_path, with_non_checkpointable_preemptor=True)
|
||||||
|
running, claim = _running_job(
|
||||||
|
queue,
|
||||||
|
intent=_intent(
|
||||||
|
setup_id="legacy-monolith-v1",
|
||||||
|
definition_sha256=NON_CHECKPOINTABLE_DEFINITION_SHA,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
lease, _ = queue.request_live(_live_intent())
|
||||||
|
paused = queue.get(running.job_id)
|
||||||
|
assert paused.state == "paused"
|
||||||
|
assert paused.preemption_requested is True
|
||||||
|
assert paused.restart_from_zero is True
|
||||||
|
assert paused.preemption_receipt_sha256 is not None
|
||||||
|
assert paused.active_claim_token is None
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"):
|
||||||
|
queue.checkpoint(
|
||||||
|
running.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
checkpoint_id="model-batch-finished",
|
||||||
|
)
|
||||||
|
assert queue.activate_live(lease.lease_id).state == "active"
|
||||||
|
assert (
|
||||||
|
queue.finish_live(
|
||||||
|
lease.lease_id,
|
||||||
|
terminal_trigger_id="k1-live-finish-001",
|
||||||
|
outcome="failed",
|
||||||
|
).state
|
||||||
|
== "failed"
|
||||||
|
)
|
||||||
|
resumed = queue.get(running.job_id)
|
||||||
|
assert resumed.state == "queued"
|
||||||
|
assert resumed.restart_from_zero is True
|
||||||
|
|
||||||
|
new_claim = queue.claim_next(
|
||||||
|
claimant_id="recorded-worker", claim_request_id="worker-claim-002"
|
||||||
|
)
|
||||||
|
assert new_claim is not None
|
||||||
|
assert new_claim.job.job_id == running.job_id
|
||||||
|
assert new_claim.job.restart_from_zero is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_request_fails_closed_without_non_checkpointable_canceler(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
running, _claim = _running_job(
|
||||||
|
queue,
|
||||||
|
intent=_intent(
|
||||||
|
setup_id="legacy-monolith-v1",
|
||||||
|
definition_sha256=NON_CHECKPOINTABLE_DEFINITION_SHA,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryRecordedPreemptionError, match="scheduler-owned"):
|
||||||
|
queue.request_live(_live_intent())
|
||||||
|
|
||||||
|
assert queue.admission_gate().blocked is True
|
||||||
|
assert queue.get(running.job_id).state == "preemption-pending"
|
||||||
|
with sqlite3.connect(queue.database_path) as connection:
|
||||||
|
intent_state = connection.execute(
|
||||||
|
"SELECT state FROM observatory_recorded_preemptions"
|
||||||
|
).fetchone()[0]
|
||||||
|
assert intent_state == "pending"
|
||||||
|
|
||||||
|
recovered = _queue(tmp_path, with_non_checkpointable_preemptor=True)
|
||||||
|
lease, created = recovered.request_live(_live_intent())
|
||||||
|
assert created is False
|
||||||
|
assert recovered.get(running.job_id).state == "paused"
|
||||||
|
assert recovered.get(running.job_id).restart_from_zero is True
|
||||||
|
assert recovered.activate_live(lease.lease_id).state == "active"
|
||||||
|
|
||||||
|
|
||||||
|
def test_durable_cancel_intent_survives_callback_crash_and_reuses_identity(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
cancellation_request_ids: list[str] = []
|
||||||
|
|
||||||
|
def crash_after_external_side_effect(request):
|
||||||
|
cancellation_request_ids.append(request.cancellation_request_id)
|
||||||
|
raise OSError("synthetic scheduler process loss after cancellation")
|
||||||
|
|
||||||
|
queue = ObservatoryRecordedJobQueue(
|
||||||
|
tmp_path,
|
||||||
|
definitions=_definitions(),
|
||||||
|
clock=lambda: NOW,
|
||||||
|
non_checkpointable_preemptor=crash_after_external_side_effect,
|
||||||
|
)
|
||||||
|
running, _claim = _running_job(
|
||||||
|
queue,
|
||||||
|
intent=_intent(
|
||||||
|
setup_id="legacy-monolith-v1",
|
||||||
|
definition_sha256=NON_CHECKPOINTABLE_DEFINITION_SHA,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
with pytest.raises(ObservatoryRecordedPreemptionError, match="did not release"):
|
||||||
|
queue.request_live(_live_intent())
|
||||||
|
|
||||||
|
assert queue.get(running.job_id).state == "preemption-pending"
|
||||||
|
|
||||||
|
def replay_exact_cancel(request):
|
||||||
|
cancellation_request_ids.append(request.cancellation_request_id)
|
||||||
|
return ObservatoryNonCheckpointableCancellationReceipt.sealed(
|
||||||
|
request,
|
||||||
|
cancellation_id=f"cancel-{request.cancellation_request_id}",
|
||||||
|
)
|
||||||
|
|
||||||
|
recovered = ObservatoryRecordedJobQueue(
|
||||||
|
tmp_path,
|
||||||
|
definitions=_definitions(),
|
||||||
|
clock=lambda: NOW,
|
||||||
|
non_checkpointable_preemptor=replay_exact_cancel,
|
||||||
|
)
|
||||||
|
lease, created = recovered.request_live(_live_intent())
|
||||||
|
|
||||||
|
assert created is False
|
||||||
|
assert len(cancellation_request_ids) == 2
|
||||||
|
assert cancellation_request_ids[0] == cancellation_request_ids[1]
|
||||||
|
assert recovered.get(running.job_id).state == "paused"
|
||||||
|
assert recovered.activate_live(lease.lease_id).state == "active"
|
||||||
|
|
||||||
|
|
||||||
|
def test_learned_model_release_list_may_be_empty_for_algorithm_only_tgs() -> None:
|
||||||
|
definition = RecordedRunDefinition(
|
||||||
|
setup_id="travel-tgs-algorithm-v1",
|
||||||
|
definition_id="travel-tgs-algorithm",
|
||||||
|
definition_version=1,
|
||||||
|
definition_sha256="7" * 64,
|
||||||
|
source_adapter_id="sealed-session-bundle",
|
||||||
|
source_adapter_version=1,
|
||||||
|
source_adapter_sha256=ADAPTER_SHA,
|
||||||
|
executor_release_id="travel-tgs-executor-v1",
|
||||||
|
executor_release_sha256=EXECUTOR_RELEASE_SHA,
|
||||||
|
executor_image_sha256=EXECUTOR_IMAGE_SHA,
|
||||||
|
model_release_ids=(),
|
||||||
|
model_manifest_sha256=MODEL_MANIFEST_SHA,
|
||||||
|
resource_profile_id="worker006-single-gpu-v1",
|
||||||
|
resource_profile_sha256=RESOURCE_PROFILE_SHA,
|
||||||
|
checkpoint_policy="cooperative",
|
||||||
|
allowed_checkpoints=("model-batch-finished",),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert definition.model_release_ids == ()
|
||||||
|
assert definition.learned_models == ()
|
||||||
|
assert definition.model_manifest_sha256 == MODEL_MANIFEST_SHA
|
||||||
|
|
||||||
|
|
||||||
|
def test_reconciliation_required_is_durable_terminal_state(tmp_path: Path) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
running, claim = _running_job(queue)
|
||||||
|
|
||||||
|
uncertain = queue.require_reconciliation(
|
||||||
|
running.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
reason_code="worker-outcome-unknown",
|
||||||
|
message="Dispatch was accepted but its terminal receipt is unavailable.",
|
||||||
|
)
|
||||||
|
restored = ObservatoryRecordedJobQueue(
|
||||||
|
tmp_path,
|
||||||
|
definitions=_definitions(),
|
||||||
|
clock=lambda: NOW,
|
||||||
|
).get(running.job_id)
|
||||||
|
assert uncertain.state == "reconciliation-required"
|
||||||
|
assert restored == uncertain
|
||||||
|
|
||||||
|
|
||||||
|
def test_reconciliation_required_quarantines_recorded_and_live_ownership(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
running, claim = _running_job(queue)
|
||||||
|
queue.require_reconciliation(
|
||||||
|
running.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
reason_code="worker-outcome-unknown",
|
||||||
|
message="Worker ownership cannot be proven released.",
|
||||||
|
)
|
||||||
|
waiting, _ = queue.submit(
|
||||||
|
_intent(idempotency_key="recorded-request-002")
|
||||||
|
)
|
||||||
|
queue.enqueue(waiting.job_id)
|
||||||
|
|
||||||
|
assert queue.claim_next(
|
||||||
|
claimant_id="recorded-worker",
|
||||||
|
claim_request_id="worker-claim-after-reconciliation",
|
||||||
|
) is None
|
||||||
|
lease, _ = queue.request_live(_live_intent())
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueBusyError, match="recorded work"):
|
||||||
|
queue.activate_live(lease.lease_id)
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_lease_requires_explicit_valid_terminal_transition(tmp_path: Path) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
lease, _ = queue.request_live(_live_intent())
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueConflictError, match="complete"):
|
||||||
|
queue.finish_live(
|
||||||
|
lease.lease_id,
|
||||||
|
terminal_trigger_id="k1-live-finish-001",
|
||||||
|
outcome="completed",
|
||||||
|
)
|
||||||
|
cancelled = queue.finish_live(
|
||||||
|
lease.lease_id,
|
||||||
|
terminal_trigger_id="k1-live-cancel-001",
|
||||||
|
outcome="cancelled",
|
||||||
|
)
|
||||||
|
assert cancelled.state == "cancelled"
|
||||||
|
assert queue.admission_gate().blocked is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_queue_detects_mutated_immutable_identity(tmp_path: Path) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
job, _ = queue.submit(_intent())
|
||||||
|
with sqlite3.connect(queue.database_path) as connection:
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE observatory_recorded_jobs SET source_catalog_sha256 = ? "
|
||||||
|
"WHERE job_id = ?",
|
||||||
|
("0" * 64, job.job_id),
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueIntegrityError, match="identity"):
|
||||||
|
queue.get(job.job_id)
|
||||||
@@ -0,0 +1,253 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from typing import Any, cast
|
||||||
|
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
from k1link.observatory.m49_queue_binding import M49QueueBindingIntegrityError
|
||||||
|
from k1link.observatory.recorded_jobs import (
|
||||||
|
ObservatoryRecordedJobIntent,
|
||||||
|
ObservatoryRecordedJobQueue,
|
||||||
|
RecordedRunDefinition,
|
||||||
|
RecordedRunDefinitionRegistry,
|
||||||
|
)
|
||||||
|
from k1link.observatory.setups import LaboratorySetupRegistry
|
||||||
|
from k1link.sessions import SessionNotFoundError
|
||||||
|
from k1link.sessions.models import SessionSummary
|
||||||
|
from k1link.web.observatory_api import build_observatory_router
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-laboratory-setups.json"
|
||||||
|
RAV00_SESSION_ID = "20260720T065719Z_viewer_live"
|
||||||
|
SETUP_ID = "m49-tgs-full-shadow-v1"
|
||||||
|
CATALOG_SHA256 = "1" * 64
|
||||||
|
SOURCE_BUNDLE_SHA256 = "2" * 64
|
||||||
|
CAPABILITY_SHA256 = "3" * 64
|
||||||
|
|
||||||
|
|
||||||
|
def _source() -> SessionSummary:
|
||||||
|
return SessionSummary(
|
||||||
|
session_id=RAV00_SESSION_ID,
|
||||||
|
display_name="RAVNOVES00",
|
||||||
|
status="ready",
|
||||||
|
started_at_utc="2026-07-20T06:57:19Z",
|
||||||
|
completed_at_utc="2026-07-20T07:06:15Z",
|
||||||
|
duration_seconds=536.0,
|
||||||
|
modalities=("point-cloud", "trajectory", "video"),
|
||||||
|
source_count=3,
|
||||||
|
total_bytes=1,
|
||||||
|
replayable=True,
|
||||||
|
origin="recorded",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _Store:
|
||||||
|
def get_session(self, session_id: str) -> SimpleNamespace:
|
||||||
|
if session_id != RAV00_SESSION_ID:
|
||||||
|
raise SessionNotFoundError(session_id)
|
||||||
|
return SimpleNamespace(summary=_source())
|
||||||
|
|
||||||
|
|
||||||
|
class _Admission:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
definition: RecordedRunDefinition,
|
||||||
|
catalog_sha256: str,
|
||||||
|
) -> None:
|
||||||
|
self.definition = definition
|
||||||
|
self.catalog_sha256 = catalog_sha256
|
||||||
|
|
||||||
|
def intent(self, *, idempotency_key: str) -> ObservatoryRecordedJobIntent:
|
||||||
|
return ObservatoryRecordedJobIntent(
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
source_session_id=RAV00_SESSION_ID,
|
||||||
|
source_catalog_sha256=self.catalog_sha256,
|
||||||
|
source_bundle_sha256=SOURCE_BUNDLE_SHA256,
|
||||||
|
source_capability_manifest_sha256=CAPABILITY_SHA256,
|
||||||
|
setup_id=self.definition.setup_id,
|
||||||
|
definition_sha256=self.definition.definition_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _BindingService:
|
||||||
|
def __init__(self, definition: RecordedRunDefinition) -> None:
|
||||||
|
self.config = SimpleNamespace(setup=SimpleNamespace(setup_id=SETUP_ID))
|
||||||
|
self.definition = definition
|
||||||
|
self.catalog_sha256 = CATALOG_SHA256
|
||||||
|
self.check_count = 0
|
||||||
|
self.admit_count = 0
|
||||||
|
|
||||||
|
def check(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
source_session_id: str,
|
||||||
|
setup_id: str,
|
||||||
|
definition_sha256: str,
|
||||||
|
) -> _Admission:
|
||||||
|
self.check_count += 1
|
||||||
|
return self._resolve(source_session_id, setup_id, definition_sha256)
|
||||||
|
|
||||||
|
def admit(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
source_session_id: str,
|
||||||
|
setup_id: str,
|
||||||
|
definition_sha256: str,
|
||||||
|
) -> _Admission:
|
||||||
|
self.admit_count += 1
|
||||||
|
return self._resolve(source_session_id, setup_id, definition_sha256)
|
||||||
|
|
||||||
|
def _resolve(
|
||||||
|
self,
|
||||||
|
source_session_id: str,
|
||||||
|
setup_id: str,
|
||||||
|
definition_sha256: str,
|
||||||
|
) -> _Admission:
|
||||||
|
if (
|
||||||
|
source_session_id != RAV00_SESSION_ID
|
||||||
|
or setup_id != self.definition.setup_id
|
||||||
|
or definition_sha256 != self.definition.definition_sha256
|
||||||
|
):
|
||||||
|
raise M49QueueBindingIntegrityError("not the exact M4.9 binding")
|
||||||
|
return _Admission(self.definition, self.catalog_sha256)
|
||||||
|
|
||||||
|
|
||||||
|
def _services(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> tuple[
|
||||||
|
LaboratorySetupRegistry,
|
||||||
|
_BindingService,
|
||||||
|
ObservatoryRecordedJobQueue,
|
||||||
|
RecordedRunDefinition,
|
||||||
|
]:
|
||||||
|
registry = LaboratorySetupRegistry.from_file(
|
||||||
|
REGISTRY_PATH,
|
||||||
|
repository_root=REPOSITORY_ROOT,
|
||||||
|
)
|
||||||
|
catalog = registry.catalog(_source())
|
||||||
|
setups = cast(list[dict[str, Any]], catalog["setups"])
|
||||||
|
definition_document = cast(dict[str, object], setups[0]["run_definition"])
|
||||||
|
definition = RecordedRunDefinition(
|
||||||
|
setup_id=SETUP_ID,
|
||||||
|
definition_id=str(definition_document["definition_id"]),
|
||||||
|
definition_version=int(definition_document["version"]),
|
||||||
|
definition_sha256=str(definition_document["definition_sha256"]),
|
||||||
|
source_adapter_id="ravnoves00-m49-source-pack",
|
||||||
|
source_adapter_version=1,
|
||||||
|
source_adapter_sha256="4" * 64,
|
||||||
|
executor_release_id="m49-tgs-full-shadow-worker-release",
|
||||||
|
executor_release_sha256="5" * 64,
|
||||||
|
executor_image_sha256="6" * 64,
|
||||||
|
model_release_ids=(),
|
||||||
|
model_manifest_sha256="7" * 64,
|
||||||
|
resource_profile_id="worker006-cpu-single-run-v1",
|
||||||
|
resource_profile_sha256="8" * 64,
|
||||||
|
checkpoint_policy="non-checkpointable",
|
||||||
|
)
|
||||||
|
binding = _BindingService(definition)
|
||||||
|
queue = ObservatoryRecordedJobQueue(
|
||||||
|
tmp_path,
|
||||||
|
definitions=RecordedRunDefinitionRegistry((definition,)),
|
||||||
|
)
|
||||||
|
return registry, binding, queue, definition
|
||||||
|
|
||||||
|
|
||||||
|
def test_exact_m49_preflight_is_queueable_and_submit_is_idempotently_queued(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
registry, binding, queue, definition = _services(tmp_path)
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(
|
||||||
|
build_observatory_router(
|
||||||
|
_Store(), # type: ignore[arg-type]
|
||||||
|
setup_registry=registry,
|
||||||
|
recorded_binding_service=binding, # type: ignore[arg-type]
|
||||||
|
recorded_job_queue=queue,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
preflight = client.post(
|
||||||
|
"/api/v1/observatory/run-preflights",
|
||||||
|
json={
|
||||||
|
"schema_version": "missioncore.observatory-run-preflight-request/v1",
|
||||||
|
"source_session_id": RAV00_SESSION_ID,
|
||||||
|
"setup_id": SETUP_ID,
|
||||||
|
"definition_sha256": definition.definition_sha256,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert preflight.status_code == 200
|
||||||
|
assert preflight.json()["outcome"] == "queueable"
|
||||||
|
assert preflight.json()["submission_allowed"] is True
|
||||||
|
assert next(
|
||||||
|
check
|
||||||
|
for check in preflight.json()["checks"]
|
||||||
|
if check["check_id"] == "durable-queue"
|
||||||
|
)["outcome"] == "pass"
|
||||||
|
assert binding.check_count == 1
|
||||||
|
assert binding.admit_count == 0
|
||||||
|
|
||||||
|
request = {
|
||||||
|
"schema_version": "missioncore.observatory-recorded-run-submit/v1",
|
||||||
|
"idempotency_key": "observatory-ui:m49:stable-request",
|
||||||
|
"source_session_id": RAV00_SESSION_ID,
|
||||||
|
"setup_id": SETUP_ID,
|
||||||
|
}
|
||||||
|
submitted = client.post("/api/v1/observatory/runs", json=request)
|
||||||
|
binding.catalog_sha256 = "9" * 64
|
||||||
|
repeated = client.post("/api/v1/observatory/runs", json=request)
|
||||||
|
|
||||||
|
assert submitted.status_code == 202
|
||||||
|
assert repeated.status_code == 202
|
||||||
|
assert submitted.json()["job_id"] == repeated.json()["job_id"]
|
||||||
|
assert submitted.json()["state"] == "queued"
|
||||||
|
assert submitted.json()["source"]["session_id"] == RAV00_SESSION_ID
|
||||||
|
assert submitted.json()["setup"]["setup_id"] == SETUP_ID
|
||||||
|
assert submitted.json()["executor"]["learned_models"] == []
|
||||||
|
assert submitted.json()["priority"] == {
|
||||||
|
"class": "recorded",
|
||||||
|
"rank": 100,
|
||||||
|
"server_owned": True,
|
||||||
|
}
|
||||||
|
assert binding.admit_count == 1
|
||||||
|
|
||||||
|
listed = client.get(
|
||||||
|
"/api/v1/observatory/runs",
|
||||||
|
params={"source_session_id": RAV00_SESSION_ID, "setup_id": SETUP_ID},
|
||||||
|
)
|
||||||
|
assert listed.status_code == 200
|
||||||
|
assert listed.json()["schema_version"] == (
|
||||||
|
"missioncore.observatory-recorded-job-list/v1"
|
||||||
|
)
|
||||||
|
assert [item["job_id"] for item in listed.json()["items"]] == [
|
||||||
|
submitted.json()["job_id"]
|
||||||
|
]
|
||||||
|
assert queue.admission_gate().blocked is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_recorded_run_routes_fail_closed_when_queue_initialization_failed() -> None:
|
||||||
|
registry = LaboratorySetupRegistry.from_file(
|
||||||
|
REGISTRY_PATH,
|
||||||
|
repository_root=REPOSITORY_ROOT,
|
||||||
|
)
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(
|
||||||
|
build_observatory_router(
|
||||||
|
_Store(), # type: ignore[arg-type]
|
||||||
|
setup_registry=registry,
|
||||||
|
recorded_job_queue_error="queue database is incompatible",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
response = client.get(
|
||||||
|
"/api/v1/observatory/runs",
|
||||||
|
params={"source_session_id": RAV00_SESSION_ID, "setup_id": SETUP_ID},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 503
|
||||||
|
assert response.json()["detail"] == "Durable-очередь расчётов недоступна."
|
||||||
@@ -127,6 +127,7 @@ def test_repository_setup_registry_keeps_real_definition_and_pre_definition_resu
|
|||||||
]
|
]
|
||||||
rav00 = registry.catalog(_source(RAV00_SESSION_ID, "RAVNOVES00"))
|
rav00 = registry.catalog(_source(RAV00_SESSION_ID, "RAVNOVES00"))
|
||||||
m49, current = rav00["setups"]
|
m49, current = rav00["setups"]
|
||||||
|
assert m49["display_name"] == "M4.9T5 · TRAVEL TGS · CPU-only, без ML"
|
||||||
assert m49["origin"] == "archived-definition"
|
assert m49["origin"] == "archived-definition"
|
||||||
assert m49["compatibility"]["compatible"] is True
|
assert m49["compatibility"]["compatible"] is True
|
||||||
assert m49["preflight"] == {
|
assert m49["preflight"] == {
|
||||||
@@ -153,6 +154,8 @@ def test_repository_setup_registry_keeps_real_definition_and_pre_definition_resu
|
|||||||
available_observatory_result_ids=frozenset({RAV004_RESULT_ID}),
|
available_observatory_result_ids=frozenset({RAV004_RESULT_ID}),
|
||||||
)
|
)
|
||||||
existing = rav004["setups"][1]
|
existing = rav004["setups"][1]
|
||||||
|
assert existing["display_name"] == "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39"
|
||||||
|
assert "полный TGS и независимый YOLOX отсутствуют" in existing["description"]
|
||||||
assert existing["origin"] == "existing-result"
|
assert existing["origin"] == "existing-result"
|
||||||
assert existing["run_definition"] is None
|
assert existing["run_definition"] is None
|
||||||
assert existing["preflight"]["outcome"] == "existing"
|
assert existing["preflight"]["outcome"] == "existing"
|
||||||
|
|||||||
Reference in New Issue
Block a user