feat(observatory): add portable LAB V1 foundation
This commit is contained in:
@@ -1,16 +1,34 @@
|
||||
import { preflightObservatoryPortableLaboratorySetup } from "./portableLaboratorySetups";
|
||||
|
||||
export { fetchObservatoryPortableLaboratorySetups } from "./portableLaboratorySetups";
|
||||
|
||||
const CATALOG_SCHEMA = "missioncore.observatory-laboratory-setup-catalog/v1";
|
||||
const PREFLIGHT_REQUEST_SCHEMA = "missioncore.observatory-run-preflight-request/v1";
|
||||
const PREFLIGHT_SCHEMA = "missioncore.observatory-run-preflight/v1";
|
||||
const SHA256 = /^[a-f0-9]{64}$/;
|
||||
|
||||
export type ObservatoryLaboratorySetupOrigin = "archived-definition" | "existing-result";
|
||||
export type ObservatoryLaboratorySetupAction = "open-existing" | "open-legacy" | "blocked";
|
||||
export type ObservatoryLaboratorySetupOrigin =
|
||||
| "archived-definition"
|
||||
| "existing-result"
|
||||
| "portable-definition";
|
||||
export type ObservatoryLaboratorySetupAction =
|
||||
| "open-existing"
|
||||
| "open-legacy"
|
||||
| "blocked";
|
||||
|
||||
export interface ObservatoryLaboratoryRunDefinition {
|
||||
readonly definitionId: string;
|
||||
readonly version: number;
|
||||
readonly workId: string;
|
||||
readonly workId: string | null;
|
||||
readonly definitionSha256: string;
|
||||
readonly resultSchema: string | null;
|
||||
readonly resultKind: string | null;
|
||||
readonly models: readonly {
|
||||
readonly name: string;
|
||||
readonly releaseId: string;
|
||||
readonly modelId: string;
|
||||
readonly architecture: string;
|
||||
}[];
|
||||
readonly configuration: readonly {
|
||||
readonly role: string;
|
||||
readonly sha256: string;
|
||||
@@ -38,7 +56,7 @@ export interface ObservatoryLaboratorySetup {
|
||||
};
|
||||
readonly executor: {
|
||||
readonly contourId: string;
|
||||
readonly state: "not-installed";
|
||||
readonly state: "not-installed" | "ready";
|
||||
readonly reasonCode: string;
|
||||
readonly reason: string;
|
||||
};
|
||||
@@ -47,7 +65,7 @@ export interface ObservatoryLaboratorySetup {
|
||||
readonly outcome: "existing" | "blocked";
|
||||
readonly action: ObservatoryLaboratorySetupAction;
|
||||
readonly reason: string;
|
||||
readonly submissionAllowed: false;
|
||||
readonly submissionAllowed: boolean;
|
||||
readonly existingResultIds: readonly string[];
|
||||
};
|
||||
}
|
||||
@@ -124,6 +142,9 @@ export async function preflightObservatoryLaboratorySetup(
|
||||
fetcher?: ObservatoryLaboratorySetupFetch;
|
||||
} = {},
|
||||
): Promise<ObservatoryLaboratoryRunPreflight> {
|
||||
if (setup.origin === "portable-definition") {
|
||||
return preflightObservatoryPortableLaboratorySetup(sourceSessionId, setup);
|
||||
}
|
||||
const response = await request(
|
||||
fetcher,
|
||||
"/api/v1/observatory/run-preflights",
|
||||
@@ -245,6 +266,9 @@ function decodeRunDefinition(value: unknown): ObservatoryLaboratoryRunDefinition
|
||||
version: positiveInteger(row.version, "definition version"),
|
||||
workId: text(row.work_id, "work_id"),
|
||||
definitionSha256: digest,
|
||||
resultSchema: null,
|
||||
resultKind: null,
|
||||
models: [],
|
||||
configuration: array(row.configuration, "configuration").map((item) => {
|
||||
const reference = record(item, "configuration reference");
|
||||
exactKeys(reference, ["role", "sha256"], "configuration reference");
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
import type {
|
||||
ObservatoryLaboratoryRunDefinition,
|
||||
ObservatoryLaboratorySetup,
|
||||
ObservatoryLaboratorySetupCatalog,
|
||||
} from "./laboratorySetups";
|
||||
|
||||
const PORTABLE_CATALOG_SCHEMA = "missioncore.observatory-portable-setup-catalog/v2";
|
||||
const SHA256 = /^[a-f0-9]{64}$/;
|
||||
|
||||
export class ObservatoryPortableSetupDecodeError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "ObservatoryPortableSetupDecodeError";
|
||||
}
|
||||
}
|
||||
|
||||
export function decodePortableCatalog(value: unknown): ObservatoryLaboratorySetupCatalog {
|
||||
const row = record(value, "portable-каталог сетапов");
|
||||
exactKeys(
|
||||
row,
|
||||
["authority", "schema_version", "setups", "source_session_id"],
|
||||
"portable-каталог сетапов",
|
||||
);
|
||||
exact(row.schema_version, PORTABLE_CATALOG_SCHEMA, "schema_version portable-каталога");
|
||||
observationAuthority(row.authority);
|
||||
return {
|
||||
sourceSessionId: text(row.source_session_id, "portable source_session_id"),
|
||||
setups: array(row.setups, "portable setups").map(decodePortableSetup),
|
||||
};
|
||||
}
|
||||
|
||||
function decodePortableSetup(value: unknown): ObservatoryLaboratorySetup {
|
||||
const row = record(value, "portable-сетап");
|
||||
exactKeys(row, [
|
||||
"authority", "description", "display_name", "executor", "existing_results",
|
||||
"origin", "preflight", "run_definition", "setup_id", "source_compatibility",
|
||||
"source_requirements",
|
||||
], "portable-сетап");
|
||||
exact(row.origin, "portable-definition", "portable origin");
|
||||
observationAuthority(row.authority);
|
||||
decodePortableSourceRequirements(row.source_requirements);
|
||||
|
||||
const compatibility = record(row.source_compatibility, "portable source_compatibility");
|
||||
const compatibilityKeys = compatibility.evidence === undefined
|
||||
? ["compatible", "outcome", "reason"]
|
||||
: ["compatible", "evidence", "outcome", "reason"];
|
||||
exactKeys(compatibility, compatibilityKeys, "portable source_compatibility");
|
||||
const compatible = boolean(compatibility.compatible, "portable compatible");
|
||||
exact(
|
||||
compatibility.outcome,
|
||||
compatible ? "pass" : "blocked",
|
||||
"portable compatibility outcome",
|
||||
);
|
||||
const compatibilityReason = text(compatibility.reason, "portable compatibility reason");
|
||||
if (compatibility.evidence !== undefined) {
|
||||
const evidence = record(compatibility.evidence, "portable compatibility evidence");
|
||||
exactKeys(evidence, [
|
||||
"frame_count", "timeline_end_seconds", "timeline_start_seconds",
|
||||
], "portable compatibility evidence");
|
||||
positiveInteger(evidence.frame_count, "portable frame_count");
|
||||
finiteNumber(evidence.timeline_start_seconds, "portable timeline_start_seconds");
|
||||
finiteNumber(evidence.timeline_end_seconds, "portable timeline_end_seconds");
|
||||
}
|
||||
|
||||
const executor = record(row.executor, "portable executor");
|
||||
exactKeys(executor, ["contour_id", "ready", "reason", "state"], "portable executor");
|
||||
const executorState = oneOf(
|
||||
executor.state,
|
||||
["not-installed", "ready"] as const,
|
||||
"portable executor state",
|
||||
);
|
||||
const executorReady = boolean(executor.ready, "portable executor ready");
|
||||
if (executorReady !== (executorState === "ready")) {
|
||||
throw new ObservatoryPortableSetupDecodeError(
|
||||
"Portable executor: состояние готовности противоречиво.",
|
||||
);
|
||||
}
|
||||
const executorReason = executor.reason === null
|
||||
? "Исполнитель установлен."
|
||||
: text(executor.reason, "portable executor reason");
|
||||
|
||||
const preflight = record(row.preflight, "portable preflight");
|
||||
exactKeys(preflight, [
|
||||
"action", "existing_result_ids", "outcome", "reason", "submission_allowed",
|
||||
], "portable preflight");
|
||||
exact(preflight.outcome, "blocked", "portable preflight outcome");
|
||||
exact(preflight.action, "blocked", "portable preflight action");
|
||||
const submissionAllowed = boolean(
|
||||
preflight.submission_allowed,
|
||||
"portable preflight submission_allowed",
|
||||
);
|
||||
if (submissionAllowed) {
|
||||
throw new ObservatoryPortableSetupDecodeError(
|
||||
"Portable preflight: постановка в очередь ещё не поддерживается.",
|
||||
);
|
||||
}
|
||||
const existingResults = array(row.existing_results, "portable existing_results");
|
||||
const existingResultIds = array(
|
||||
preflight.existing_result_ids,
|
||||
"portable existing_result_ids",
|
||||
);
|
||||
if (existingResults.length > 0 || existingResultIds.length > 0) {
|
||||
throw new ObservatoryPortableSetupDecodeError(
|
||||
"Portable result: проверяемая привязка результата к RunDefinition ещё не поддерживается.",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
setupId: text(row.setup_id, "portable setup_id"),
|
||||
displayName: text(row.display_name, "portable display_name"),
|
||||
description: text(row.description, "portable description"),
|
||||
origin: "portable-definition",
|
||||
runDefinition: decodePortableRunDefinition(row.run_definition),
|
||||
compatibility: {
|
||||
compatible,
|
||||
reasons: compatible
|
||||
? []
|
||||
: [{ code: "source-capability-blocked", message: compatibilityReason }],
|
||||
},
|
||||
executor: {
|
||||
contourId: text(executor.contour_id, "portable executor contour_id"),
|
||||
state: executorState,
|
||||
reasonCode: executorReady
|
||||
? "portable-executor-ready"
|
||||
: "portable-executor-not-installed",
|
||||
reason: executorReason,
|
||||
},
|
||||
preservedResults: [],
|
||||
preflight: {
|
||||
outcome: "blocked",
|
||||
action: "blocked",
|
||||
reason: text(preflight.reason, "portable preflight reason"),
|
||||
submissionAllowed,
|
||||
existingResultIds: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function decodePortableSourceRequirements(value: unknown): void {
|
||||
const row = record(value, "portable source_requirements");
|
||||
exactKeys(row, [
|
||||
"archive_id", "calibration_identity_sha256", "calibration_slot", "camera_height",
|
||||
"camera_semantic_channel_id", "camera_source_id", "camera_width",
|
||||
"exactly_one_media_epoch", "plugin_id", "recorded_media_init_sha256",
|
||||
"recorded_media_type", "required_modalities", "seekable",
|
||||
], "portable source_requirements");
|
||||
for (const key of [
|
||||
"archive_id", "calibration_slot", "camera_semantic_channel_id", "camera_source_id",
|
||||
"plugin_id", "recorded_media_type",
|
||||
]) text(row[key], `portable source_requirements.${key}`);
|
||||
for (const key of ["calibration_identity_sha256", "recorded_media_init_sha256"]) {
|
||||
const digest = text(row[key], `portable source_requirements.${key}`);
|
||||
if (!SHA256.test(digest)) {
|
||||
throw new ObservatoryPortableSetupDecodeError(
|
||||
`portable source_requirements.${key}: некорректный digest.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
positiveInteger(row.camera_width, "portable camera_width");
|
||||
positiveInteger(row.camera_height, "portable camera_height");
|
||||
exact(row.exactly_one_media_epoch, true, "portable exactly_one_media_epoch");
|
||||
exact(row.seekable, true, "portable seekable");
|
||||
array(row.required_modalities, "portable required_modalities")
|
||||
.forEach((item) => text(item, "portable required modality"));
|
||||
}
|
||||
|
||||
function decodePortableRunDefinition(value: unknown): ObservatoryLaboratoryRunDefinition {
|
||||
const row = record(value, "portable RunDefinition");
|
||||
exactKeys(row, [
|
||||
"definition_id", "definition_sha256", "models", "result_kind", "result_schema",
|
||||
"version",
|
||||
], "portable RunDefinition");
|
||||
const digest = text(row.definition_sha256, "portable definition_sha256");
|
||||
if (!SHA256.test(digest)) {
|
||||
throw new ObservatoryPortableSetupDecodeError(
|
||||
"Portable RunDefinition: некорректный digest.",
|
||||
);
|
||||
}
|
||||
return {
|
||||
definitionId: text(row.definition_id, "portable definition_id"),
|
||||
version: positiveInteger(row.version, "portable definition version"),
|
||||
workId: null,
|
||||
definitionSha256: digest,
|
||||
resultSchema: text(row.result_schema, "portable result_schema"),
|
||||
resultKind: text(row.result_kind, "portable result_kind"),
|
||||
models: array(row.models, "portable models").map((item) => {
|
||||
const model = record(item, "portable model");
|
||||
exactKeys(
|
||||
model,
|
||||
["architecture", "model_id", "name", "release_id"],
|
||||
"portable model",
|
||||
);
|
||||
return {
|
||||
name: text(model.name, "portable model name"),
|
||||
releaseId: text(model.release_id, "portable model release_id"),
|
||||
modelId: text(model.model_id, "portable model model_id"),
|
||||
architecture: text(model.architecture, "portable model architecture"),
|
||||
};
|
||||
}),
|
||||
configuration: [],
|
||||
};
|
||||
}
|
||||
|
||||
function observationAuthority(value: unknown): void {
|
||||
const row = record(value, "authority");
|
||||
const keys = [
|
||||
"commands_enabled",
|
||||
"actuation_allowed",
|
||||
"navigation_or_safety_accepted",
|
||||
"production_accepted",
|
||||
];
|
||||
exactKeys(row, keys, "authority");
|
||||
for (const key of keys) exact(row[key], false, `authority.${key}`);
|
||||
}
|
||||
|
||||
function record(value: unknown, label: string): Record<string, unknown> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
throw new ObservatoryPortableSetupDecodeError(
|
||||
`${label}: ожидался объект.`,
|
||||
);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function array(value: unknown, label: string): unknown[] {
|
||||
if (!Array.isArray(value)) {
|
||||
throw new ObservatoryPortableSetupDecodeError(
|
||||
`${label}: ожидался массив.`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function text(value: unknown, label: string): string {
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
throw new ObservatoryPortableSetupDecodeError(
|
||||
`${label}: ожидался текст.`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function boolean(value: unknown, label: string): boolean {
|
||||
if (typeof value !== "boolean") {
|
||||
throw new ObservatoryPortableSetupDecodeError(
|
||||
`${label}: ожидался boolean.`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function positiveInteger(value: unknown, label: string): number {
|
||||
if (!Number.isInteger(value) || Number(value) < 1) {
|
||||
throw new ObservatoryPortableSetupDecodeError(
|
||||
`${label}: ожидалось положительное целое.`,
|
||||
);
|
||||
}
|
||||
return Number(value);
|
||||
}
|
||||
|
||||
function finiteNumber(value: unknown, label: string): number {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) {
|
||||
throw new ObservatoryPortableSetupDecodeError(
|
||||
`${label}: ожидалось конечное число.`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function exact<T>(value: unknown, expected: T, label: string): T {
|
||||
if (value !== expected) {
|
||||
throw new ObservatoryPortableSetupDecodeError(
|
||||
`${label}: значение изменилось.`,
|
||||
);
|
||||
}
|
||||
return expected;
|
||||
}
|
||||
|
||||
function oneOf<const T extends readonly string[]>(
|
||||
value: unknown,
|
||||
allowed: T,
|
||||
label: string,
|
||||
): T[number] {
|
||||
if (typeof value !== "string" || !allowed.includes(value)) {
|
||||
throw new ObservatoryPortableSetupDecodeError(
|
||||
`${label}: значение не поддерживается.`,
|
||||
);
|
||||
}
|
||||
return value as T[number];
|
||||
}
|
||||
|
||||
function exactKeys(
|
||||
value: Record<string, unknown>,
|
||||
expected: readonly string[],
|
||||
label: string,
|
||||
): void {
|
||||
const actual = Object.keys(value).sort();
|
||||
const sortedExpected = [...expected].sort();
|
||||
if (
|
||||
actual.length !== sortedExpected.length
|
||||
|| actual.some((key, index) => key !== sortedExpected[index])
|
||||
) {
|
||||
throw new ObservatoryPortableSetupDecodeError(
|
||||
`${label}: обнаружены неизвестные поля.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
import { decodePortableCatalog } from "./portableLaboratorySetupDecoder";
|
||||
import type {
|
||||
ObservatoryLaboratoryRunPreflight,
|
||||
ObservatoryLaboratorySetup,
|
||||
ObservatoryLaboratorySetupCatalog,
|
||||
} from "./laboratorySetups";
|
||||
|
||||
export type ObservatoryPortableLaboratorySetupFetch = (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>;
|
||||
|
||||
class ObservatoryPortableLaboratorySetupContractError extends Error {
|
||||
readonly status: number | null;
|
||||
|
||||
constructor(message: string, status: number | null = null) {
|
||||
super(message);
|
||||
this.name = "ObservatoryPortableLaboratorySetupContractError";
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchObservatoryPortableLaboratorySetups(
|
||||
sourceSessionId: string,
|
||||
{
|
||||
signal,
|
||||
fetcher = globalThis.fetch,
|
||||
}: {
|
||||
signal?: AbortSignal;
|
||||
fetcher?: ObservatoryPortableLaboratorySetupFetch;
|
||||
} = {},
|
||||
): Promise<ObservatoryLaboratorySetupCatalog> {
|
||||
const response = await request(
|
||||
fetcher,
|
||||
`/api/v1/observatory/portable-laboratory-setups?source_session_id=${encodeURIComponent(sourceSessionId)}`,
|
||||
{ method: "GET", headers: { Accept: "application/json" }, signal },
|
||||
);
|
||||
const body = await responseBody(response);
|
||||
if (!response.ok) throw apiError(body, response.status);
|
||||
const catalog = decodePortableCatalog(body);
|
||||
if (catalog.sourceSessionId !== sourceSessionId) {
|
||||
throw new ObservatoryPortableLaboratorySetupContractError(
|
||||
"Portable-каталог сетапов относится к другой исходной сессии.",
|
||||
);
|
||||
}
|
||||
return catalog;
|
||||
}
|
||||
|
||||
export function preflightObservatoryPortableLaboratorySetup(
|
||||
sourceSessionId: string,
|
||||
setup: ObservatoryLaboratorySetup,
|
||||
): ObservatoryLaboratoryRunPreflight {
|
||||
const definitionSha256 = setup.runDefinition?.definitionSha256 ?? null;
|
||||
if (setup.origin !== "portable-definition" || definitionSha256 === null) {
|
||||
throw new ObservatoryPortableLaboratorySetupContractError(
|
||||
"Portable-сетап не содержит RunDefinition.",
|
||||
);
|
||||
}
|
||||
if (
|
||||
setup.preflight.outcome !== "blocked"
|
||||
|| setup.preflight.action !== "blocked"
|
||||
|| setup.preflight.existingResultIds.length > 0
|
||||
|| setup.preservedResults.length > 0
|
||||
) {
|
||||
throw new ObservatoryPortableLaboratorySetupContractError(
|
||||
"Portable-result ещё не имеет проверяемой привязки к RunDefinition.",
|
||||
);
|
||||
}
|
||||
return {
|
||||
sourceSessionId,
|
||||
setupId: setup.setupId,
|
||||
definitionSha256,
|
||||
outcome: "blocked",
|
||||
submissionAllowed: false,
|
||||
checks: [
|
||||
{
|
||||
checkId: "source-compatibility",
|
||||
outcome: setup.compatibility.compatible ? "pass" : "fail",
|
||||
reasonCode: setup.compatibility.compatible
|
||||
? "source-capability-admitted"
|
||||
: "source-capability-blocked",
|
||||
message: setup.compatibility.compatible
|
||||
? "Запись соответствует portable-профилю LAB V1."
|
||||
: setup.compatibility.reasons[0]?.message
|
||||
?? "Запись не соответствует portable-профилю LAB V1.",
|
||||
},
|
||||
{
|
||||
checkId: "executor",
|
||||
outcome: setup.executor.state === "ready" ? "pass" : "fail",
|
||||
reasonCode: setup.executor.reasonCode,
|
||||
message: setup.executor.reason,
|
||||
},
|
||||
{
|
||||
checkId: "durable-queue",
|
||||
outcome: "fail",
|
||||
reasonCode: "portable-dispatch-unavailable",
|
||||
message: setup.preflight.reason,
|
||||
},
|
||||
],
|
||||
existingResultIds: [],
|
||||
};
|
||||
}
|
||||
|
||||
async function request(
|
||||
fetcher: ObservatoryPortableLaboratorySetupFetch,
|
||||
input: string,
|
||||
init: RequestInit,
|
||||
): Promise<Response> {
|
||||
try {
|
||||
return await fetcher(input, init);
|
||||
} catch (error) {
|
||||
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
||||
throw new ObservatoryPortableLaboratorySetupContractError(
|
||||
"Portable-каталог сетапов недоступен.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function responseBody(response: Response): Promise<unknown> {
|
||||
const textBody = await response.text();
|
||||
if (!textBody) return undefined;
|
||||
try {
|
||||
return JSON.parse(textBody) as unknown;
|
||||
} catch {
|
||||
return textBody;
|
||||
}
|
||||
}
|
||||
|
||||
function apiError(
|
||||
body: unknown,
|
||||
status: number,
|
||||
): ObservatoryPortableLaboratorySetupContractError {
|
||||
const detail = body && typeof body === "object" && !Array.isArray(body)
|
||||
? (body as Record<string, unknown>).detail
|
||||
: null;
|
||||
return new ObservatoryPortableLaboratorySetupContractError(
|
||||
typeof detail === "string" && detail.trim()
|
||||
? detail
|
||||
: `Observatory API вернул HTTP ${status}.`,
|
||||
status,
|
||||
);
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
|
||||
import {
|
||||
fetchObservatoryLaboratorySetups,
|
||||
fetchObservatoryPortableLaboratorySetups,
|
||||
preflightObservatoryLaboratorySetup,
|
||||
type ObservatoryLaboratoryRunPreflight,
|
||||
type ObservatoryLaboratorySetupCatalog,
|
||||
@@ -41,22 +42,35 @@ export function useObservatoryLaboratorySetups(sourceSessionId: string) {
|
||||
setState((current) => activeCatalog && current !== "idle" ? "refreshing" : "loading");
|
||||
setError(null);
|
||||
setPreflight({ kind: "idle" });
|
||||
const portableResult = fetchObservatoryPortableLaboratorySetups(
|
||||
sourceSessionId,
|
||||
{ signal: request.signal },
|
||||
).then(
|
||||
(value) => ({ status: "fulfilled" as const, value }),
|
||||
(reason: unknown) => ({ status: "rejected" as const, reason }),
|
||||
);
|
||||
void fetchObservatoryLaboratorySetups(sourceSessionId, { signal: request.signal })
|
||||
.then((next) => {
|
||||
.then(async (legacyCatalog) => {
|
||||
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
||||
setCatalog(next);
|
||||
publishSetupCatalog(legacyCatalog, setCatalog, setSelectedSetupId);
|
||||
setState("ready");
|
||||
setSelectedSetupId((current) => {
|
||||
if (next.setups.some(
|
||||
(setup) => setup.setupId === current && setup.compatibility.compatible,
|
||||
)) return current;
|
||||
return next.setups.find(
|
||||
(setup) => setup.compatibility.compatible && setup.preflight.outcome === "existing",
|
||||
)?.setupId
|
||||
?? next.setups.find((setup) => setup.compatibility.compatible)?.setupId
|
||||
?? next.setups[0]?.setupId
|
||||
?? "";
|
||||
});
|
||||
const optionalPortable = await portableResult;
|
||||
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
||||
if (optionalPortable.status === "fulfilled") {
|
||||
publishSetupCatalog(
|
||||
mergeSetupCatalogs(legacyCatalog, optionalPortable.value),
|
||||
setCatalog,
|
||||
setSelectedSetupId,
|
||||
);
|
||||
setError(null);
|
||||
return;
|
||||
}
|
||||
setError(
|
||||
optionalPortable.reason instanceof Error
|
||||
&& optionalPortable.reason.message.trim()
|
||||
? optionalPortable.reason.message
|
||||
: "Portable-каталог LAB V1 недоступен.",
|
||||
);
|
||||
})
|
||||
.catch((caught: unknown) => {
|
||||
if (request.signal.aborted || requestSequence.current !== sequence) return;
|
||||
@@ -129,6 +143,39 @@ export function useObservatoryLaboratorySetups(sourceSessionId: string) {
|
||||
};
|
||||
}
|
||||
|
||||
function publishSetupCatalog(
|
||||
next: ObservatoryLaboratorySetupCatalog,
|
||||
setCatalog: (catalog: ObservatoryLaboratorySetupCatalog) => void,
|
||||
setSelectedSetupId: (update: (current: string) => string) => void,
|
||||
): void {
|
||||
setCatalog(next);
|
||||
setSelectedSetupId((current) => {
|
||||
if (next.setups.some(
|
||||
(setup) => setup.setupId === current && setup.compatibility.compatible,
|
||||
)) return current;
|
||||
return next.setups.find(
|
||||
(setup) => setup.compatibility.compatible && setup.preflight.outcome === "existing",
|
||||
)?.setupId
|
||||
?? next.setups.find((setup) => setup.compatibility.compatible)?.setupId
|
||||
?? next.setups[0]?.setupId
|
||||
?? "";
|
||||
});
|
||||
}
|
||||
|
||||
function mergeSetupCatalogs(
|
||||
legacy: ObservatoryLaboratorySetupCatalog,
|
||||
portable: ObservatoryLaboratorySetupCatalog,
|
||||
): ObservatoryLaboratorySetupCatalog {
|
||||
if (legacy.sourceSessionId !== portable.sourceSessionId) {
|
||||
throw new Error("Каталоги сетапов относятся к разным исходным сессиям.");
|
||||
}
|
||||
const setups = [...legacy.setups, ...portable.setups];
|
||||
if (new Set(setups.map((setup) => setup.setupId)).size !== setups.length) {
|
||||
throw new Error("Каталоги сетапов содержат повторяющиеся идентификаторы.");
|
||||
}
|
||||
return { sourceSessionId: legacy.sourceSessionId, setups };
|
||||
}
|
||||
|
||||
export type ObservatoryLaboratorySetupsController = ReturnType<
|
||||
typeof useObservatoryLaboratorySetups
|
||||
>;
|
||||
|
||||
@@ -104,7 +104,7 @@ const recordedJobStatus: Record<
|
||||
label: "Ждём подтверждения остановки",
|
||||
tone: "warning",
|
||||
},
|
||||
succeeded: { label: "Готово", tone: "success" },
|
||||
succeeded: { label: "Вычислено · ждёт публикации", tone: "warning" },
|
||||
failed: { label: "Ошибка расчёта", tone: "danger" },
|
||||
"reconciliation-required": { label: "Нужна сверка", tone: "warning" },
|
||||
};
|
||||
@@ -210,7 +210,13 @@ export function ObservatoryWorkspace({
|
||||
value: setup.setupId,
|
||||
label: setup.displayName,
|
||||
description: setup.compatibility.compatible
|
||||
? setup.origin === "existing-result" ? "Готовый результат" : "Совместимый архивный сетап"
|
||||
? setup.origin === "existing-result"
|
||||
? "Готовый результат"
|
||||
: setup.origin === "portable-definition"
|
||||
? setup.executor.state === "ready"
|
||||
? "Запись совместима · Worker установлен, запуск закрыт"
|
||||
: "Запись совместима · Worker не установлен"
|
||||
: "Совместимый архивный сетап"
|
||||
: "Несовместим с выбранной сессией",
|
||||
})) ?? []
|
||||
), [setupController.catalog]);
|
||||
@@ -489,6 +495,20 @@ export function ObservatoryWorkspace({
|
||||
<StatusBadge tone="neutral">Проверяем сетап</StatusBadge>
|
||||
) : queueStateBusy ? (
|
||||
<StatusBadge tone="neutral">Читаем очередь</StatusBadge>
|
||||
) : setupController.selectedSetup?.origin === "portable-definition"
|
||||
&& runPreflight ? (
|
||||
<StatusBadge
|
||||
tone={setupController.selectedSetup.compatibility.compatible
|
||||
? "warning"
|
||||
: "danger"}
|
||||
title={setupController.selectedSetup.preflight.reason}
|
||||
>
|
||||
{setupController.selectedSetup.compatibility.compatible
|
||||
? setupController.selectedSetup.executor.state === "not-installed"
|
||||
? "Worker LAB V1 не установлен"
|
||||
: "Запуск LAB V1 недоступен"
|
||||
: "Запись несовместима"}
|
||||
</StatusBadge>
|
||||
) : null}
|
||||
{canSubmitRecordedJob ? (
|
||||
<Button
|
||||
|
||||
@@ -5,6 +5,7 @@ import { createServer } from "vite";
|
||||
|
||||
let server;
|
||||
let fetchObservatoryLaboratorySetups;
|
||||
let fetchObservatoryPortableLaboratorySetups;
|
||||
let preflightObservatoryLaboratorySetup;
|
||||
let ObservatoryLaboratorySetupContractError;
|
||||
|
||||
@@ -70,6 +71,76 @@ function setup() {
|
||||
};
|
||||
}
|
||||
|
||||
function portableSetup() {
|
||||
return {
|
||||
setup_id: "lab-v1-eomt-ddrnet-portable-v1",
|
||||
display_name: "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||
description: "Проверка записанной K1-сессии моделями EoMT и DDRNet; только наблюдение.",
|
||||
origin: "portable-definition",
|
||||
source_requirements: {
|
||||
plugin_id: "nodedc.device.xgrids-lixelkity-k1",
|
||||
archive_id: "xgrids-k1.viewer-live.evidence",
|
||||
required_modalities: ["point-cloud", "trajectory", "video"],
|
||||
camera_source_id: "sensor.camera.right",
|
||||
camera_semantic_channel_id: "camera.video.recorded",
|
||||
recorded_media_type: 'video/mp4; codecs="avc1.641028"',
|
||||
recorded_media_init_sha256: "1".repeat(64),
|
||||
camera_width: 800,
|
||||
camera_height: 600,
|
||||
calibration_slot: "camera_1",
|
||||
calibration_identity_sha256: "2".repeat(64),
|
||||
exactly_one_media_epoch: true,
|
||||
seekable: true,
|
||||
},
|
||||
run_definition: {
|
||||
definition_id: "lab-v1-eomt-ddrnet-portable",
|
||||
version: 1,
|
||||
definition_sha256: "3".repeat(64),
|
||||
result_schema: "missioncore.recorded-eomt-ddrnet-review/v2",
|
||||
result_kind: "recorded-perception-qualification",
|
||||
models: [
|
||||
{
|
||||
name: "EoMT Cityscapes Large 1024",
|
||||
release_id: "eomt-cityscapes-large-1024-v1",
|
||||
model_id: "tue-mps/cityscapes_semantic_eomt_large_1024",
|
||||
architecture: "EomtForUniversalSegmentation",
|
||||
},
|
||||
{
|
||||
name: "DDRNet-39",
|
||||
release_id: "lab-v1-ddrnet-39-goose-fine-64-v1",
|
||||
model_id: "goose-ddrnet-class-512",
|
||||
architecture: "ddrnet_39",
|
||||
},
|
||||
],
|
||||
},
|
||||
source_compatibility: {
|
||||
outcome: "pass",
|
||||
compatible: true,
|
||||
reason: "Запись соответствует требованиям EoMT + DDRNet.",
|
||||
evidence: {
|
||||
frame_count: 6830,
|
||||
timeline_start_seconds: 0,
|
||||
timeline_end_seconds: 819,
|
||||
},
|
||||
},
|
||||
executor: {
|
||||
contour_id: "worker-006",
|
||||
state: "not-installed",
|
||||
ready: false,
|
||||
reason: "Immutable executor release не установлен.",
|
||||
},
|
||||
existing_results: [],
|
||||
preflight: {
|
||||
outcome: "blocked",
|
||||
action: "blocked",
|
||||
reason: "Вычислительный контур LAB V1 пока недоступен.",
|
||||
submission_allowed: false,
|
||||
existing_result_ids: [],
|
||||
},
|
||||
authority,
|
||||
};
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
server = await createServer({
|
||||
appType: "custom",
|
||||
@@ -78,6 +149,7 @@ before(async () => {
|
||||
});
|
||||
({
|
||||
fetchObservatoryLaboratorySetups,
|
||||
fetchObservatoryPortableLaboratorySetups,
|
||||
preflightObservatoryLaboratorySetup,
|
||||
ObservatoryLaboratorySetupContractError,
|
||||
} = await server.ssrLoadModule("/src/core/observatory/laboratorySetups.ts"));
|
||||
@@ -111,6 +183,104 @@ test("Observatory setup catalog keeps definition identity separate from executor
|
||||
assert.equal(calls[0].init.method, "GET");
|
||||
});
|
||||
|
||||
test("portable LAB V1 reports compatible source separately from unavailable Worker", async () => {
|
||||
const calls = [];
|
||||
const selected = (await fetchObservatoryPortableLaboratorySetups("source-a", {
|
||||
fetcher: async (input, init) => {
|
||||
calls.push({ input: String(input), init });
|
||||
return new Response(JSON.stringify({
|
||||
schema_version: "missioncore.observatory-portable-setup-catalog/v2",
|
||||
source_session_id: "source-a",
|
||||
setups: [portableSetup()],
|
||||
authority,
|
||||
}), { status: 200 });
|
||||
},
|
||||
})).setups[0];
|
||||
|
||||
assert.equal(selected.origin, "portable-definition");
|
||||
assert.equal(selected.compatibility.compatible, true);
|
||||
assert.equal(selected.executor.state, "not-installed");
|
||||
assert.equal(selected.preflight.submissionAllowed, false);
|
||||
assert.deepEqual(selected.runDefinition.models.map((model) => model.name), [
|
||||
"EoMT Cityscapes Large 1024",
|
||||
"DDRNet-39",
|
||||
]);
|
||||
assert.equal(
|
||||
calls[0].input,
|
||||
"/api/v1/observatory/portable-laboratory-setups?source_session_id=source-a",
|
||||
);
|
||||
|
||||
let unexpectedNetworkCall = false;
|
||||
const preflight = await preflightObservatoryLaboratorySetup("source-a", selected, {
|
||||
fetcher: async () => {
|
||||
unexpectedNetworkCall = true;
|
||||
throw new Error("portable preflight must use its server projection");
|
||||
},
|
||||
});
|
||||
assert.equal(unexpectedNetworkCall, false);
|
||||
assert.equal(preflight.outcome, "blocked");
|
||||
assert.equal(preflight.submissionAllowed, false);
|
||||
assert.equal(preflight.checks[0].outcome, "pass");
|
||||
assert.equal(preflight.checks[1].outcome, "fail");
|
||||
});
|
||||
|
||||
test("portable LAB V1 rejects a premature enqueue projection", async () => {
|
||||
const premature = portableSetup();
|
||||
premature.executor = {
|
||||
contour_id: "worker-006",
|
||||
state: "ready",
|
||||
ready: true,
|
||||
reason: null,
|
||||
};
|
||||
premature.preflight = {
|
||||
outcome: "ready",
|
||||
action: "enqueue",
|
||||
reason: "Запись готова к постановке в очередь.",
|
||||
submission_allowed: true,
|
||||
existing_result_ids: [],
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
fetchObservatoryPortableLaboratorySetups("source-a", {
|
||||
fetcher: async () => new Response(JSON.stringify({
|
||||
schema_version: "missioncore.observatory-portable-setup-catalog/v2",
|
||||
source_session_id: "source-a",
|
||||
setups: [premature],
|
||||
authority,
|
||||
}), { status: 200 }),
|
||||
}),
|
||||
/значение изменилось|значение не поддерживается|постановка в очередь ещё не поддерживается/,
|
||||
);
|
||||
});
|
||||
|
||||
test("portable LAB V1 rejects an unbound existing result projection", async () => {
|
||||
const spoofed = portableSetup();
|
||||
spoofed.existing_results = [{
|
||||
result_id: `legacy-vegetation-${"4".repeat(64)}`,
|
||||
result_kind: "recorded-perception-qualification",
|
||||
created_at_utc: "2026-08-30T18:16:00Z",
|
||||
}];
|
||||
spoofed.preflight = {
|
||||
outcome: "existing",
|
||||
action: "open-existing",
|
||||
reason: "Результат якобы готов.",
|
||||
submission_allowed: false,
|
||||
existing_result_ids: [spoofed.existing_results[0].result_id],
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
fetchObservatoryPortableLaboratorySetups("source-a", {
|
||||
fetcher: async () => new Response(JSON.stringify({
|
||||
schema_version: "missioncore.observatory-portable-setup-catalog/v2",
|
||||
source_session_id: "source-a",
|
||||
setups: [spoofed],
|
||||
authority,
|
||||
}), { status: 200 }),
|
||||
}),
|
||||
/значение изменилось|проверяемая привязка результата/,
|
||||
);
|
||||
});
|
||||
|
||||
test("Observatory preflight sends the exact selected definition and never submits a run", async () => {
|
||||
const selected = (await fetchObservatoryLaboratorySetups("source-a", {
|
||||
fetcher: async () => new Response(JSON.stringify({
|
||||
|
||||
@@ -239,7 +239,10 @@ test("Observatory keeps one compact selector axis without the obsolete setup det
|
||||
workspace,
|
||||
/"preemption-pending": \{[\s\S]*label: "Ждём подтверждения остановки"/,
|
||||
);
|
||||
assert.match(workspace, /succeeded: \{ label: "Готово"/);
|
||||
assert.doesNotMatch(setupHook, /Promise\.allSettled/);
|
||||
assert.match(setupHook, /publishSetupCatalog\(legacyCatalog/);
|
||||
assert.match(workspace, /Worker установлен, запуск закрыт/);
|
||||
assert.match(workspace, /succeeded: \{ label: "Вычислено · ждёт публикации"/);
|
||||
assert.match(workspace, /failed: \{ label: "Ошибка расчёта"/);
|
||||
assert.match(workspace, /"reconciliation-required": \{ label: "Нужна сверка"/);
|
||||
assert.match(
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
{
|
||||
"schema_version": "missioncore.observatory-portable-run-definition-registry/v2",
|
||||
"definitions": [
|
||||
{
|
||||
"setup_id": "lab-v1-eomt-ddrnet-portable-v1",
|
||||
"definition_id": "lab-v1-eomt-ddrnet-portable",
|
||||
"version": 1,
|
||||
"definition_sha256": "57bf8f0859e10e54e30322c9a8aa28b427699f6fe6b5267e279ec3390fa78466",
|
||||
"source_requirements": {
|
||||
"plugin_id": "nodedc.device.xgrids-lixelkity-k1",
|
||||
"archive_id": "xgrids-k1.viewer-live.evidence",
|
||||
"required_modalities": [
|
||||
"point-cloud",
|
||||
"trajectory",
|
||||
"video"
|
||||
],
|
||||
"camera_source_id": "sensor.camera.right",
|
||||
"camera_semantic_channel_id": "camera.video.recorded",
|
||||
"recorded_media_type": "video/mp4; codecs=\"avc1.641028\"",
|
||||
"recorded_media_init_sha256": "e2279963e16d84c91d68e7dbb1f7efed840533387dfeb844b7398bff45fbde38",
|
||||
"camera_width": 800,
|
||||
"camera_height": 600,
|
||||
"calibration_slot": "camera_1",
|
||||
"calibration_identity_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9",
|
||||
"exactly_one_media_epoch": true,
|
||||
"seekable": true
|
||||
},
|
||||
"source_adapter": {
|
||||
"adapter_id": "xgrids-k1-recorded-observatory-v2",
|
||||
"version": 2,
|
||||
"contract_sha256": "4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de"
|
||||
},
|
||||
"components": [
|
||||
{
|
||||
"component_id": "ddrnet-full-route-runtime-config-v1",
|
||||
"kind": "configuration",
|
||||
"sha256": "ec7464c2818a707c79aadaa6494625b76d45fb4fe59b74b5208ff9bbd7c9006a"
|
||||
},
|
||||
{
|
||||
"component_id": "eomt-recorded-dependency-set-v1",
|
||||
"kind": "dependency-set",
|
||||
"sha256": "4eb1f8d33236806e74f9e5bb96b7dce2ac37623dc39b2184be2aa8d7d00e983e"
|
||||
},
|
||||
{
|
||||
"component_id": "eomt-recorded-orchestrator-v1",
|
||||
"kind": "orchestrator",
|
||||
"sha256": "d3e9435939444ab35b27a744ac314e289ebd66a13fa56e3d59f121e088d22774"
|
||||
},
|
||||
{
|
||||
"component_id": "eomt-recorded-profile-v1",
|
||||
"kind": "profile",
|
||||
"sha256": "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875"
|
||||
},
|
||||
{
|
||||
"component_id": "eomt-recorded-runner-v1",
|
||||
"kind": "runner",
|
||||
"sha256": "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4"
|
||||
},
|
||||
{
|
||||
"component_id": "k1-camera-1-calibration-v1",
|
||||
"kind": "calibration",
|
||||
"sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
},
|
||||
{
|
||||
"component_id": "k1-valid-fov-identity-v1",
|
||||
"kind": "valid-fov-identity",
|
||||
"sha256": "b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2"
|
||||
},
|
||||
{
|
||||
"component_id": "k1-valid-fov-mask-v1",
|
||||
"kind": "valid-fov-mask",
|
||||
"sha256": "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63"
|
||||
},
|
||||
{
|
||||
"component_id": "vegetation-mission-policy-v1",
|
||||
"kind": "policy",
|
||||
"sha256": "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35"
|
||||
},
|
||||
{
|
||||
"component_id": "vegetation-provider-label-map-v1",
|
||||
"kind": "provider-map",
|
||||
"sha256": "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352"
|
||||
}
|
||||
],
|
||||
"models": [
|
||||
{
|
||||
"release_id": "eomt-cityscapes-large-1024-v1",
|
||||
"model_id": "tue-mps/cityscapes_semantic_eomt_large_1024",
|
||||
"revision": "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f",
|
||||
"architecture": "EomtForUniversalSegmentation",
|
||||
"artifacts": [
|
||||
{
|
||||
"role": "config-json",
|
||||
"byte_length": 1575,
|
||||
"sha256": "7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650"
|
||||
},
|
||||
{
|
||||
"role": "model-weights",
|
||||
"byte_length": 1276175488,
|
||||
"sha256": "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782"
|
||||
},
|
||||
{
|
||||
"role": "preprocessor-config",
|
||||
"byte_length": 666,
|
||||
"sha256": "97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"release_id": "lab-v1-ddrnet-39-goose-fine-64-v1",
|
||||
"model_id": "goose-ddrnet-class-512",
|
||||
"revision": null,
|
||||
"architecture": "ddrnet_39",
|
||||
"artifacts": [
|
||||
{
|
||||
"role": "checkpoint",
|
||||
"byte_length": 259419077,
|
||||
"sha256": "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"resource_profile": {
|
||||
"schema_version": "missioncore.observatory-portable-resource-profile/v2",
|
||||
"profile_id": "worker006-single-gpu-sequential-ai-v1",
|
||||
"contour_id": "worker-006",
|
||||
"accelerator_id": "nvidia-rtx-4090",
|
||||
"concurrency": 1,
|
||||
"checkpoint_policy": "non-checkpointable",
|
||||
"allowed_checkpoints": [],
|
||||
"profile_sha256": "7468138cad115210eda18e7a3350e3423d3d39ac0aae650b3266cdc2bc63fd5d"
|
||||
},
|
||||
"result_contract": {
|
||||
"schema_version": "missioncore.observatory-portable-result-contract/v2",
|
||||
"contract_id": "recorded-eomt-ddrnet-review-v2",
|
||||
"version": 2,
|
||||
"result_schema": "missioncore.recorded-eomt-ddrnet-review/v2",
|
||||
"result_kind": "recorded-perception-qualification",
|
||||
"publication": "observatory",
|
||||
"contract_sha256": "b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a"
|
||||
},
|
||||
"executor": {
|
||||
"contour_id": "worker-006",
|
||||
"state": "not-installed",
|
||||
"release_id": null,
|
||||
"release_sha256": null,
|
||||
"image_sha256": null,
|
||||
"reason_code": "eomt-executor-release-unsealed",
|
||||
"reason": "Immutable EoMT plus DDRNet executor release and image are not sealed or installed on Worker 006."
|
||||
},
|
||||
"authority": {
|
||||
"commands_enabled": false,
|
||||
"actuation_allowed": false,
|
||||
"navigation_or_safety_accepted": false,
|
||||
"production_accepted": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -300,16 +300,29 @@ branches rather than hidden retries. The durable queue serializes one recorded
|
||||
compute owner and does not equate a queued receipt with a Worker execution
|
||||
receipt.
|
||||
|
||||
The only currently queueable pair is the exact `RAVNOVES00` source and
|
||||
`M4.9T5 · TRAVEL TGS · CPU-only, без ML` RunDefinition. It seals an explicitly
|
||||
empty learned-model list because TGS is an algorithmic CPU pipeline, not an
|
||||
unknown model dependency. The binding pins the exact source pack rather than a
|
||||
volatile whole-catalog digest; the server captures and seals the current catalog
|
||||
snapshot into each admitted job. `LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39`
|
||||
remains a result-only replay: no independently versioned durable executor exists
|
||||
for that published result, so Mission Core does not invent one. Every other
|
||||
session/setup pair fails closed until its own source adapter, capability manifest,
|
||||
RunDefinition and executor identities are admitted.
|
||||
The only pair currently admitted to the durable queue is the exact `RAVNOVES00`
|
||||
source and `M4.9T5 · TRAVEL TGS · CPU-only, без ML` RunDefinition. It seals an
|
||||
explicitly empty learned-model list because TGS is an algorithmic CPU pipeline,
|
||||
not an unknown model dependency. The binding pins the exact source pack rather
|
||||
than a volatile whole-catalog digest; the server captures and seals the current
|
||||
catalog snapshot into each admitted job.
|
||||
|
||||
`LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39` now has a source-independent
|
||||
portable RunDefinition and lightweight recorded-source capability probe. The
|
||||
definition seals the exact model/component/resource identities, observation-only
|
||||
authority and generic `missioncore.recorded-eomt-ddrnet-review/v2` result
|
||||
contract; it contains no source Session id or label. Compatibility is derived
|
||||
from the selected Session's real K1 capabilities rather than its name. A
|
||||
compatible source may therefore report capability `pass` independently from
|
||||
executor readiness.
|
||||
|
||||
That portable foundation is not an executable product path yet. Its executor is
|
||||
`not-installed`, and there is no accepted server-side definition-SHA/check-SHA
|
||||
fenced check/submit API, generic v2 result assembler/publisher or deployed Worker
|
||||
executor. Preflight consequently remains blocked and the UI must not promise or
|
||||
expose enqueue. The old `missioncore.lab-v1-vegetation-shadow/v1` result is not an
|
||||
exact/existing result of the generic portable definition, even for its original
|
||||
source; it remains available only in the immutable legacy LAB catalog.
|
||||
|
||||
Recorded work has priority rank `100`. A future live K1 lease has rank `0` and
|
||||
closes new recorded claims while it is pending or active. Cooperative executors
|
||||
@@ -323,11 +336,14 @@ pending for retry; a conflicting or indeterminate receipt enters reconciliation.
|
||||
Every unresolved form blocks live activation and concurrent ownership.
|
||||
An explicit live terminal trigger requeues paused recorded work.
|
||||
|
||||
ADR 0046 records this decision. The durable queue and identity contracts are
|
||||
implemented; Worker claim transport, installation of the exact M4.9T5 executor
|
||||
and wiring from the real K1 lifecycle to live-lease triggers remain pending.
|
||||
Therefore a submitted M4.9T5 job may honestly wait in `queued` without implying
|
||||
that Worker 006 can execute it yet.
|
||||
ADR 0046 records this decision. Durable queue and identity contracts, the
|
||||
authenticated Worker pull protocol and transport-agnostic Worker agent exist as
|
||||
foundation. The production app hard-disables the Worker router even when a valid
|
||||
credential is present until an expiring claim lease and verified result publisher
|
||||
are accepted. Installation of the exact executors, Worker deployment and wiring
|
||||
from the real K1 lifecycle to live-lease triggers remain pending. Therefore a
|
||||
submitted M4.9T5 job may honestly wait in `queued` without implying that Worker
|
||||
006 can execute it yet; portable LAB V1 cannot currently be submitted at all.
|
||||
|
||||
Worker telemetry remains secondary observation evidence. It does not replace the
|
||||
authoritative queue ledger, result validation or common laboratory receipt. K1
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
# ADR 0046: Durable Observatory recorded queue and live K1 priority
|
||||
|
||||
Date: 2026-08-31
|
||||
Status: accepted; durable queue core implemented; Worker and live-trigger wiring pending
|
||||
Status: accepted; portable LAB V1 admission and Worker pull foundations implemented;
|
||||
production executor, claim lease, result publication, Worker deployment and
|
||||
live-trigger wiring pending
|
||||
|
||||
## Context
|
||||
|
||||
@@ -51,11 +53,24 @@ The same key with changed identity is a conflict. Jobs advance through `accepted
|
||||
`paused`, `preemption-pending` and `reconciliation-required` expose safety-relevant
|
||||
branches. Mission Core admits only one active recorded compute owner.
|
||||
|
||||
The queue is an orchestration and evidence boundary, not a Worker transport. A
|
||||
queued job proves durable admission, not remote execution or eventual success.
|
||||
Worker claims, terminal result publication and telemetry must later use a separate
|
||||
authenticated, versioned service boundary; no arbitrary PowerShell or filesystem
|
||||
payload is added to this contract.
|
||||
The queue remains the authoritative orchestration and evidence boundary. A queued
|
||||
job proves durable admission, not remote execution or eventual success. An
|
||||
authenticated, versioned Worker pull protocol is now implemented as a separate
|
||||
boundary. The configured Worker contour may claim one server-sealed job and
|
||||
advance it through start, checkpoint, succeed or fail receipts. The caller cannot
|
||||
supply commands, paths, environment variables, container images, model identities,
|
||||
resource policy or priority. This protocol and its tests are foundation only.
|
||||
The production app hard-disables the Worker router even when a valid private
|
||||
credential exists. A token must not turn an incomplete transport into a production
|
||||
execution surface before an expiring claim lease and verified result publisher
|
||||
are implemented and accepted. The gate remains isolated from K1, Simulation and
|
||||
legacy LAB startup.
|
||||
|
||||
The transport receipt is not result publication. A Worker `succeed` transition
|
||||
records the returned result id and SHA-256 against the claim, but a separate
|
||||
publisher must still validate the generic result contract, immutable artifacts,
|
||||
source/job/definition/model identities and observation-only authority before the
|
||||
result enters the Observatory catalog.
|
||||
|
||||
## Current setup admission
|
||||
|
||||
@@ -74,12 +89,41 @@ binding deliberately does not pin a digest of the whole mutable catalog: the
|
||||
exact compute input is the immutable source pack, while the catalog snapshot is
|
||||
job-specific provenance.
|
||||
|
||||
`LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39` remains result-only. The
|
||||
published evidence stays viewable, but the setup cannot be queued until a new
|
||||
independently versioned source adapter, RunDefinition and executable release are
|
||||
created and accepted. The same fail-closed rule applies to every other
|
||||
session/setup combination. The product may grow toward any admitted setup over
|
||||
any compatible recorded source, but compatibility is evidence, not an assumption.
|
||||
`LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39` now has a source-independent
|
||||
portable RunDefinition. Its identity contains the exact EoMT and DDRNet model
|
||||
artifacts, algorithm/configuration components, resource profile, observation-only
|
||||
authority and the generic `missioncore.recorded-eomt-ddrnet-review/v2` result
|
||||
contract. It contains no source session id, source label, host path, command,
|
||||
environment or caller-selected priority. The old
|
||||
`missioncore.lab-v1-vegetation-shadow/v1` result is not an exact or existing
|
||||
result of this generic v2 definition, including for its original RAV004 source.
|
||||
It remains available only through the immutable legacy LAB catalog; the portable
|
||||
catalog neither maps nor projects it.
|
||||
|
||||
Compatibility is decided by capability admission against the immutable selected
|
||||
Session, not by `RAVNOVES*` naming. The source must satisfy the admitted K1
|
||||
plugin/archive, point-cloud/trajectory/video modalities, RIGHT-camera semantic
|
||||
channel, exact recorded media profile, single seekable media epoch and sealed
|
||||
calibration identity. Catalog compatibility uses a lightweight `probe` result
|
||||
containing only the selected Session/catalog identity, source-adapter identity and
|
||||
camera segment count. It neither prepares/restores a recorded-media sidecar nor
|
||||
builds a source bundle or capability manifest. A later explicit admission performs
|
||||
the full immutable read, emits content-addressed path-free documents and rejects
|
||||
source or definition drift before writing contracts or a queue intent.
|
||||
|
||||
This portable definition is intentionally not executable yet. Its executor state
|
||||
is `not-installed`; no executor release or image identity is sealed. Conversion to
|
||||
the durable queue definition therefore fails before source persistence or queue
|
||||
mutation. A compatible recording, including RAV004, has only a successful
|
||||
source-capability probe, not an existing portable result or runnable LAB V1 job.
|
||||
The legacy RAV004 result remains independently viewable in the legacy catalog.
|
||||
The same fail-closed rule applies to every other session/setup combination.
|
||||
|
||||
The implemented Worker 006 agent core accepts only path-free queue projections,
|
||||
validates their sealed identities and resolves execution through a local
|
||||
four-digest allowlist: executor release, image, model manifest and resource
|
||||
profile. Network polling cadence, model/runtime installation and deployment are
|
||||
outside that core and are not inferred from its presence in the repository.
|
||||
|
||||
## Priority and preemption protocol
|
||||
|
||||
@@ -110,22 +154,43 @@ zero; partial staging never becomes evidence.
|
||||
## Acceptance boundary
|
||||
|
||||
This decision accepts the durable queue core and its identity, lifecycle,
|
||||
idempotency and preemption contracts. It does not claim that the complete compute
|
||||
loop is deployed. The following remain implementation gates:
|
||||
idempotency and preemption contracts; the source-independent LAB V1
|
||||
RunDefinition; capability-based recorded-source admission; the fail-closed bridge
|
||||
from portable identities to the durable queue; the authenticated Worker pull
|
||||
protocol; and the transport-agnostic Worker agent core. It does not claim that the
|
||||
complete compute loop is deployed. The following remain implementation gates:
|
||||
|
||||
- authenticated Worker claim and result transport;
|
||||
- sealing and installing a production LAB V1 executor release and container image
|
||||
with the exact EoMT/DDRNet artifacts on Worker 006;
|
||||
- implementing and accepting the generic v2 result assembler, validator and
|
||||
idempotent Observatory publisher rather than trusting a terminal Worker receipt;
|
||||
- replacing the current one-shot claim receipt with an expiring, renewable claim
|
||||
lease whose loss prevents stale execution and terminal acknowledgement;
|
||||
- deploying, credentialing and physically validating the Worker pull agent and
|
||||
its local four-digest executor allowlist;
|
||||
- installation of the exact M4.9T5 executor on the Worker;
|
||||
- binding real K1 acquisition start/terminal events to live-lease triggers;
|
||||
- physical acceptance that cancellation releases the required Worker resources
|
||||
and that recorded work resumes only after the live lease terminates.
|
||||
|
||||
Until those gates close, an M4.9T5 submission may remain honestly `queued`. LAB V1
|
||||
cannot be submitted, and no UI state may present it as executable.
|
||||
cannot be submitted. The UI must project source compatibility independently from
|
||||
executor readiness: a compatible recording may show capability `pass` while the
|
||||
`not-installed` executor keeps preflight blocked. The portable v2 setup always
|
||||
reports no existing result; its historical vegetation result stays in legacy LAB.
|
||||
Even a future `ready` executor cannot make this projector expose enqueue until
|
||||
server-side definition-SHA and check-SHA fenced check/submit endpoints are
|
||||
implemented and accepted. No current UI state may promise or expose LAB V1
|
||||
enqueue.
|
||||
|
||||
## Consequences
|
||||
|
||||
- The UI stays a bounded selector and submitter; executable authority remains on
|
||||
the server.
|
||||
- The UI stays a bounded selector and read-only preflight surface; executable
|
||||
authority remains on the server, and LAB V1 enqueue stays absent while its
|
||||
executor is `not-installed`.
|
||||
- Source compatibility and executor readiness are separate product facts; neither
|
||||
labels nor an existing result upgrades a compatible Session into an executable
|
||||
definition.
|
||||
- Every accepted job is reproducible from sealed source, setup, executor, model
|
||||
and resource identities rather than mutable host paths.
|
||||
- Live K1 can receive hard priority without silently losing or concurrently
|
||||
@@ -133,6 +198,7 @@ cannot be submitted, and no UI state may present it as executable.
|
||||
- Non-checkpointable replay pays the deliberate cost of discard and
|
||||
restart-from-zero after preemption.
|
||||
- K1 acquisition/control principles, the legacy LAB archive and
|
||||
Simulation/Gaussian remain unchanged.
|
||||
Simulation/Gaussian remain unchanged; this foundation adds no viewer profile,
|
||||
data migration, equipment command or lifecycle coupling to those contours.
|
||||
- Adding a setup or source requires an admitted adapter and RunDefinition; a
|
||||
display name or historical result is insufficient.
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
"""Portable recorded-source admission bound to the durable Observatory queue.
|
||||
|
||||
The binding resolves a server-owned portable RunDefinition before inspecting a
|
||||
session. It therefore cannot write source contracts or jobs for an unavailable
|
||||
executor. A successful check is fenced by a content digest; admission repeats
|
||||
the source read and persists only if the exact checked snapshot still holds.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJob,
|
||||
ObservatoryRecordedJobIntent,
|
||||
ObservatoryRecordedJobQueue,
|
||||
RecordedRunDefinition,
|
||||
)
|
||||
from k1link.observatory.source_admission import (
|
||||
PortableRecordedSourceAdmission,
|
||||
PortableRecordedSourceCapability,
|
||||
PortableSourceAdmissionStaleError,
|
||||
RecordedK1SourceAdmissionService,
|
||||
)
|
||||
from k1link.sessions.media import RecordedMediaInspector
|
||||
from k1link.sessions.store import SessionStore
|
||||
|
||||
PORTABLE_QUEUE_BINDING_SCHEMA: Final = "missioncore.observatory-portable-queue-binding/v1"
|
||||
_SHA256: Final = re.compile(r"^[a-f0-9]{64}$")
|
||||
_IDEMPOTENCY_KEY: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$")
|
||||
_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
|
||||
|
||||
class PortableQueueBindingError(RuntimeError):
|
||||
"""A portable source cannot be bound to the recorded queue."""
|
||||
|
||||
|
||||
class PortableQueueBindingIntegrityError(PortableQueueBindingError):
|
||||
"""A registry, source admission, or queue identity disagrees."""
|
||||
|
||||
|
||||
class PortableQueueBindingStaleCheckError(PortableQueueBindingError):
|
||||
"""The source changed after the operator-visible compatibility check."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableRecordedRunPreparation:
|
||||
"""Path-free immutable bridge from source admission to a queue intent."""
|
||||
|
||||
definition: RecordedRunDefinition
|
||||
source: PortableRecordedSourceAdmission
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.definition.source_adapter_sha256 != self.source.source_adapter_sha256:
|
||||
raise PortableQueueBindingIntegrityError(
|
||||
"portable registry and source admission adapter identities disagree"
|
||||
)
|
||||
|
||||
@property
|
||||
def check_sha256(self) -> str:
|
||||
return _sha256(self.identity_document())
|
||||
|
||||
def identity_document(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": PORTABLE_QUEUE_BINDING_SCHEMA,
|
||||
"source": self.source.as_dict(),
|
||||
"definition": _definition_document(self.definition),
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
def intent(self, *, idempotency_key: str) -> ObservatoryRecordedJobIntent:
|
||||
_pattern(idempotency_key, _IDEMPOTENCY_KEY, "idempotency key")
|
||||
return ObservatoryRecordedJobIntent(
|
||||
idempotency_key=idempotency_key,
|
||||
source_session_id=self.source.source_session_id,
|
||||
source_catalog_sha256=self.source.source_catalog_sha256,
|
||||
source_bundle_sha256=self.source.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=(self.source.source_capability_manifest_sha256),
|
||||
setup_id=self.definition.setup_id,
|
||||
definition_sha256=self.definition.definition_sha256,
|
||||
)
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {**self.identity_document(), "check_sha256": self.check_sha256}
|
||||
|
||||
|
||||
class PortableRecordedQueueBindingService:
|
||||
"""Resolve, check, admit, and submit portable recorded computations."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
data_dir: Path,
|
||||
session_store: SessionStore,
|
||||
media_inspector: RecordedMediaInspector,
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
queue: ObservatoryRecordedJobQueue | None = None,
|
||||
) -> None:
|
||||
self.data_dir = data_dir.expanduser().resolve()
|
||||
if self.data_dir != session_store.data_dir:
|
||||
raise PortableQueueBindingIntegrityError(
|
||||
"portable binding and SessionStore roots disagree"
|
||||
)
|
||||
if queue is not None and self.data_dir != queue.data_dir:
|
||||
raise PortableQueueBindingIntegrityError(
|
||||
"portable binding and recorded queue roots disagree"
|
||||
)
|
||||
self._session_store = session_store
|
||||
self._media_inspector = media_inspector
|
||||
self._definitions = definitions
|
||||
self._queue = queue
|
||||
|
||||
def probe(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> PortableRecordedSourceCapability:
|
||||
"""Return cheap authoritative compatibility without replay preparation."""
|
||||
|
||||
portable, recorded = self._resolve_definition(setup_id, definition_sha256)
|
||||
capability = self._source_service(portable).probe(source_session_id)
|
||||
if recorded.source_adapter_sha256 != capability.source_adapter_sha256:
|
||||
raise PortableQueueBindingIntegrityError(
|
||||
"portable registry and source capability adapter identities disagree"
|
||||
)
|
||||
return capability
|
||||
|
||||
def check(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> PortableRecordedRunPreparation:
|
||||
"""Verify one source/setup combination without persistent writes."""
|
||||
|
||||
portable, recorded = self._resolve_definition(setup_id, definition_sha256)
|
||||
source_service = self._source_service(portable)
|
||||
return self._bind(recorded, source_service.check(source_session_id))
|
||||
|
||||
def admit(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
expected_check_sha256: str,
|
||||
) -> PortableRecordedRunPreparation:
|
||||
"""Persist exact checked contracts and reject changed source state."""
|
||||
|
||||
_digest(expected_check_sha256, "expected portable check sha256")
|
||||
portable, recorded = self._resolve_definition(setup_id, definition_sha256)
|
||||
source_service = self._source_service(portable)
|
||||
checked = self._bind(recorded, source_service.check(source_session_id))
|
||||
if checked.check_sha256 != expected_check_sha256:
|
||||
raise PortableQueueBindingStaleCheckError(
|
||||
"portable source or definition changed after its check"
|
||||
)
|
||||
try:
|
||||
source = source_service.admit(
|
||||
source_session_id,
|
||||
expected_admission_sha256=checked.source.identity_sha256,
|
||||
)
|
||||
except PortableSourceAdmissionStaleError as exc:
|
||||
raise PortableQueueBindingStaleCheckError(
|
||||
"portable source changed while admission was being committed"
|
||||
) from exc
|
||||
admitted = self._bind(recorded, source)
|
||||
if admitted.check_sha256 != expected_check_sha256:
|
||||
raise PortableQueueBindingStaleCheckError(
|
||||
"portable source changed while admission was being committed"
|
||||
)
|
||||
return admitted
|
||||
|
||||
def submit(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
expected_check_sha256: str,
|
||||
idempotency_key: str,
|
||||
enqueue: bool = True,
|
||||
) -> tuple[ObservatoryRecordedJob, bool]:
|
||||
"""Admit and atomically submit the resulting path-free queue intent."""
|
||||
|
||||
if self._queue is None:
|
||||
raise PortableQueueBindingIntegrityError("portable recorded queue is unavailable")
|
||||
preparation = self.admit(
|
||||
source_session_id=source_session_id,
|
||||
setup_id=setup_id,
|
||||
definition_sha256=definition_sha256,
|
||||
expected_check_sha256=expected_check_sha256,
|
||||
)
|
||||
return self._queue.submit(
|
||||
preparation.intent(idempotency_key=idempotency_key),
|
||||
enqueue=enqueue,
|
||||
)
|
||||
|
||||
def _resolve_definition(
|
||||
self,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> tuple[PortableRunDefinition, RecordedRunDefinition]:
|
||||
portable = self._definitions.resolve(setup_id, definition_sha256)
|
||||
# This conversion is deliberately first: not-installed executors fail
|
||||
# before the SessionStore, immutable document store, or queue is touched.
|
||||
recorded = portable.to_recorded_run_definition()
|
||||
if self._queue is not None:
|
||||
try:
|
||||
queue_definition = self._queue.resolve_definition(
|
||||
setup_id,
|
||||
definition_sha256,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise PortableQueueBindingIntegrityError(
|
||||
"portable definition is absent from the recorded queue"
|
||||
) from exc
|
||||
if queue_definition != recorded:
|
||||
raise PortableQueueBindingIntegrityError(
|
||||
"portable registry and recorded queue definitions disagree"
|
||||
)
|
||||
return portable, recorded
|
||||
|
||||
def _source_service(
|
||||
self,
|
||||
definition: PortableRunDefinition,
|
||||
) -> RecordedK1SourceAdmissionService:
|
||||
return RecordedK1SourceAdmissionService(
|
||||
data_dir=self.data_dir,
|
||||
session_store=self._session_store,
|
||||
media_inspector=self._media_inspector,
|
||||
requirements=definition.to_source_admission_requirements(),
|
||||
prepare_media=False,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _bind(
|
||||
definition: RecordedRunDefinition,
|
||||
source: PortableRecordedSourceAdmission,
|
||||
) -> PortableRecordedRunPreparation:
|
||||
return PortableRecordedRunPreparation(definition=definition, source=source)
|
||||
|
||||
|
||||
def _definition_document(definition: RecordedRunDefinition) -> dict[str, object]:
|
||||
return {
|
||||
"setup_id": definition.setup_id,
|
||||
"definition_id": definition.definition_id,
|
||||
"definition_version": definition.definition_version,
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
"source_adapter_id": definition.source_adapter_id,
|
||||
"source_adapter_version": definition.source_adapter_version,
|
||||
"source_adapter_sha256": definition.source_adapter_sha256,
|
||||
"executor_release_id": definition.executor_release_id,
|
||||
"executor_release_sha256": definition.executor_release_sha256,
|
||||
"executor_image_sha256": definition.executor_image_sha256,
|
||||
"model_release_ids": list(definition.model_release_ids),
|
||||
"model_manifest_sha256": definition.model_manifest_sha256,
|
||||
"resource_profile_id": definition.resource_profile_id,
|
||||
"resource_profile_sha256": definition.resource_profile_sha256,
|
||||
"checkpoint_policy": definition.checkpoint_policy,
|
||||
"allowed_checkpoints": list(definition.allowed_checkpoints),
|
||||
}
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _sha256(value: object) -> str:
|
||||
return hashlib.sha256(_canonical_json(value)).hexdigest()
|
||||
|
||||
|
||||
def _digest(value: object, label: str) -> str:
|
||||
return _pattern(value, _SHA256, label)
|
||||
|
||||
|
||||
def _pattern(value: object, pattern: re.Pattern[str], label: str) -> str:
|
||||
if not isinstance(value, str) or pattern.fullmatch(value) is None:
|
||||
raise ValueError(f"{label} is invalid")
|
||||
return value
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,234 @@
|
||||
"""UI-ready Observatory projection for the portable LAB V1 definition.
|
||||
|
||||
This module deliberately does not extend the legacy setup registry and does
|
||||
not submit work. It projects two independent facts for one selected source:
|
||||
|
||||
* the result of a lightweight recorded-source capability probe;
|
||||
* the executor state sealed by the portable RunDefinition;
|
||||
|
||||
Keeping those facts separate prevents a compatible new recording from being
|
||||
described as incompatible merely because the executor is not installed yet.
|
||||
Historical vegetation-shadow results belong only to the legacy catalog and
|
||||
never become exact results of the generic portable v2 definition.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Protocol, runtime_checkable
|
||||
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.source_admission import (
|
||||
PortableRecordedSourceCapability,
|
||||
PortableSourceAdmissionError,
|
||||
)
|
||||
from k1link.sessions.models import SessionSummary
|
||||
|
||||
PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-setup-catalog/v2"
|
||||
)
|
||||
PORTABLE_LAB_V1_SETUP_ID: Final = "lab-v1-eomt-ddrnet-portable-v1"
|
||||
PORTABLE_LAB_V1_DISPLAY_NAME: Final = "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39"
|
||||
|
||||
_SOURCE_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
_OBSERVATION_ONLY_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
|
||||
_MODEL_PRESENTATION: Final = {
|
||||
"eomt-cityscapes-large-1024-v1": "EoMT Cityscapes Large 1024",
|
||||
"lab-v1-ddrnet-39-goose-fine-64-v1": "DDRNet-39",
|
||||
}
|
||||
|
||||
|
||||
class PortableSetupProjectionError(RuntimeError):
|
||||
"""The portable setup cannot be projected without weakening its contract."""
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class PortableSourceCapabilityProbeService(Protocol):
|
||||
"""A definition-bound lightweight source-capability service."""
|
||||
|
||||
def probe(self, source_session_id: str) -> PortableRecordedSourceCapability:
|
||||
"""Probe one source without preparing media or persisting documents."""
|
||||
|
||||
|
||||
type PortableSourceCapabilityProbe = (
|
||||
Callable[[str], PortableRecordedSourceCapability] | PortableSourceCapabilityProbeService
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SourceCompatibility:
|
||||
compatible: bool
|
||||
capability: PortableRecordedSourceCapability | None
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"outcome": "pass" if self.compatible else "blocked",
|
||||
"compatible": self.compatible,
|
||||
"reason": (
|
||||
"Запись соответствует требованиям EoMT + DDRNet."
|
||||
if self.compatible
|
||||
else "Запись не соответствует требованиям EoMT + DDRNet."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class PortableLabV1SetupProjector:
|
||||
"""Project the generic portable LAB V1 setup for one selected source."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
registry: PortableRunDefinitionRegistry,
|
||||
capability_probe: PortableSourceCapabilityProbe,
|
||||
) -> None:
|
||||
self._definition = _resolve_lab_v1_definition(registry)
|
||||
if not isinstance(
|
||||
capability_probe,
|
||||
PortableSourceCapabilityProbeService,
|
||||
) and not callable(capability_probe):
|
||||
raise PortableSetupProjectionError("portable source capability probe is unavailable")
|
||||
self._capability_probe = capability_probe
|
||||
_validate_model_presentation(self._definition)
|
||||
if self._definition.authority.as_dict() != _OBSERVATION_ONLY_AUTHORITY:
|
||||
raise PortableSetupProjectionError("portable LAB V1 authority is not observation-only")
|
||||
|
||||
def catalog(self, source: SessionSummary) -> dict[str, object]:
|
||||
"""Return a one-setup v2 catalog projection for ``source``."""
|
||||
|
||||
return {
|
||||
"schema_version": PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA,
|
||||
"source_session_id": source.session_id,
|
||||
"setups": [self.project(source)],
|
||||
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
|
||||
def project(self, source: SessionSummary) -> dict[str, object]:
|
||||
"""Return a strict, observation-only setup projection."""
|
||||
|
||||
_validate_source_id(source.session_id)
|
||||
compatibility = self._probe_source(source.session_id)
|
||||
definition = self._definition
|
||||
executor = definition.executor
|
||||
return {
|
||||
"setup_id": definition.setup_id,
|
||||
"display_name": PORTABLE_LAB_V1_DISPLAY_NAME,
|
||||
"description": (
|
||||
"Проверка записанной K1-сессии моделями EoMT и DDRNet; только наблюдение."
|
||||
),
|
||||
"origin": "portable-definition",
|
||||
"source_requirements": definition.source_requirements.as_dict(),
|
||||
"run_definition": {
|
||||
"definition_id": definition.definition_id,
|
||||
"version": definition.version,
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
"result_schema": definition.result_contract.result_schema,
|
||||
"result_kind": definition.result_contract.result_kind,
|
||||
"models": _project_models(definition),
|
||||
},
|
||||
"source_compatibility": compatibility.as_dict(),
|
||||
"executor": {
|
||||
"contour_id": executor.contour_id,
|
||||
"state": executor.state,
|
||||
"ready": executor.ready,
|
||||
"reason": executor.reason,
|
||||
},
|
||||
"existing_results": [],
|
||||
"preflight": {
|
||||
"outcome": "blocked",
|
||||
"action": "blocked",
|
||||
"reason": self._preflight_reason(compatibility),
|
||||
"submission_allowed": False,
|
||||
"existing_result_ids": [],
|
||||
},
|
||||
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
|
||||
def _probe_source(self, source_session_id: str) -> _SourceCompatibility:
|
||||
try:
|
||||
if isinstance(
|
||||
self._capability_probe,
|
||||
PortableSourceCapabilityProbeService,
|
||||
):
|
||||
capability = self._capability_probe.probe(source_session_id)
|
||||
else:
|
||||
capability = self._capability_probe(source_session_id)
|
||||
except PortableSourceAdmissionError:
|
||||
return _SourceCompatibility(compatible=False, capability=None)
|
||||
if not isinstance(capability, PortableRecordedSourceCapability):
|
||||
raise PortableSetupProjectionError("capability probe returned an invalid result")
|
||||
if capability.source_session_id != source_session_id:
|
||||
raise PortableSetupProjectionError(
|
||||
"capability probe is bound to another source session"
|
||||
)
|
||||
if capability.source_adapter_sha256 != self._definition.source_adapter.contract_sha256:
|
||||
raise PortableSetupProjectionError("capability probe uses another source adapter")
|
||||
return _SourceCompatibility(compatible=True, capability=capability)
|
||||
|
||||
def _preflight_reason(self, compatibility: _SourceCompatibility) -> str:
|
||||
if not compatibility.compatible:
|
||||
return "Запись не соответствует требованиям этого сетапа."
|
||||
if not self._definition.executor.ready:
|
||||
return "Вычислительный контур LAB V1 пока недоступен."
|
||||
return (
|
||||
"Server-side проверка definition/check SHA и постановка portable "
|
||||
"LAB V1 в очередь пока недоступны."
|
||||
)
|
||||
|
||||
|
||||
def _resolve_lab_v1_definition(
|
||||
registry: PortableRunDefinitionRegistry,
|
||||
) -> PortableRunDefinition:
|
||||
matching = tuple(
|
||||
definition
|
||||
for definition in registry.definitions
|
||||
if definition.setup_id == PORTABLE_LAB_V1_SETUP_ID
|
||||
)
|
||||
if len(matching) != 1:
|
||||
raise PortableSetupProjectionError("portable LAB V1 definition is unavailable or ambiguous")
|
||||
return matching[0]
|
||||
|
||||
|
||||
def _validate_model_presentation(definition: PortableRunDefinition) -> None:
|
||||
releases = {model.release_id: model for model in definition.models}
|
||||
if set(releases) != set(_MODEL_PRESENTATION):
|
||||
raise PortableSetupProjectionError(
|
||||
"portable LAB V1 model set does not match its presentation contract"
|
||||
)
|
||||
eomt = releases["eomt-cityscapes-large-1024-v1"]
|
||||
ddrnet = releases["lab-v1-ddrnet-39-goose-fine-64-v1"]
|
||||
if (
|
||||
eomt.model_id != "tue-mps/cityscapes_semantic_eomt_large_1024"
|
||||
or ddrnet.architecture != "ddrnet_39"
|
||||
):
|
||||
raise PortableSetupProjectionError(
|
||||
"portable LAB V1 model identities do not match their presentation"
|
||||
)
|
||||
|
||||
|
||||
def _project_models(definition: PortableRunDefinition) -> list[dict[str, object]]:
|
||||
by_release = {model.release_id: model for model in definition.models}
|
||||
return [
|
||||
{
|
||||
"name": _MODEL_PRESENTATION[release_id],
|
||||
"release_id": release_id,
|
||||
"model_id": by_release[release_id].model_id,
|
||||
"architecture": by_release[release_id].architecture,
|
||||
}
|
||||
for release_id in _MODEL_PRESENTATION
|
||||
]
|
||||
|
||||
|
||||
def _validate_source_id(source_session_id: str) -> None:
|
||||
if _SOURCE_ID.fullmatch(source_session_id) is None:
|
||||
raise PortableSetupProjectionError("source session id is invalid")
|
||||
@@ -34,16 +34,10 @@ from uuid import uuid4
|
||||
from k1link.artifacts import utc_now_iso
|
||||
|
||||
OBSERVATORY_RECORDED_JOB_SCHEMA: Final = "missioncore.observatory-recorded-job/v1"
|
||||
OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA: Final = (
|
||||
"missioncore.observatory-recorded-job-request/v1"
|
||||
)
|
||||
OBSERVATORY_RECORDED_CLAIM_SCHEMA: Final = (
|
||||
"missioncore.observatory-recorded-job-claim/v1"
|
||||
)
|
||||
OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA: Final = "missioncore.observatory-recorded-job-request/v1"
|
||||
OBSERVATORY_RECORDED_CLAIM_SCHEMA: Final = "missioncore.observatory-recorded-job-claim/v1"
|
||||
OBSERVATORY_LIVE_LEASE_SCHEMA: Final = "missioncore.observatory-live-k1-lease/v1"
|
||||
OBSERVATORY_LIVE_LEASE_REQUEST_SCHEMA: Final = (
|
||||
"missioncore.observatory-live-k1-lease-request/v1"
|
||||
)
|
||||
OBSERVATORY_LIVE_LEASE_REQUEST_SCHEMA: Final = "missioncore.observatory-live-k1-lease-request/v1"
|
||||
RECORDED_JOB_DATABASE_NAME: Final = "observatory-recorded-jobs.sqlite3"
|
||||
MAX_RECORDED_JOBS: Final = 10_000
|
||||
MAX_RECORDED_CLAIM_RECEIPTS: Final = 50_000
|
||||
@@ -67,9 +61,7 @@ type RecordedJobState = Literal[
|
||||
"reconciliation-required",
|
||||
]
|
||||
type CheckpointPolicy = Literal["cooperative", "non-checkpointable"]
|
||||
type LiveLeaseState = Literal[
|
||||
"pending", "active", "completed", "failed", "cancelled"
|
||||
]
|
||||
type LiveLeaseState = Literal["pending", "active", "completed", "failed", "cancelled"]
|
||||
type LiveTerminalOutcome = Literal["completed", "failed", "cancelled"]
|
||||
|
||||
_JOB_ID = re.compile(r"^observatory-run-[a-f0-9]{32}$")
|
||||
@@ -80,9 +72,7 @@ _SESSION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
_CHECKPOINT_ID = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_TERMINAL_STATES: Final = frozenset(
|
||||
{"succeeded", "failed", "reconciliation-required"}
|
||||
)
|
||||
_TERMINAL_STATES: Final = frozenset({"succeeded", "failed", "reconciliation-required"})
|
||||
_OPEN_LIVE_STATES: Final = frozenset({"pending", "active"})
|
||||
_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
@@ -299,8 +289,7 @@ class RecordedRunDefinitionRegistry:
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
keys = [
|
||||
(definition.setup_id, definition.definition_sha256)
|
||||
for definition in self.definitions
|
||||
(definition.setup_id, definition.definition_sha256) for definition in self.definitions
|
||||
]
|
||||
version_keys = [
|
||||
(definition.definition_id, definition.definition_version)
|
||||
@@ -358,9 +347,7 @@ class ObservatoryRecordedJobIntent:
|
||||
"source_session_id": self.source_session_id,
|
||||
"source_catalog_sha256": self.source_catalog_sha256,
|
||||
"source_bundle_sha256": self.source_bundle_sha256,
|
||||
"source_capability_manifest_sha256": (
|
||||
self.source_capability_manifest_sha256
|
||||
),
|
||||
"source_capability_manifest_sha256": (self.source_capability_manifest_sha256),
|
||||
"setup_id": self.setup_id,
|
||||
"definition_sha256": self.definition_sha256,
|
||||
}
|
||||
@@ -471,18 +458,12 @@ class ObservatoryRecordedJob:
|
||||
"recorded-job preemption marker is invalid"
|
||||
)
|
||||
if self.claim_generation < 0:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job claim generation is invalid"
|
||||
)
|
||||
raise ObservatoryRecordedQueueIntegrityError("recorded-job claim generation is invalid")
|
||||
_validate_optional_pattern(self.active_claim_token, _TOKEN, "active claim token")
|
||||
_validate_optional_pattern(self.active_claimant_id, _IDENTIFIER, "claimant id")
|
||||
_validate_optional_pattern(
|
||||
self.last_checkpoint_id, _CHECKPOINT_ID, "last checkpoint id"
|
||||
)
|
||||
_validate_optional_pattern(self.last_checkpoint_id, _CHECKPOINT_ID, "last checkpoint id")
|
||||
if not isinstance(self.restart_from_zero, bool):
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job restart marker is invalid"
|
||||
)
|
||||
raise ObservatoryRecordedQueueIntegrityError("recorded-job restart marker is invalid")
|
||||
_validate_optional_pattern(
|
||||
self.preemption_receipt_sha256,
|
||||
_SHA256,
|
||||
@@ -513,9 +494,7 @@ class ObservatoryRecordedJob:
|
||||
"session_id": self.source_session_id,
|
||||
"catalog_sha256": self.source_catalog_sha256,
|
||||
"bundle_sha256": self.source_bundle_sha256,
|
||||
"capability_manifest_sha256": (
|
||||
self.source_capability_manifest_sha256
|
||||
),
|
||||
"capability_manifest_sha256": (self.source_capability_manifest_sha256),
|
||||
"adapter": {
|
||||
"adapter_id": self.source_adapter_id,
|
||||
"version": self.source_adapter_version,
|
||||
@@ -661,9 +640,7 @@ class ObservatoryLiveLease:
|
||||
and self.terminal_request_sha256 is not None
|
||||
and self.terminated_at_utc is not None
|
||||
):
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"live terminal identity is incomplete"
|
||||
)
|
||||
raise ObservatoryRecordedQueueIntegrityError("live terminal identity is incomplete")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
@@ -730,9 +707,7 @@ class ObservatoryNonCheckpointableCancellationRequest:
|
||||
"schema_version": OBSERVATORY_LIVE_LEASE_REQUEST_SCHEMA,
|
||||
"cancellation_request_id": self.cancellation_request_id,
|
||||
"job_id": self.job_id,
|
||||
"claim_token_sha256": hashlib.sha256(
|
||||
self.claim_token.encode()
|
||||
).hexdigest(),
|
||||
"claim_token_sha256": hashlib.sha256(self.claim_token.encode()).hexdigest(),
|
||||
"claim_generation": self.claim_generation,
|
||||
"live_trigger_id": self.live_trigger_id,
|
||||
"executor_release_sha256": self.executor_release_sha256,
|
||||
@@ -781,11 +756,7 @@ class ObservatoryNonCheckpointableCancellationReceipt:
|
||||
_validate_pattern(self.cancellation_id, _IDEMPOTENCY_KEY, "cancellation id")
|
||||
_validate_digest(self.request_sha256, "cancellation request sha256")
|
||||
_validate_digest(self.receipt_sha256, "cancellation receipt sha256")
|
||||
if not (
|
||||
self.resources_released
|
||||
and self.staging_discarded
|
||||
and self.restart_from_zero
|
||||
):
|
||||
if not (self.resources_released and self.staging_discarded and self.restart_from_zero):
|
||||
raise ValueError(
|
||||
"cancellation receipt must release resources, discard staging, "
|
||||
"and require restart from zero"
|
||||
@@ -819,17 +790,13 @@ class ObservatoryRecordedJobQueue:
|
||||
*,
|
||||
definitions: RecordedRunDefinitionRegistry,
|
||||
clock: Callable[[], str] = utc_now_iso,
|
||||
non_checkpointable_preemptor: (
|
||||
ObservatoryNonCheckpointablePreemptor | None
|
||||
) = None,
|
||||
non_checkpointable_preemptor: (ObservatoryNonCheckpointablePreemptor | None) = None,
|
||||
max_jobs: int = MAX_RECORDED_JOBS,
|
||||
max_claim_receipts: int = MAX_RECORDED_CLAIM_RECEIPTS,
|
||||
max_live_leases: int = MAX_LIVE_LEASES,
|
||||
) -> None:
|
||||
_validate_quota(max_jobs, MAX_RECORDED_JOBS, "recorded job")
|
||||
_validate_quota(
|
||||
max_claim_receipts, MAX_RECORDED_CLAIM_RECEIPTS, "claim receipt"
|
||||
)
|
||||
_validate_quota(max_claim_receipts, MAX_RECORDED_CLAIM_RECEIPTS, "claim receipt")
|
||||
_validate_quota(max_live_leases, MAX_LIVE_LEASES, "live lease")
|
||||
self.data_dir = data_dir.expanduser().resolve()
|
||||
self.database_path = self.data_dir / RECORDED_JOB_DATABASE_NAME
|
||||
@@ -842,6 +809,15 @@ class ObservatoryRecordedJobQueue:
|
||||
self._lock = threading.RLock()
|
||||
self._initialize()
|
||||
|
||||
def resolve_definition(
|
||||
self,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> RecordedRunDefinition:
|
||||
"""Expose the immutable server allowlist without exposing queue internals."""
|
||||
|
||||
return self._definitions.resolve(setup_id, definition_sha256)
|
||||
|
||||
def submit(
|
||||
self,
|
||||
intent: ObservatoryRecordedJobIntent,
|
||||
@@ -875,9 +851,7 @@ class ObservatoryRecordedJobQueue:
|
||||
limit=self._max_jobs,
|
||||
label="recorded job",
|
||||
)
|
||||
definition = self._definitions.resolve(
|
||||
intent.setup_id, intent.definition_sha256
|
||||
)
|
||||
definition = self._definitions.resolve(intent.setup_id, intent.definition_sha256)
|
||||
now = self._timestamp()
|
||||
job_id = f"observatory-run-{uuid4().hex}"
|
||||
identity_sha256 = _job_identity_sha256(intent, definition)
|
||||
@@ -973,8 +947,7 @@ class ObservatoryRecordedJobQueue:
|
||||
request_sha256 = _claim_request_sha256(claimant_id, claim_request_id)
|
||||
with self._transaction() as connection:
|
||||
receipt = connection.execute(
|
||||
"SELECT * FROM observatory_recorded_claim_receipts "
|
||||
"WHERE claim_request_id = ?",
|
||||
"SELECT * FROM observatory_recorded_claim_receipts WHERE claim_request_id = ?",
|
||||
(claim_request_id,),
|
||||
).fetchone()
|
||||
if receipt is not None:
|
||||
@@ -1062,11 +1035,7 @@ class ObservatoryRecordedJobQueue:
|
||||
raise ObservatoryRecordedQueueConflictError(
|
||||
f"cannot start recorded job from {job.state}"
|
||||
)
|
||||
state = (
|
||||
"paused"
|
||||
if self._open_live_lease_row(connection) is not None
|
||||
else "running"
|
||||
)
|
||||
state = "paused" if self._open_live_lease_row(connection) is not None else "running"
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs SET state = ?, "
|
||||
"preemption_requested = ?, updated_at_utc = ? WHERE job_id = ?",
|
||||
@@ -1179,14 +1148,11 @@ class ObservatoryRecordedJobQueue:
|
||||
with self._read_connection() as connection:
|
||||
return self._get_job(connection, job_id)
|
||||
|
||||
def get_by_idempotency_key(
|
||||
self, idempotency_key: str
|
||||
) -> ObservatoryRecordedJob:
|
||||
def get_by_idempotency_key(self, idempotency_key: str) -> ObservatoryRecordedJob:
|
||||
_validate_pattern(idempotency_key, _IDEMPOTENCY_KEY, "idempotency key")
|
||||
with self._read_connection() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM observatory_recorded_jobs "
|
||||
"WHERE idempotency_key = ?",
|
||||
"SELECT * FROM observatory_recorded_jobs WHERE idempotency_key = ?",
|
||||
(idempotency_key,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
@@ -1226,9 +1192,7 @@ class ObservatoryRecordedJobQueue:
|
||||
).fetchall()
|
||||
return tuple(_job_from_row(row) for row in rows)
|
||||
|
||||
def request_live(
|
||||
self, intent: ObservatoryLiveLeaseIntent
|
||||
) -> tuple[ObservatoryLiveLease, bool]:
|
||||
def request_live(self, intent: ObservatoryLiveLeaseIntent) -> tuple[ObservatoryLiveLease, bool]:
|
||||
"""Close recorded admission without allowing a monolith to delay live K1."""
|
||||
|
||||
created = False
|
||||
@@ -1357,9 +1321,7 @@ class ObservatoryRecordedJobQueue:
|
||||
"scheduler did not release non-checkpointable replay resources"
|
||||
) from exc
|
||||
if (
|
||||
not isinstance(
|
||||
receipt, ObservatoryNonCheckpointableCancellationReceipt
|
||||
)
|
||||
not isinstance(receipt, ObservatoryNonCheckpointableCancellationReceipt)
|
||||
or receipt.request_sha256 != request.request_sha256
|
||||
):
|
||||
with self._transaction() as connection:
|
||||
@@ -1423,9 +1385,7 @@ class ObservatoryRecordedJobQueue:
|
||||
"""Release live priority only on an exact explicit terminal trigger."""
|
||||
|
||||
_validate_pattern(lease_id, _LEASE_ID, "live lease id")
|
||||
_validate_pattern(
|
||||
terminal_trigger_id, _IDEMPOTENCY_KEY, "live terminal trigger id"
|
||||
)
|
||||
_validate_pattern(terminal_trigger_id, _IDEMPOTENCY_KEY, "live terminal trigger id")
|
||||
if outcome not in ("completed", "failed", "cancelled"):
|
||||
raise ValueError("live terminal outcome is invalid")
|
||||
terminal_request_sha256 = _sha256(
|
||||
@@ -1500,9 +1460,7 @@ class ObservatoryRecordedJobQueue:
|
||||
with self._read_connection() as connection:
|
||||
row = self._open_live_lease_row(connection)
|
||||
if row is None:
|
||||
return ObservatoryRecordedAdmissionGate(
|
||||
blocked=False, lease_id=None, lease_state=None
|
||||
)
|
||||
return ObservatoryRecordedAdmissionGate(blocked=False, lease_id=None, lease_state=None)
|
||||
state = str(row["state"])
|
||||
if state == "pending":
|
||||
lease_state: Literal["pending", "active"] = "pending"
|
||||
@@ -1556,8 +1514,7 @@ class ObservatoryRecordedJobQueue:
|
||||
f"cannot terminate recorded job from {job.state}"
|
||||
)
|
||||
if state == "succeeded" and (
|
||||
job.preemption_requested
|
||||
or self._open_live_lease_row(connection) is not None
|
||||
job.preemption_requested or self._open_live_lease_row(connection) is not None
|
||||
):
|
||||
raise ObservatoryRecordedQueueConflictError(
|
||||
"cannot publish recorded success while live preemption is open"
|
||||
@@ -1599,14 +1556,10 @@ class ObservatoryRecordedJobQueue:
|
||||
job=self._get_job(connection, job_id),
|
||||
)
|
||||
|
||||
def _require_active_claim(
|
||||
self, job: ObservatoryRecordedJob, claim_token: str
|
||||
) -> None:
|
||||
def _require_active_claim(self, job: ObservatoryRecordedJob, claim_token: str) -> None:
|
||||
_validate_pattern(claim_token, _TOKEN, "claim token")
|
||||
if job.active_claim_token != claim_token:
|
||||
raise ObservatoryRecordedQueueStaleClaimError(
|
||||
"recorded-job claim token is stale"
|
||||
)
|
||||
raise ObservatoryRecordedQueueStaleClaimError("recorded-job claim token is stale")
|
||||
|
||||
def _cancellation_request(
|
||||
self,
|
||||
@@ -1642,8 +1595,7 @@ class ObservatoryRecordedJobQueue:
|
||||
) -> None:
|
||||
with self._transaction() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM observatory_recorded_preemptions "
|
||||
"WHERE cancellation_request_id = ?",
|
||||
"SELECT * FROM observatory_recorded_preemptions WHERE cancellation_request_id = ?",
|
||||
(request.cancellation_request_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
@@ -1704,9 +1656,7 @@ class ObservatoryRecordedJobQueue:
|
||||
(receipt.receipt_sha256, now, request.cancellation_request_id),
|
||||
)
|
||||
|
||||
def _get_job(
|
||||
self, connection: sqlite3.Connection, job_id: str
|
||||
) -> ObservatoryRecordedJob:
|
||||
def _get_job(self, connection: sqlite3.Connection, job_id: str) -> ObservatoryRecordedJob:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM observatory_recorded_jobs WHERE job_id = ?", (job_id,)
|
||||
).fetchone()
|
||||
@@ -1724,17 +1674,12 @@ class ObservatoryRecordedJobQueue:
|
||||
raise ObservatoryRecordedQueueNotFoundError(lease_id)
|
||||
return _live_lease_from_row(row)
|
||||
|
||||
def _open_live_lease_row(
|
||||
self, connection: sqlite3.Connection
|
||||
) -> sqlite3.Row | None:
|
||||
def _open_live_lease_row(self, connection: sqlite3.Connection) -> sqlite3.Row | None:
|
||||
rows = connection.execute(
|
||||
"SELECT * FROM observatory_live_leases "
|
||||
"WHERE state IN ('pending', 'active') LIMIT 2"
|
||||
"SELECT * FROM observatory_live_leases WHERE state IN ('pending', 'active') LIMIT 2"
|
||||
).fetchall()
|
||||
if len(rows) > 1:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"multiple live K1 leases are open"
|
||||
)
|
||||
raise ObservatoryRecordedQueueIntegrityError("multiple live K1 leases are open")
|
||||
return None if not rows else rows[0]
|
||||
|
||||
def _require_capacity(
|
||||
@@ -1788,9 +1733,7 @@ class ObservatoryRecordedJobQueue:
|
||||
"SELECT name FROM pragma_table_info(?)", (table,)
|
||||
).fetchall()
|
||||
if len(columns) != column_count:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
f"{table} schema is incompatible"
|
||||
)
|
||||
raise ObservatoryRecordedQueueIntegrityError(f"{table} schema is incompatible")
|
||||
indexes = connection.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'index' "
|
||||
"AND name = 'observatory_one_open_live_lease'"
|
||||
@@ -1816,14 +1759,10 @@ class ObservatoryRecordedJobQueue:
|
||||
),
|
||||
):
|
||||
count = len(
|
||||
connection.execute(
|
||||
f"SELECT 1 FROM {table} LIMIT ?", (limit + 1,)
|
||||
).fetchall()
|
||||
connection.execute(f"SELECT 1 FROM {table} LIMIT ?", (limit + 1,)).fetchall()
|
||||
)
|
||||
if count > limit:
|
||||
raise ObservatoryRecordedQueueCapacityError(
|
||||
f"{label} quota is exceeded"
|
||||
)
|
||||
raise ObservatoryRecordedQueueCapacityError(f"{label} quota is exceeded")
|
||||
|
||||
def _validate_storage_paths(self, *, require_database: bool = False) -> None:
|
||||
paths = (
|
||||
@@ -1834,9 +1773,7 @@ class ObservatoryRecordedJobQueue:
|
||||
total_bytes = 0
|
||||
for path in paths:
|
||||
if path.is_symlink() or (path.exists() and not path.is_file()):
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job queue path is unsafe"
|
||||
)
|
||||
raise ObservatoryRecordedQueueIntegrityError("recorded-job queue path is unsafe")
|
||||
if path.exists():
|
||||
total_bytes += path.stat().st_size
|
||||
if require_database and not self.database_path.is_file():
|
||||
@@ -1874,9 +1811,7 @@ class ObservatoryRecordedJobQueue:
|
||||
except ObservatoryRecordedQueueError:
|
||||
raise
|
||||
except (OSError, sqlite3.Error) as exc:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job queue read failed"
|
||||
) from exc
|
||||
raise ObservatoryRecordedQueueIntegrityError("recorded-job queue read failed") from exc
|
||||
|
||||
@contextmanager
|
||||
def _connect(self) -> Iterator[sqlite3.Connection]:
|
||||
@@ -1889,9 +1824,7 @@ class ObservatoryRecordedJobQueue:
|
||||
try:
|
||||
connection.execute("PRAGMA foreign_keys = ON")
|
||||
connection.execute("PRAGMA synchronous = FULL")
|
||||
connection.execute(
|
||||
f"PRAGMA busy_timeout = {_SQLITE_BUSY_TIMEOUT_MILLISECONDS}"
|
||||
)
|
||||
connection.execute(f"PRAGMA busy_timeout = {_SQLITE_BUSY_TIMEOUT_MILLISECONDS}")
|
||||
connection.execute("PRAGMA journal_mode = WAL")
|
||||
yield connection
|
||||
finally:
|
||||
@@ -1944,9 +1877,7 @@ def _job_from_row(row: sqlite3.Row) -> ObservatoryRecordedJob:
|
||||
source_session_id=row["source_session_id"],
|
||||
source_catalog_sha256=row["source_catalog_sha256"],
|
||||
source_bundle_sha256=row["source_bundle_sha256"],
|
||||
source_capability_manifest_sha256=row[
|
||||
"source_capability_manifest_sha256"
|
||||
],
|
||||
source_capability_manifest_sha256=row["source_capability_manifest_sha256"],
|
||||
setup_id=row["setup_id"],
|
||||
definition_id=row["definition_id"],
|
||||
definition_version=row["definition_version"],
|
||||
@@ -1982,9 +1913,7 @@ def _job_from_row(row: sqlite3.Row) -> ObservatoryRecordedJob:
|
||||
updated_at_utc=row["updated_at_utc"],
|
||||
)
|
||||
except (IndexError, KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"stored recorded job is invalid"
|
||||
) from exc
|
||||
raise ObservatoryRecordedQueueIntegrityError("stored recorded job is invalid") from exc
|
||||
|
||||
|
||||
def _live_lease_from_row(row: sqlite3.Row) -> ObservatoryLiveLease:
|
||||
@@ -2005,9 +1934,7 @@ def _live_lease_from_row(row: sqlite3.Row) -> ObservatoryLiveLease:
|
||||
terminated_at_utc=row["terminated_at_utc"],
|
||||
)
|
||||
except (IndexError, KeyError, TypeError, ValueError) as exc:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"stored live K1 lease is invalid"
|
||||
) from exc
|
||||
raise ObservatoryRecordedQueueIntegrityError("stored live K1 lease is invalid") from exc
|
||||
|
||||
|
||||
def _job_identity_sha256(
|
||||
@@ -2019,9 +1946,7 @@ def _job_identity_sha256(
|
||||
"source_session_id": intent.source_session_id,
|
||||
"source_catalog_sha256": intent.source_catalog_sha256,
|
||||
"source_bundle_sha256": intent.source_bundle_sha256,
|
||||
"source_capability_manifest_sha256": (
|
||||
intent.source_capability_manifest_sha256
|
||||
),
|
||||
"source_capability_manifest_sha256": (intent.source_capability_manifest_sha256),
|
||||
"setup_id": definition.setup_id,
|
||||
"definition_id": definition.definition_id,
|
||||
"definition_version": definition.definition_version,
|
||||
@@ -2100,11 +2025,7 @@ def _sha256(value: object) -> str:
|
||||
|
||||
|
||||
def _validate_quota(value: object, maximum: int, label: str) -> None:
|
||||
if (
|
||||
not isinstance(value, int)
|
||||
or isinstance(value, bool)
|
||||
or not 1 <= value <= maximum
|
||||
):
|
||||
if not isinstance(value, int) or isinstance(value, bool) or not 1 <= value <= maximum:
|
||||
raise ValueError(f"{label} quota is invalid")
|
||||
|
||||
|
||||
@@ -2122,9 +2043,7 @@ def _validate_pattern(value: object, pattern: re.Pattern[str], label: str) -> No
|
||||
raise ValueError(f"{label} is invalid")
|
||||
|
||||
|
||||
def _validate_optional_pattern(
|
||||
value: object | None, pattern: re.Pattern[str], label: str
|
||||
) -> None:
|
||||
def _validate_optional_pattern(value: object | None, pattern: re.Pattern[str], label: str) -> None:
|
||||
if value is not None:
|
||||
_validate_pattern(value, pattern, label)
|
||||
|
||||
@@ -2146,11 +2065,7 @@ def _validate_timestamp(value: object, label: str) -> None:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"{label} is invalid") from exc
|
||||
if (
|
||||
parsed.tzinfo is None
|
||||
or parsed.utcoffset() != timedelta(0)
|
||||
or not value.endswith("Z")
|
||||
):
|
||||
if parsed.tzinfo is None or parsed.utcoffset() != timedelta(0) or not value.endswith("Z"):
|
||||
raise ValueError(f"{label} must use UTC")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,978 @@
|
||||
"""Capability-based admission for portable recorded K1 laboratory runs.
|
||||
|
||||
The portable setup identity is deliberately independent from a session name.
|
||||
This module binds one concrete SessionStore snapshot to an allowlisted K1
|
||||
source profile and emits path-free, content-addressed documents for the Worker
|
||||
boundary. It does not enqueue work or expose host filesystem locations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import stat
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from k1link.sessions.media import (
|
||||
CAMERA_ARCHIVE_SCHEMA,
|
||||
MAX_MEDIA_SUMMARY_BYTES,
|
||||
MAX_SAFE_INTEGER,
|
||||
RecordedMediaEpoch,
|
||||
RecordedMediaInspector,
|
||||
RecordedMediaManifest,
|
||||
)
|
||||
from k1link.sessions.models import (
|
||||
RecordedMediaArtifact,
|
||||
ReplayCommand,
|
||||
SessionArtifact,
|
||||
SessionDetail,
|
||||
SessionSource,
|
||||
)
|
||||
from k1link.sessions.store import SessionStore
|
||||
|
||||
PORTABLE_SOURCE_BUNDLE_SCHEMA: Final = "missioncore.portable-recorded-source-bundle/v1"
|
||||
PORTABLE_SOURCE_CAPABILITY_SCHEMA: Final = "missioncore.portable-recorded-source-capability/v1"
|
||||
PORTABLE_SOURCE_ADAPTER_SCHEMA: Final = "missioncore.portable-source-adapter/v1"
|
||||
PORTABLE_SOURCE_DOCUMENT_DIRECTORY: Final = "observatory-portable-source-contracts"
|
||||
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9.-]{2,127}$")
|
||||
_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
_MEDIA_TYPE = re.compile(r'^[a-z0-9.+-]+/[a-z0-9.+-]+(?:; codecs="[A-Za-z0-9.]+")?$')
|
||||
_EPOCH = re.compile(r"^epoch-([1-9][0-9]*)$")
|
||||
_ALLOWED_MODALITIES: Final = frozenset({"point-cloud", "trajectory", "video"})
|
||||
_SEALED_ARTIFACT_STATES: Final = frozenset({"verified", "validated-structure"})
|
||||
_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
|
||||
|
||||
class PortableSourceAdmissionError(RuntimeError):
|
||||
"""A recorded session does not satisfy the portable source contract."""
|
||||
|
||||
|
||||
class PortableSourceAdmissionIntegrityError(PortableSourceAdmissionError):
|
||||
"""A catalog, media, or persisted source identity changed."""
|
||||
|
||||
|
||||
class PortableSourceNotPreparedError(PortableSourceAdmissionError):
|
||||
"""A compatible source has no previously validated media sidecar."""
|
||||
|
||||
|
||||
class PortableSourceAdmissionStaleError(PortableSourceAdmissionIntegrityError):
|
||||
"""The catalog changed across a checked admission boundary."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RecordedK1SourceRequirements:
|
||||
"""Typed matcher owned by a portable RunDefinition.
|
||||
|
||||
``camera_init_sha256`` is the exact ISO-BMFF initialization segment for the
|
||||
admitted codec/resolution profile. Width and height are asserted by that
|
||||
immutable media-profile attestation; arbitrary init segments are rejected.
|
||||
Calibration remains an external, digest-bound rig profile until recordings
|
||||
carry their own calibration snapshot.
|
||||
"""
|
||||
|
||||
adapter_id: str
|
||||
adapter_version: int
|
||||
plugin_id: str
|
||||
archive_id: str
|
||||
required_modalities: tuple[str, ...]
|
||||
camera_source_id: str
|
||||
camera_semantic_channel_id: str
|
||||
camera_media_type: str
|
||||
camera_init_sha256: str
|
||||
expected_width: int
|
||||
expected_height: int
|
||||
calibration_slot: str
|
||||
calibration_sha256: str
|
||||
require_single_camera_epoch: bool = True
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.adapter_id, _IDENTIFIER, "source adapter id")
|
||||
if (
|
||||
not isinstance(self.adapter_version, int)
|
||||
or isinstance(self.adapter_version, bool)
|
||||
or self.adapter_version < 1
|
||||
):
|
||||
raise ValueError("source adapter version is invalid")
|
||||
_pattern(self.plugin_id, _IDENTIFIER, "device plugin id")
|
||||
_pattern(self.archive_id, _IDENTIFIER, "archive id")
|
||||
if (
|
||||
not self.required_modalities
|
||||
or len(set(self.required_modalities)) != len(self.required_modalities)
|
||||
or not set(self.required_modalities).issubset(_ALLOWED_MODALITIES)
|
||||
or "video" not in self.required_modalities
|
||||
):
|
||||
raise ValueError("portable source modalities are invalid")
|
||||
_pattern(self.camera_source_id, _SOURCE_ID, "camera source id")
|
||||
_pattern(
|
||||
self.camera_semantic_channel_id,
|
||||
_IDENTIFIER,
|
||||
"camera semantic channel id",
|
||||
)
|
||||
_pattern(self.camera_media_type, _MEDIA_TYPE, "camera media type")
|
||||
_digest(self.camera_init_sha256, "camera init sha256")
|
||||
if (
|
||||
not isinstance(self.expected_width, int)
|
||||
or isinstance(self.expected_width, bool)
|
||||
or not isinstance(self.expected_height, int)
|
||||
or isinstance(self.expected_height, bool)
|
||||
or not 1 <= self.expected_width <= 16_384
|
||||
or not 1 <= self.expected_height <= 16_384
|
||||
):
|
||||
raise ValueError("camera dimensions are invalid")
|
||||
_pattern(self.calibration_slot, _SOURCE_ID, "calibration slot")
|
||||
_digest(self.calibration_sha256, "calibration sha256")
|
||||
if self.require_single_camera_epoch is not True:
|
||||
raise ValueError("portable source v1 requires one camera epoch")
|
||||
|
||||
def adapter_document(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": PORTABLE_SOURCE_ADAPTER_SCHEMA,
|
||||
"adapter_id": self.adapter_id,
|
||||
"version": self.adapter_version,
|
||||
"source": {
|
||||
"plugin_id": self.plugin_id,
|
||||
"archive_id": self.archive_id,
|
||||
"required_modalities": list(self.required_modalities),
|
||||
"camera_source_id": self.camera_source_id,
|
||||
"camera_semantic_channel_id": self.camera_semantic_channel_id,
|
||||
},
|
||||
"camera_profile": {
|
||||
"media_type": self.camera_media_type,
|
||||
"init_sha256": self.camera_init_sha256,
|
||||
"width": self.expected_width,
|
||||
"height": self.expected_height,
|
||||
"attestation": "exact-isobmff-init-sha256",
|
||||
},
|
||||
"calibration": {
|
||||
"slot": self.calibration_slot,
|
||||
"sha256": self.calibration_sha256,
|
||||
"binding": "external-rig-profile",
|
||||
},
|
||||
"camera_epoch_policy": "exactly-one-complete-epoch",
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
@property
|
||||
def adapter_sha256(self) -> str:
|
||||
return _sha256(self.adapter_document())
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableRecordedSourceCapability:
|
||||
"""Cheap catalog/summary attestation used by compatibility surfaces.
|
||||
|
||||
This is intentionally not a replay admission. In particular it carries
|
||||
no timeline or content-addressed source bundle because proving those facts
|
||||
requires the prepared replay sidecar and a full archive validation.
|
||||
"""
|
||||
|
||||
source_session_id: str
|
||||
source_catalog_sha256: str
|
||||
source_adapter_sha256: str
|
||||
camera_segment_count: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.source_session_id, _SOURCE_ID, "source session id")
|
||||
_digest(self.source_catalog_sha256, "source catalog sha256")
|
||||
_digest(self.source_adapter_sha256, "source adapter sha256")
|
||||
if (
|
||||
not isinstance(self.camera_segment_count, int)
|
||||
or isinstance(self.camera_segment_count, bool)
|
||||
or not 1 <= self.camera_segment_count <= MAX_SAFE_INTEGER
|
||||
):
|
||||
raise ValueError("camera segment count is invalid")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": "missioncore.portable-recorded-source-probe/v1",
|
||||
"source_session_id": self.source_session_id,
|
||||
"source_catalog_sha256": self.source_catalog_sha256,
|
||||
"source_adapter_sha256": self.source_adapter_sha256,
|
||||
"camera_segment_count": self.camera_segment_count,
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableRecordedSourceAdmission:
|
||||
source_session_id: str
|
||||
source_catalog_sha256: str
|
||||
source_bundle_sha256: str
|
||||
source_capability_manifest_sha256: str
|
||||
source_adapter_sha256: str
|
||||
frame_count: int
|
||||
timeline_start_seconds: float
|
||||
timeline_end_seconds: float
|
||||
camera_generation_sha256: str
|
||||
source_bundle: bytes
|
||||
capability_manifest: bytes
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.source_session_id, _SOURCE_ID, "source session id")
|
||||
for label, value in (
|
||||
("source catalog sha256", self.source_catalog_sha256),
|
||||
("source bundle sha256", self.source_bundle_sha256),
|
||||
(
|
||||
"source capability manifest sha256",
|
||||
self.source_capability_manifest_sha256,
|
||||
),
|
||||
("source adapter sha256", self.source_adapter_sha256),
|
||||
("camera generation sha256", self.camera_generation_sha256),
|
||||
):
|
||||
_digest(value, label)
|
||||
if self.frame_count < 1 or self.timeline_end_seconds <= self.timeline_start_seconds:
|
||||
raise ValueError("portable source timeline is invalid")
|
||||
if hashlib.sha256(self.source_bundle).hexdigest() != self.source_bundle_sha256:
|
||||
raise ValueError("source bundle bytes do not match their identity")
|
||||
if (
|
||||
hashlib.sha256(self.capability_manifest).hexdigest()
|
||||
!= self.source_capability_manifest_sha256
|
||||
):
|
||||
raise ValueError("capability bytes do not match their identity")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": "missioncore.portable-recorded-source-admission/v1",
|
||||
"source_session_id": self.source_session_id,
|
||||
"source_catalog_sha256": self.source_catalog_sha256,
|
||||
"source_bundle_sha256": self.source_bundle_sha256,
|
||||
"source_capability_manifest_sha256": (self.source_capability_manifest_sha256),
|
||||
"source_adapter_sha256": self.source_adapter_sha256,
|
||||
"camera": {
|
||||
"generation_sha256": self.camera_generation_sha256,
|
||||
"frame_count": self.frame_count,
|
||||
"timeline_start_seconds": self.timeline_start_seconds,
|
||||
"timeline_end_seconds": self.timeline_end_seconds,
|
||||
},
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
@property
|
||||
def identity_sha256(self) -> str:
|
||||
"""Content identity used to fence check-to-admit transitions."""
|
||||
|
||||
return _sha256(self.as_dict())
|
||||
|
||||
|
||||
class RecordedK1SourceAdmissionService:
|
||||
"""Bind compatible recorded K1 sessions without trusting their labels."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
data_dir: Path,
|
||||
session_store: SessionStore,
|
||||
media_inspector: RecordedMediaInspector,
|
||||
requirements: RecordedK1SourceRequirements,
|
||||
prepare_media: bool = True,
|
||||
) -> None:
|
||||
self.data_dir = data_dir.expanduser().resolve()
|
||||
if self.data_dir != session_store.data_dir:
|
||||
raise ValueError("portable admission and SessionStore roots disagree")
|
||||
self._session_store = session_store
|
||||
self._media_inspector = media_inspector
|
||||
self._prepare_media = prepare_media
|
||||
self.requirements = requirements
|
||||
self._document_root = self.data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY
|
||||
|
||||
def probe(self, source_session_id: str) -> PortableRecordedSourceCapability:
|
||||
"""Prove cheap setup compatibility without replay/media preparation.
|
||||
|
||||
The probe reads one atomic catalog snapshot, the catalog's single
|
||||
recorded-video handle, and one bounded canonical camera summary. It
|
||||
deliberately never calls ``prepare_replay`` or ``RecordedMediaInspector``
|
||||
and never enters the archive's index or segment directory.
|
||||
"""
|
||||
|
||||
_pattern(source_session_id, _SOURCE_ID, "source session id")
|
||||
try:
|
||||
detail, catalog_sha256 = self._session_store.get_session_with_catalog_snapshot(
|
||||
source_session_id
|
||||
)
|
||||
recorded_media = self._session_store.list_recorded_media(source_session_id)
|
||||
except Exception as exc:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded source capability could not be resolved"
|
||||
) from exc
|
||||
_digest(catalog_sha256, "source catalog sha256")
|
||||
selected_sources = self._verify_catalog(
|
||||
detail,
|
||||
expected_session_id=source_session_id,
|
||||
)
|
||||
media_artifact = self._verify_recorded_media_artifact(
|
||||
detail=detail,
|
||||
camera_source=selected_sources["video"],
|
||||
recorded_media=recorded_media,
|
||||
)
|
||||
segment_count = self._probe_camera_summary(media_artifact)
|
||||
return PortableRecordedSourceCapability(
|
||||
source_session_id=detail.summary.session_id,
|
||||
source_catalog_sha256=catalog_sha256,
|
||||
source_adapter_sha256=self.requirements.adapter_sha256,
|
||||
camera_segment_count=segment_count,
|
||||
)
|
||||
|
||||
def check(self, source_session_id: str) -> PortableRecordedSourceAdmission:
|
||||
return self._prepare(
|
||||
source_session_id,
|
||||
persist=False,
|
||||
prepare_media=False,
|
||||
)
|
||||
|
||||
def admit(
|
||||
self,
|
||||
source_session_id: str,
|
||||
*,
|
||||
expected_admission_sha256: str | None = None,
|
||||
) -> PortableRecordedSourceAdmission:
|
||||
"""Persist only the source identity most recently admitted by a check.
|
||||
|
||||
``expected_admission_sha256`` is optional for existing internal callers.
|
||||
Portable queue binding supplies it so a changed SessionStore snapshot is
|
||||
rejected before either immutable source document is written.
|
||||
"""
|
||||
|
||||
if expected_admission_sha256 is not None:
|
||||
_digest(expected_admission_sha256, "expected source admission sha256")
|
||||
return self._prepare(
|
||||
source_session_id,
|
||||
persist=True,
|
||||
prepare_media=self._prepare_media,
|
||||
expected_admission_sha256=expected_admission_sha256,
|
||||
)
|
||||
|
||||
def _prepare(
|
||||
self,
|
||||
source_session_id: str,
|
||||
*,
|
||||
persist: bool,
|
||||
prepare_media: bool,
|
||||
expected_admission_sha256: str | None = None,
|
||||
) -> PortableRecordedSourceAdmission:
|
||||
_pattern(source_session_id, _SOURCE_ID, "source session id")
|
||||
try:
|
||||
detail, catalog_sha256 = self._session_store.get_session_with_catalog_snapshot(
|
||||
source_session_id
|
||||
)
|
||||
replay = self._session_store.prepare_replay(source_session_id)
|
||||
recorded_media = self._session_store.list_recorded_media(source_session_id)
|
||||
except Exception as exc:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded source could not be resolved"
|
||||
) from exc
|
||||
_digest(catalog_sha256, "source catalog sha256")
|
||||
selected_sources = self._verify_catalog(
|
||||
detail,
|
||||
expected_session_id=source_session_id,
|
||||
)
|
||||
camera_source = selected_sources["video"]
|
||||
media_artifact = self._verify_recorded_media_artifact(
|
||||
detail=detail,
|
||||
camera_source=camera_source,
|
||||
recorded_media=recorded_media,
|
||||
)
|
||||
self._verify_replay(
|
||||
detail=detail,
|
||||
selected_sources=selected_sources,
|
||||
replay=replay,
|
||||
)
|
||||
try:
|
||||
media = (
|
||||
self._media_inspector.inspect(media_artifact, replay)
|
||||
if prepare_media
|
||||
else self._media_inspector.restore_prepared(media_artifact, replay)
|
||||
)
|
||||
except Exception as exc:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera manifest failed validation"
|
||||
) from exc
|
||||
if media is None:
|
||||
raise PortableSourceNotPreparedError("recorded camera manifest has not been prepared")
|
||||
epoch = self._verify_media(
|
||||
media,
|
||||
expected_session_id=detail.summary.session_id,
|
||||
media_artifact=media_artifact,
|
||||
)
|
||||
source_bundle_document = self._source_bundle_document(
|
||||
detail=detail,
|
||||
catalog_sha256=catalog_sha256,
|
||||
selected_sources=selected_sources,
|
||||
replay=replay,
|
||||
media=media,
|
||||
)
|
||||
source_bundle = _canonical_json(source_bundle_document)
|
||||
source_bundle_sha256 = hashlib.sha256(source_bundle).hexdigest()
|
||||
capability_document = self._capability_document(
|
||||
detail=detail,
|
||||
catalog_sha256=catalog_sha256,
|
||||
source_bundle_sha256=source_bundle_sha256,
|
||||
selected_sources=selected_sources,
|
||||
media=media,
|
||||
)
|
||||
capability_manifest = _canonical_json(capability_document)
|
||||
capability_sha256 = hashlib.sha256(capability_manifest).hexdigest()
|
||||
admission = PortableRecordedSourceAdmission(
|
||||
source_session_id=detail.summary.session_id,
|
||||
source_catalog_sha256=catalog_sha256,
|
||||
source_bundle_sha256=source_bundle_sha256,
|
||||
source_capability_manifest_sha256=capability_sha256,
|
||||
source_adapter_sha256=self.requirements.adapter_sha256,
|
||||
frame_count=len(epoch.segments),
|
||||
timeline_start_seconds=epoch.timeline_start_seconds,
|
||||
timeline_end_seconds=epoch.timeline_end_seconds,
|
||||
camera_generation_sha256=media.generation_sha256,
|
||||
source_bundle=source_bundle,
|
||||
capability_manifest=capability_manifest,
|
||||
)
|
||||
if (
|
||||
expected_admission_sha256 is not None
|
||||
and admission.identity_sha256 != expected_admission_sha256
|
||||
):
|
||||
raise PortableSourceAdmissionStaleError(
|
||||
"recorded source changed after its admission check"
|
||||
)
|
||||
if persist:
|
||||
try:
|
||||
final_detail, final_catalog_sha256 = (
|
||||
self._session_store.get_session_with_catalog_snapshot(source_session_id)
|
||||
)
|
||||
except Exception as exc:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded source catalog could not be rechecked"
|
||||
) from exc
|
||||
_digest(final_catalog_sha256, "source catalog sha256")
|
||||
if final_catalog_sha256 != catalog_sha256 or final_detail != detail:
|
||||
raise PortableSourceAdmissionStaleError(
|
||||
"recorded source catalog changed during admission"
|
||||
)
|
||||
_write_immutable_document(
|
||||
self._document_root,
|
||||
source_bundle_sha256,
|
||||
source_bundle,
|
||||
)
|
||||
_write_immutable_document(
|
||||
self._document_root,
|
||||
capability_sha256,
|
||||
capability_manifest,
|
||||
)
|
||||
return admission
|
||||
|
||||
def _verify_catalog(
|
||||
self,
|
||||
detail: SessionDetail,
|
||||
*,
|
||||
expected_session_id: str,
|
||||
) -> dict[str, SessionSource]:
|
||||
expected = self.requirements
|
||||
summary = detail.summary
|
||||
if (
|
||||
summary.session_id != expected_session_id
|
||||
or summary.lab is not None
|
||||
or summary.status != "ready"
|
||||
or not summary.replayable
|
||||
or detail.plugin_id != expected.plugin_id
|
||||
or detail.archive_id != expected.archive_id
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"session is not an admitted recorded K1 source"
|
||||
)
|
||||
if not set(expected.required_modalities).issubset(
|
||||
summary.modalities
|
||||
) or summary.source_count != len(detail.sources):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"required recorded source modalities are unavailable"
|
||||
)
|
||||
artifacts = {artifact.artifact_id: artifact for artifact in detail.artifacts}
|
||||
selected: dict[str, SessionSource] = {}
|
||||
for modality in expected.required_modalities:
|
||||
matches = tuple(source for source in detail.sources if source.modality == modality)
|
||||
if len(matches) != 1:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded modality does not have one canonical source"
|
||||
)
|
||||
source = matches[0]
|
||||
if source.status != "recorded" or not source.seekable:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded source is not sealed and seekable"
|
||||
)
|
||||
artifact = artifacts.get(source.artifact_id)
|
||||
if artifact is None or artifact.integrity_status not in _SEALED_ARTIFACT_STATES:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded source artifact is not sealed"
|
||||
)
|
||||
if modality != "video" and artifact.sha256 is None:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"spatial source artifact has no content identity"
|
||||
)
|
||||
selected[modality] = source
|
||||
camera = selected["video"]
|
||||
recorded_video_artifacts = tuple(
|
||||
artifact for artifact in detail.artifacts if artifact.kind == "recorded-video"
|
||||
)
|
||||
if (
|
||||
camera.source_id != expected.camera_source_id
|
||||
or camera.semantic_channel_id != expected.camera_semantic_channel_id
|
||||
or len(recorded_video_artifacts) != 1
|
||||
or recorded_video_artifacts[0].artifact_id != camera.artifact_id
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera capability does not match the setup"
|
||||
)
|
||||
return selected
|
||||
|
||||
def _verify_recorded_media_artifact(
|
||||
self,
|
||||
*,
|
||||
detail: SessionDetail,
|
||||
camera_source: SessionSource,
|
||||
recorded_media: tuple[RecordedMediaArtifact, ...],
|
||||
) -> RecordedMediaArtifact:
|
||||
artifacts = {artifact.artifact_id: artifact for artifact in detail.artifacts}
|
||||
catalog_artifact = artifacts.get(camera_source.artifact_id)
|
||||
matching_media = tuple(
|
||||
item for item in recorded_media if item.artifact_id == camera_source.artifact_id
|
||||
)
|
||||
if (
|
||||
catalog_artifact is None
|
||||
or catalog_artifact.kind != "recorded-video"
|
||||
or len(recorded_media) != 1
|
||||
or len(matching_media) != 1
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError("recorded camera artifact is unavailable")
|
||||
media_artifact = matching_media[0]
|
||||
if (
|
||||
media_artifact.session_id != detail.summary.session_id
|
||||
or media_artifact.artifact_id != catalog_artifact.artifact_id
|
||||
or media_artifact.byte_length != catalog_artifact.byte_length
|
||||
or media_artifact.byte_length < 1
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera artifact identity disagrees with the catalog"
|
||||
)
|
||||
_pattern(
|
||||
media_artifact.public_source_id,
|
||||
_SOURCE_ID,
|
||||
"recorded camera public source id",
|
||||
)
|
||||
return media_artifact
|
||||
|
||||
def _verify_replay(
|
||||
self,
|
||||
*,
|
||||
detail: SessionDetail,
|
||||
selected_sources: dict[str, SessionSource],
|
||||
replay: ReplayCommand,
|
||||
) -> None:
|
||||
if (
|
||||
replay.session_id != detail.summary.session_id
|
||||
or replay.plugin_id != detail.plugin_id
|
||||
or replay.plugin_id != self.requirements.plugin_id
|
||||
or replay.speed != 1.0
|
||||
or replay.loop
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"spatial replay is bound to another source contract"
|
||||
)
|
||||
catalog_artifacts = {artifact.artifact_id: artifact for artifact in detail.artifacts}
|
||||
replay_ids = tuple(artifact.artifact_id for artifact in replay.artifacts)
|
||||
if not replay_ids or len(set(replay_ids)) != len(replay_ids):
|
||||
raise PortableSourceAdmissionIntegrityError("spatial replay members are not unique")
|
||||
for replay_artifact in replay.artifacts:
|
||||
catalog_artifact = catalog_artifacts.get(replay_artifact.artifact_id)
|
||||
if (
|
||||
catalog_artifact is None
|
||||
or catalog_artifact.integrity_status not in _SEALED_ARTIFACT_STATES
|
||||
or replay_artifact.media_type != catalog_artifact.media_type
|
||||
or replay_artifact.file_byte_length != catalog_artifact.byte_length
|
||||
or not 1 <= replay_artifact.replay_byte_length <= replay_artifact.file_byte_length
|
||||
or replay_artifact.expected_sha256 != catalog_artifact.sha256
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"spatial replay member disagrees with the catalog"
|
||||
)
|
||||
required_spatial_artifact_ids = {
|
||||
selected_sources[modality].artifact_id
|
||||
for modality in self.requirements.required_modalities
|
||||
if modality != "video"
|
||||
}
|
||||
if (
|
||||
replay.primary_artifact_id not in replay_ids
|
||||
or not required_spatial_artifact_ids.issubset(replay_ids)
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"required spatial artifacts are absent from replay"
|
||||
)
|
||||
|
||||
def _probe_camera_summary(
|
||||
self,
|
||||
media_artifact: RecordedMediaArtifact,
|
||||
) -> int:
|
||||
summary, epoch_ordinal = _read_canonical_camera_summary(media_artifact.source_path)
|
||||
segment_count = summary.get("segment_count")
|
||||
if (
|
||||
summary.get("schema_version") != CAMERA_ARCHIVE_SCHEMA
|
||||
or summary.get("source_id") != self.requirements.camera_source_id
|
||||
or summary.get("codec_epoch") != epoch_ordinal
|
||||
or summary.get("status") != "complete"
|
||||
or summary.get("synchronization") != "host-arrival-best-effort"
|
||||
or summary.get("failure_code") is not None
|
||||
or summary.get("init_sha256") != self.requirements.camera_init_sha256
|
||||
or not isinstance(segment_count, int)
|
||||
or isinstance(segment_count, bool)
|
||||
or not 1 <= segment_count <= MAX_SAFE_INTEGER
|
||||
or summary.get("entry_count") != segment_count
|
||||
or summary.get("media_segment_count") != segment_count
|
||||
or summary.get("commit_policy") != "per-segment-fsync"
|
||||
or summary.get("artifacts")
|
||||
!= {
|
||||
"init": "init.mp4",
|
||||
"segments": "segments",
|
||||
"index": "index.jsonl",
|
||||
}
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError("recorded camera summary is incompatible")
|
||||
return segment_count
|
||||
|
||||
def _verify_media(
|
||||
self,
|
||||
manifest: RecordedMediaManifest,
|
||||
*,
|
||||
expected_session_id: str,
|
||||
media_artifact: RecordedMediaArtifact,
|
||||
) -> RecordedMediaEpoch:
|
||||
expected = self.requirements
|
||||
if (
|
||||
manifest.session_id != expected_session_id
|
||||
or manifest.session_id != media_artifact.session_id
|
||||
or manifest.artifact_id != media_artifact.artifact_id
|
||||
or manifest.public_source_id != media_artifact.public_source_id
|
||||
or manifest.byte_length != media_artifact.byte_length
|
||||
or len(manifest.epochs) != 1
|
||||
or manifest.synchronization != "host-arrival-best-effort"
|
||||
or not _SHA256.fullmatch(manifest.generation_sha256)
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera epoch topology is unsupported"
|
||||
)
|
||||
epoch = manifest.epochs[0]
|
||||
if (
|
||||
epoch.media_type != expected.camera_media_type
|
||||
or epoch.init_sha256 != expected.camera_init_sha256
|
||||
or not epoch.segments
|
||||
or epoch.timeline_end_seconds <= epoch.timeline_start_seconds
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera media profile is incompatible"
|
||||
)
|
||||
return epoch
|
||||
|
||||
def _source_bundle_document(
|
||||
self,
|
||||
*,
|
||||
detail: SessionDetail,
|
||||
catalog_sha256: str,
|
||||
selected_sources: dict[str, SessionSource],
|
||||
replay: ReplayCommand,
|
||||
media: RecordedMediaManifest,
|
||||
) -> dict[str, object]:
|
||||
epoch = media.epochs[0]
|
||||
artifacts = {artifact.artifact_id: artifact for artifact in detail.artifacts}
|
||||
return {
|
||||
"schema_version": PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||
"source_session_id": detail.summary.session_id,
|
||||
"source_catalog_sha256": catalog_sha256,
|
||||
"plugin_id": detail.plugin_id,
|
||||
"archive_id": detail.archive_id,
|
||||
"source_adapter": {
|
||||
"id": self.requirements.adapter_id,
|
||||
"version": self.requirements.adapter_version,
|
||||
"sha256": self.requirements.adapter_sha256,
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
**selected_sources[modality].as_dict(),
|
||||
"artifact": _artifact_document(
|
||||
artifacts[selected_sources[modality].artifact_id]
|
||||
),
|
||||
}
|
||||
for modality in self.requirements.required_modalities
|
||||
],
|
||||
"spatial_replay": {
|
||||
"primary_artifact_id": replay.primary_artifact_id,
|
||||
"members": [
|
||||
{
|
||||
"artifact_id": artifact.artifact_id,
|
||||
"media_type": artifact.media_type,
|
||||
"byte_length": artifact.file_byte_length,
|
||||
"replay_byte_length": artifact.replay_byte_length,
|
||||
"sha256": artifact.expected_sha256,
|
||||
}
|
||||
for artifact in replay.artifacts
|
||||
],
|
||||
"timeline_origin_epoch_ns": replay.timeline_origin_epoch_ns,
|
||||
"timeline_origin_monotonic_ns": replay.timeline_origin_monotonic_ns,
|
||||
},
|
||||
"camera": {
|
||||
"artifact_id": media.artifact_id,
|
||||
"public_source_id": media.public_source_id,
|
||||
"generation_sha256": media.generation_sha256,
|
||||
"synchronization": media.synchronization,
|
||||
"epoch": {
|
||||
"ordinal": epoch.ordinal,
|
||||
"media_type": epoch.media_type,
|
||||
"init": {
|
||||
"byte_length": epoch.init_byte_length,
|
||||
"sha256": epoch.init_sha256,
|
||||
},
|
||||
"timeline_start_seconds": epoch.timeline_start_seconds,
|
||||
"timeline_end_seconds": epoch.timeline_end_seconds,
|
||||
"segments": [
|
||||
{
|
||||
"sequence": segment.sequence,
|
||||
"byte_length": segment.byte_length,
|
||||
"sha256": segment.sha256,
|
||||
"random_access": segment.random_access,
|
||||
"end_time_seconds": segment.end_time_seconds,
|
||||
}
|
||||
for segment in epoch.segments
|
||||
],
|
||||
},
|
||||
},
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
def _capability_document(
|
||||
self,
|
||||
*,
|
||||
detail: SessionDetail,
|
||||
catalog_sha256: str,
|
||||
source_bundle_sha256: str,
|
||||
selected_sources: dict[str, SessionSource],
|
||||
media: RecordedMediaManifest,
|
||||
) -> dict[str, object]:
|
||||
epoch = media.epochs[0]
|
||||
return {
|
||||
"schema_version": PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||
"source_session_id": detail.summary.session_id,
|
||||
"source_catalog_sha256": catalog_sha256,
|
||||
"source_bundle_sha256": source_bundle_sha256,
|
||||
"source_adapter_sha256": self.requirements.adapter_sha256,
|
||||
"modalities": [
|
||||
{
|
||||
"modality": modality,
|
||||
"source_id": selected_sources[modality].source_id,
|
||||
"semantic_channel_id": (selected_sources[modality].semantic_channel_id),
|
||||
"seekable": True,
|
||||
}
|
||||
for modality in self.requirements.required_modalities
|
||||
],
|
||||
"camera_profile": {
|
||||
"media_type": epoch.media_type,
|
||||
"init_sha256": epoch.init_sha256,
|
||||
"width": self.requirements.expected_width,
|
||||
"height": self.requirements.expected_height,
|
||||
"profile_attestation": "exact-isobmff-init-sha256",
|
||||
"generation_sha256": media.generation_sha256,
|
||||
"frame_count": len(epoch.segments),
|
||||
"timeline_start_seconds": epoch.timeline_start_seconds,
|
||||
"timeline_end_seconds": epoch.timeline_end_seconds,
|
||||
},
|
||||
"calibration": {
|
||||
"slot": self.requirements.calibration_slot,
|
||||
"sha256": self.requirements.calibration_sha256,
|
||||
"binding": "external-rig-profile",
|
||||
},
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
|
||||
def _read_canonical_camera_summary(
|
||||
source_path: Path,
|
||||
) -> tuple[dict[str, object], int]:
|
||||
"""Read one direct epoch summary through bounded no-follow descriptors."""
|
||||
|
||||
source_descriptor = -1
|
||||
epoch_descriptor = -1
|
||||
summary_descriptor = -1
|
||||
try:
|
||||
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
||||
source_descriptor = os.open(source_path, directory_flags)
|
||||
if not stat.S_ISDIR(os.fstat(source_descriptor).st_mode):
|
||||
raise PortableSourceAdmissionIntegrityError("recorded camera source is not a directory")
|
||||
epoch_name: str | None = None
|
||||
epoch_ordinal: int | None = None
|
||||
with os.scandir(source_descriptor) as entries:
|
||||
for entry in entries:
|
||||
match = _EPOCH.fullmatch(entry.name)
|
||||
if (
|
||||
epoch_name is not None
|
||||
or match is None
|
||||
or not entry.is_dir(follow_symlinks=False)
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera source does not have one canonical epoch"
|
||||
)
|
||||
epoch_name = entry.name
|
||||
epoch_ordinal = int(match.group(1))
|
||||
if epoch_name is None or epoch_ordinal is None:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera source does not have one canonical epoch"
|
||||
)
|
||||
epoch_descriptor = os.open(
|
||||
epoch_name,
|
||||
directory_flags,
|
||||
dir_fd=source_descriptor,
|
||||
)
|
||||
if not stat.S_ISDIR(os.fstat(epoch_descriptor).st_mode):
|
||||
raise PortableSourceAdmissionIntegrityError("recorded camera epoch is not a directory")
|
||||
summary_descriptor = os.open(
|
||||
"summary.json",
|
||||
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
|
||||
dir_fd=epoch_descriptor,
|
||||
)
|
||||
before = os.fstat(summary_descriptor)
|
||||
if not stat.S_ISREG(before.st_mode) or not 1 <= before.st_size <= MAX_MEDIA_SUMMARY_BYTES:
|
||||
raise PortableSourceAdmissionIntegrityError("recorded camera summary is outside bounds")
|
||||
chunks: list[bytes] = []
|
||||
remaining = before.st_size
|
||||
while remaining:
|
||||
chunk = os.read(summary_descriptor, remaining)
|
||||
if not chunk:
|
||||
break
|
||||
chunks.append(chunk)
|
||||
remaining -= len(chunk)
|
||||
payload = b"".join(chunks)
|
||||
after = os.fstat(summary_descriptor)
|
||||
if len(payload) != before.st_size or (
|
||||
after.st_dev,
|
||||
after.st_ino,
|
||||
after.st_size,
|
||||
after.st_mtime_ns,
|
||||
) != (
|
||||
before.st_dev,
|
||||
before.st_ino,
|
||||
before.st_size,
|
||||
before.st_mtime_ns,
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera summary changed while it was read"
|
||||
)
|
||||
try:
|
||||
value = json.loads(payload)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera summary is invalid"
|
||||
) from exc
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise PortableSourceAdmissionIntegrityError("recorded camera summary is not an object")
|
||||
return value, epoch_ordinal
|
||||
except PortableSourceAdmissionIntegrityError:
|
||||
raise
|
||||
except OSError as exc:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera summary is unavailable"
|
||||
) from exc
|
||||
finally:
|
||||
for descriptor in (
|
||||
summary_descriptor,
|
||||
epoch_descriptor,
|
||||
source_descriptor,
|
||||
):
|
||||
if descriptor >= 0:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _artifact_document(artifact: SessionArtifact) -> dict[str, object]:
|
||||
return {
|
||||
"artifact_id": artifact.artifact_id,
|
||||
"kind": artifact.kind,
|
||||
"media_type": artifact.media_type,
|
||||
"byte_length": artifact.byte_length,
|
||||
"sha256": artifact.sha256,
|
||||
"integrity_status": artifact.integrity_status,
|
||||
}
|
||||
|
||||
|
||||
def _write_immutable_document(root: Path, digest: str, payload: bytes) -> None:
|
||||
_digest(digest, "document sha256")
|
||||
if hashlib.sha256(payload).hexdigest() != digest:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"immutable document payload has another identity"
|
||||
)
|
||||
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
metadata = root.lstat()
|
||||
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode):
|
||||
raise PortableSourceAdmissionIntegrityError("portable source document root is unsafe")
|
||||
destination = root / f"{digest}.json"
|
||||
if destination.exists():
|
||||
existing = destination.read_bytes()
|
||||
if existing != payload:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"immutable source document identity collided"
|
||||
)
|
||||
return
|
||||
temporary = root / f".tmp-{secrets.token_hex(16)}"
|
||||
descriptor = os.open(
|
||||
temporary,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
||||
0o600,
|
||||
)
|
||||
published = False
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as stream:
|
||||
stream.write(payload)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, destination)
|
||||
_fsync_directory(root)
|
||||
published = True
|
||||
finally:
|
||||
if not published:
|
||||
with suppress(FileNotFoundError):
|
||||
temporary.unlink()
|
||||
|
||||
|
||||
def _fsync_directory(path: Path) -> None:
|
||||
descriptor = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _sha256(value: object) -> str:
|
||||
return hashlib.sha256(_canonical_json(value)).hexdigest()
|
||||
|
||||
|
||||
def _digest(value: object, label: str) -> str:
|
||||
if not isinstance(value, str) or _SHA256.fullmatch(value) is None:
|
||||
raise ValueError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _pattern(value: object, pattern: re.Pattern[str], label: str) -> str:
|
||||
if not isinstance(value, str) or pattern.fullmatch(value) is None:
|
||||
raise ValueError(f"{label} is invalid")
|
||||
return value
|
||||
@@ -0,0 +1,695 @@
|
||||
"""Transport-agnostic Worker 006 core for sealed Observatory jobs.
|
||||
|
||||
The agent accepts only the durable, path-free recorded-job projection. It
|
||||
validates every identity sealed by Mission Core, resolves an executor from a
|
||||
local four-digest allowlist, and passes a typed job to that executor. Server
|
||||
payloads can never provide commands, paths, environment variables, images, or
|
||||
other executable instructions.
|
||||
|
||||
Network authentication, polling cadence, local CAS resolution, ML runtimes,
|
||||
and deployment are deliberately outside this module. They are supplied by a
|
||||
transport and an executor adapter at the Worker boundary.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Annotated, Final, Literal, Protocol
|
||||
from uuid import uuid4
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
||||
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
OBSERVATORY_RECORDED_CLAIM_SCHEMA,
|
||||
OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA,
|
||||
OBSERVATORY_RECORDED_JOB_SCHEMA,
|
||||
)
|
||||
|
||||
WORKER_006_CONTOUR_ID: Final = "worker-006"
|
||||
MAX_EXECUTOR_FAILURE_MESSAGE_LENGTH: Final = 512
|
||||
|
||||
_JOB_ID_PATTERN: Final = r"^observatory-run-[a-f0-9]{32}$"
|
||||
_CLAIM_TOKEN_PATTERN: Final = r"^[a-f0-9]{64}$"
|
||||
_CLAIM_REQUEST_ID_PATTERN: Final = r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$"
|
||||
_SESSION_ID_PATTERN: Final = r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"
|
||||
_IDENTIFIER_PATTERN: Final = r"^[a-z][a-z0-9-]{2,95}$"
|
||||
_SHA256_PATTERN: Final = r"^[a-f0-9]{64}$"
|
||||
_TIMESTAMP_PATTERN: Final = (
|
||||
r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}"
|
||||
r"(?:\.[0-9]{1,9})?(?:Z|[+-][0-9]{2}:[0-9]{2})$"
|
||||
)
|
||||
_CLAIM_REQUEST_ID = re.compile(_CLAIM_REQUEST_ID_PATTERN)
|
||||
|
||||
type WorkerCycleState = Literal[
|
||||
"empty",
|
||||
"deferred",
|
||||
"succeeded",
|
||||
"failed",
|
||||
"rejected",
|
||||
]
|
||||
type RecordedJobWireState = Literal[
|
||||
"accepted",
|
||||
"queued",
|
||||
"claimed",
|
||||
"running",
|
||||
"paused",
|
||||
"preemption-pending",
|
||||
"succeeded",
|
||||
"failed",
|
||||
"reconciliation-required",
|
||||
]
|
||||
|
||||
Sha256 = Annotated[str, Field(pattern=_SHA256_PATTERN)]
|
||||
Identifier = Annotated[
|
||||
str,
|
||||
Field(min_length=3, max_length=96, pattern=_IDENTIFIER_PATTERN),
|
||||
]
|
||||
SessionId = Annotated[
|
||||
str,
|
||||
Field(min_length=1, max_length=128, pattern=_SESSION_ID_PATTERN),
|
||||
]
|
||||
Timestamp = Annotated[
|
||||
str,
|
||||
Field(min_length=20, max_length=64, pattern=_TIMESTAMP_PATTERN),
|
||||
]
|
||||
|
||||
|
||||
class ObservatoryWorkerAgentError(RuntimeError):
|
||||
"""Base error for the local Worker 006 agent core."""
|
||||
|
||||
|
||||
class ObservatoryWorkerAgentBusyError(ObservatoryWorkerAgentError):
|
||||
"""A second poll cycle attempted to overlap the active claim."""
|
||||
|
||||
|
||||
class ObservatoryWorkerClaimRejectedError(ObservatoryWorkerAgentError):
|
||||
"""A transport supplied an unknown, spoofed, or corrupted claim."""
|
||||
|
||||
|
||||
class ObservatoryWorkerExecutorUnavailableError(ObservatoryWorkerAgentError):
|
||||
"""No local adapter matches the exact sealed executor identity."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerExecutorIdentity:
|
||||
"""The only identity that may select executable Worker code."""
|
||||
|
||||
release_sha256: str
|
||||
image_sha256: str
|
||||
model_manifest_sha256: str
|
||||
resource_profile_sha256: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for label, value in (
|
||||
("executor release", self.release_sha256),
|
||||
("executor image", self.image_sha256),
|
||||
("model manifest", self.model_manifest_sha256),
|
||||
("resource profile", self.resource_profile_sha256),
|
||||
):
|
||||
if re.fullmatch(_SHA256_PATTERN, value) is None:
|
||||
raise ValueError(f"{label} SHA-256 is invalid")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SealedObservatoryRecordedJob:
|
||||
"""Validated path-free job passed to one local executor adapter."""
|
||||
|
||||
job_id: str
|
||||
request_sha256: str
|
||||
identity_sha256: str
|
||||
source_session_id: str
|
||||
source_catalog_sha256: str
|
||||
source_bundle_sha256: str
|
||||
source_capability_manifest_sha256: str
|
||||
source_adapter_id: str
|
||||
source_adapter_version: int
|
||||
source_adapter_sha256: str
|
||||
setup_id: str
|
||||
definition_id: str
|
||||
definition_version: int
|
||||
definition_sha256: str
|
||||
executor_release_id: str
|
||||
executor_identity: ObservatoryWorkerExecutorIdentity
|
||||
model_release_ids: tuple[str, ...]
|
||||
resource_profile_id: str
|
||||
checkpoint_policy: Literal["cooperative", "non-checkpointable"]
|
||||
allowed_checkpoints: tuple[str, ...]
|
||||
claim_generation: int
|
||||
restart_from_zero: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerExecutionResult:
|
||||
"""Content-addressed result identity returned by a local executor."""
|
||||
|
||||
result_id: str
|
||||
result_sha256: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if re.fullmatch(_SESSION_ID_PATTERN, self.result_id) is None:
|
||||
raise ValueError("Worker result id is invalid")
|
||||
if re.fullmatch(_SHA256_PATTERN, self.result_sha256) is None:
|
||||
raise ValueError("Worker result SHA-256 is invalid")
|
||||
|
||||
|
||||
class ObservatoryWorkerExecutor(Protocol):
|
||||
"""A local implementation selected only by its sealed digest tuple."""
|
||||
|
||||
def execute(
|
||||
self,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
) -> ObservatoryWorkerExecutionResult: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerExecutorRegistration:
|
||||
identity: ObservatoryWorkerExecutorIdentity
|
||||
adapter: ObservatoryWorkerExecutor
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerExecutorRegistry:
|
||||
"""In-memory local executor allowlist; it never accepts server code."""
|
||||
|
||||
registrations: tuple[ObservatoryWorkerExecutorRegistration, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
identities = [registration.identity for registration in self.registrations]
|
||||
if len(identities) != len(set(identities)):
|
||||
raise ValueError("Worker executor identities must be unique")
|
||||
|
||||
def resolve(
|
||||
self,
|
||||
identity: ObservatoryWorkerExecutorIdentity,
|
||||
) -> ObservatoryWorkerExecutor:
|
||||
for registration in self.registrations:
|
||||
if registration.identity == identity:
|
||||
return registration.adapter
|
||||
raise ObservatoryWorkerExecutorUnavailableError(
|
||||
"exact executor identity is not locally allowlisted"
|
||||
)
|
||||
|
||||
|
||||
class ObservatoryWorkerTransport(Protocol):
|
||||
"""State-transition port implemented by HTTP, IPC, or a test transport."""
|
||||
|
||||
def claim_next(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
claim_request_id: str,
|
||||
) -> Mapping[str, object] | None: ...
|
||||
|
||||
def start(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
) -> Mapping[str, object]: ...
|
||||
|
||||
def succeed(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
result_id: str,
|
||||
result_sha256: str,
|
||||
) -> Mapping[str, object]: ...
|
||||
|
||||
def fail(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
error_code: str,
|
||||
message: str,
|
||||
) -> Mapping[str, object]: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerCycleReport:
|
||||
state: WorkerCycleState
|
||||
claim_request_id: str
|
||||
job_id: str | None = None
|
||||
result_id: str | None = None
|
||||
failure_code: str | None = None
|
||||
|
||||
|
||||
class _StrictPayload(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||
|
||||
|
||||
class _AuthorityPayload(_StrictPayload):
|
||||
commands_enabled: Literal[False]
|
||||
actuation_allowed: Literal[False]
|
||||
navigation_or_safety_accepted: Literal[False]
|
||||
production_accepted: Literal[False]
|
||||
|
||||
|
||||
class _SourceAdapterPayload(_StrictPayload):
|
||||
adapter_id: Identifier
|
||||
version: int = Field(ge=1)
|
||||
adapter_sha256: Sha256
|
||||
|
||||
|
||||
class _SourcePayload(_StrictPayload):
|
||||
session_id: SessionId
|
||||
catalog_sha256: Sha256
|
||||
bundle_sha256: Sha256
|
||||
capability_manifest_sha256: Sha256
|
||||
adapter: _SourceAdapterPayload
|
||||
|
||||
|
||||
class _SetupPayload(_StrictPayload):
|
||||
setup_id: Identifier
|
||||
definition_id: Identifier
|
||||
definition_version: int = Field(ge=1)
|
||||
definition_sha256: Sha256
|
||||
|
||||
|
||||
class _ExecutorPayload(_StrictPayload):
|
||||
release_id: Identifier
|
||||
release_sha256: Sha256
|
||||
image_sha256: Sha256
|
||||
model_release_ids: list[Identifier] = Field(max_length=32)
|
||||
learned_models: list[Identifier] = Field(max_length=32)
|
||||
model_manifest_sha256: Sha256
|
||||
resource_profile_id: Identifier
|
||||
resource_profile_sha256: Sha256
|
||||
|
||||
|
||||
class _CheckpointPolicyPayload(_StrictPayload):
|
||||
mode: Literal["cooperative", "non-checkpointable"]
|
||||
allowed_checkpoints: list[Identifier] = Field(max_length=64)
|
||||
last_checkpoint_id: Identifier | None
|
||||
|
||||
|
||||
class _PriorityPayload(_StrictPayload):
|
||||
class_: Literal["recorded"] = Field(alias="class")
|
||||
rank: Literal[100]
|
||||
server_owned: Literal[True]
|
||||
|
||||
|
||||
class _ResultPayload(_StrictPayload):
|
||||
result_id: SessionId
|
||||
sha256: Sha256
|
||||
|
||||
|
||||
class _TerminalPayload(_StrictPayload):
|
||||
code: Identifier
|
||||
message: str = Field(min_length=1, max_length=1_000)
|
||||
|
||||
|
||||
class _RecordedJobPayload(_StrictPayload):
|
||||
schema_version: Literal["missioncore.observatory-recorded-job/v1"]
|
||||
job_id: str = Field(pattern=_JOB_ID_PATTERN)
|
||||
idempotency_key: str = Field(
|
||||
min_length=1,
|
||||
max_length=160,
|
||||
pattern=_CLAIM_REQUEST_ID_PATTERN,
|
||||
)
|
||||
request_sha256: Sha256
|
||||
identity_sha256: Sha256
|
||||
submission_receipt_sha256: Sha256
|
||||
source: _SourcePayload
|
||||
setup: _SetupPayload
|
||||
executor: _ExecutorPayload
|
||||
checkpoint_policy: _CheckpointPolicyPayload
|
||||
priority: _PriorityPayload
|
||||
state: RecordedJobWireState
|
||||
preemption_requested: bool
|
||||
restart_from_zero: bool
|
||||
preemption_receipt_sha256: Sha256 | None
|
||||
claim_generation: int = Field(ge=0)
|
||||
result: _ResultPayload | None
|
||||
terminal: _TerminalPayload | None
|
||||
created_at_utc: Timestamp
|
||||
updated_at_utc: Timestamp
|
||||
authority: _AuthorityPayload
|
||||
|
||||
|
||||
class _RecordedClaimPayload(_StrictPayload):
|
||||
schema_version: Literal["missioncore.observatory-recorded-job-claim/v1"]
|
||||
claim_request_id: str = Field(
|
||||
min_length=1,
|
||||
max_length=160,
|
||||
pattern=_CLAIM_REQUEST_ID_PATTERN,
|
||||
)
|
||||
request_sha256: Sha256
|
||||
claimant_id: Identifier
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
job: _RecordedJobPayload
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ValidatedClaim:
|
||||
claim_request_id: str
|
||||
claim_token: str
|
||||
job: SealedObservatoryRecordedJob
|
||||
|
||||
|
||||
class ObservatoryWorkerAgent:
|
||||
"""Runs at most one sealed recorded-job claim at a time on Worker 006."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
transport: ObservatoryWorkerTransport,
|
||||
executors: ObservatoryWorkerExecutorRegistry,
|
||||
claim_request_id_factory: Callable[[], str] | None = None,
|
||||
) -> None:
|
||||
self._transport = transport
|
||||
self._executors = executors
|
||||
self._claim_request_id_factory = claim_request_id_factory or _default_claim_request_id
|
||||
self._cycle_lock = threading.Lock()
|
||||
|
||||
def run_once(self) -> ObservatoryWorkerCycleReport:
|
||||
"""Claim, validate, execute, and seal one durable job if available."""
|
||||
|
||||
if not self._cycle_lock.acquire(blocking=False):
|
||||
raise ObservatoryWorkerAgentBusyError("Worker 006 already owns an active claim cycle")
|
||||
try:
|
||||
return self._run_once_locked()
|
||||
finally:
|
||||
self._cycle_lock.release()
|
||||
|
||||
def _run_once_locked(self) -> ObservatoryWorkerCycleReport:
|
||||
claim_request_id = self._claim_request_id_factory()
|
||||
if _CLAIM_REQUEST_ID.fullmatch(claim_request_id) is None:
|
||||
raise ValueError("Worker claim request id is invalid")
|
||||
payload = self._transport.claim_next(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
claim_request_id=claim_request_id,
|
||||
)
|
||||
if payload is None:
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="empty",
|
||||
claim_request_id=claim_request_id,
|
||||
)
|
||||
try:
|
||||
claim = _validate_claim(payload, claim_request_id=claim_request_id)
|
||||
except ObservatoryWorkerClaimRejectedError:
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="rejected",
|
||||
claim_request_id=claim_request_id,
|
||||
failure_code="claim-rejected",
|
||||
)
|
||||
|
||||
try:
|
||||
adapter = self._executors.resolve(claim.job.executor_identity)
|
||||
except ObservatoryWorkerExecutorUnavailableError:
|
||||
failure_code = "executor-not-allowlisted"
|
||||
acknowledgement = self._transport.fail(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
job_id=claim.job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
error_code=failure_code,
|
||||
message="Exact executor identity is not installed on Worker 006.",
|
||||
)
|
||||
_validate_transition_acknowledgement(
|
||||
acknowledgement,
|
||||
expected_job=claim.job,
|
||||
expected_state="failed",
|
||||
)
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="failed",
|
||||
claim_request_id=claim_request_id,
|
||||
job_id=claim.job.job_id,
|
||||
failure_code=failure_code,
|
||||
)
|
||||
|
||||
started_payload = self._transport.start(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
job_id=claim.job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
)
|
||||
started = _validate_transition_acknowledgement(
|
||||
started_payload,
|
||||
expected_job=claim.job,
|
||||
expected_state=("running", "paused"),
|
||||
)
|
||||
if started.state == "paused":
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="deferred",
|
||||
claim_request_id=claim_request_id,
|
||||
job_id=claim.job.job_id,
|
||||
)
|
||||
|
||||
try:
|
||||
result = adapter.execute(claim.job)
|
||||
if not isinstance(result, ObservatoryWorkerExecutionResult):
|
||||
raise TypeError("executor returned an unknown result contract")
|
||||
except Exception as exc:
|
||||
failure_code = "executor-error"
|
||||
acknowledgement = self._transport.fail(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
job_id=claim.job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
error_code=failure_code,
|
||||
message=_bounded_executor_failure(exc),
|
||||
)
|
||||
_validate_transition_acknowledgement(
|
||||
acknowledgement,
|
||||
expected_job=claim.job,
|
||||
expected_state="failed",
|
||||
)
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="failed",
|
||||
claim_request_id=claim_request_id,
|
||||
job_id=claim.job.job_id,
|
||||
failure_code=failure_code,
|
||||
)
|
||||
|
||||
acknowledgement = self._transport.succeed(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
job_id=claim.job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
result_id=result.result_id,
|
||||
result_sha256=result.result_sha256,
|
||||
)
|
||||
succeeded = _validate_transition_acknowledgement(
|
||||
acknowledgement,
|
||||
expected_job=claim.job,
|
||||
expected_state="succeeded",
|
||||
)
|
||||
if succeeded.result is None or (
|
||||
succeeded.result.result_id != result.result_id
|
||||
or succeeded.result.sha256 != result.result_sha256
|
||||
):
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker success acknowledgement changed result identity"
|
||||
)
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="succeeded",
|
||||
claim_request_id=claim_request_id,
|
||||
job_id=claim.job.job_id,
|
||||
result_id=result.result_id,
|
||||
)
|
||||
|
||||
|
||||
def _validate_claim(
|
||||
payload: Mapping[str, object],
|
||||
*,
|
||||
claim_request_id: str,
|
||||
) -> _ValidatedClaim:
|
||||
try:
|
||||
claim = _RecordedClaimPayload.model_validate(dict(payload))
|
||||
if claim.claim_request_id != claim_request_id:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim request identity changed")
|
||||
if claim.claimant_id != WORKER_006_CONTOUR_ID:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim belongs to another claimant")
|
||||
expected_claim_request_sha256 = _sha256_document(
|
||||
{
|
||||
"schema_version": OBSERVATORY_RECORDED_CLAIM_SCHEMA,
|
||||
"claim_request_id": claim_request_id,
|
||||
"claimant_id": WORKER_006_CONTOUR_ID,
|
||||
}
|
||||
)
|
||||
if claim.request_sha256 != expected_claim_request_sha256:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim request digest changed")
|
||||
if claim.job.state != "claimed" or claim.job.claim_generation < 1:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim job is not in a claimed generation"
|
||||
)
|
||||
if claim.job.result is not None or claim.job.terminal is not None:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim already carries a terminal outcome"
|
||||
)
|
||||
job = _seal_job(claim.job)
|
||||
except ObservatoryWorkerClaimRejectedError:
|
||||
raise
|
||||
except (TypeError, ValueError, ValidationError) as exc:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim payload violates the sealed protocol"
|
||||
) from exc
|
||||
return _ValidatedClaim(
|
||||
claim_request_id=claim.claim_request_id,
|
||||
claim_token=claim.claim_token,
|
||||
job=job,
|
||||
)
|
||||
|
||||
|
||||
def _seal_job(payload: _RecordedJobPayload) -> SealedObservatoryRecordedJob:
|
||||
if payload.executor.learned_models != payload.executor.model_release_ids:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim learned-model identities changed")
|
||||
if len(set(payload.executor.model_release_ids)) != len(payload.executor.model_release_ids):
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim model identities are not unique")
|
||||
if len(set(payload.checkpoint_policy.allowed_checkpoints)) != len(
|
||||
payload.checkpoint_policy.allowed_checkpoints
|
||||
):
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim checkpoint identities are not unique"
|
||||
)
|
||||
if (
|
||||
payload.checkpoint_policy.mode == "cooperative"
|
||||
and not payload.checkpoint_policy.allowed_checkpoints
|
||||
) or (
|
||||
payload.checkpoint_policy.mode == "non-checkpointable"
|
||||
and payload.checkpoint_policy.allowed_checkpoints
|
||||
):
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim checkpoint policy is inconsistent")
|
||||
|
||||
expected_request_sha256 = _sha256_document(
|
||||
{
|
||||
"schema_version": OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA,
|
||||
"idempotency_key": payload.idempotency_key,
|
||||
"source_session_id": payload.source.session_id,
|
||||
"source_catalog_sha256": payload.source.catalog_sha256,
|
||||
"source_bundle_sha256": payload.source.bundle_sha256,
|
||||
"source_capability_manifest_sha256": (payload.source.capability_manifest_sha256),
|
||||
"setup_id": payload.setup.setup_id,
|
||||
"definition_sha256": payload.setup.definition_sha256,
|
||||
}
|
||||
)
|
||||
if payload.request_sha256 != expected_request_sha256:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker recorded-job request identity changed")
|
||||
expected_identity_sha256 = _sha256_document(
|
||||
{
|
||||
"schema_version": OBSERVATORY_RECORDED_JOB_SCHEMA,
|
||||
"source_session_id": payload.source.session_id,
|
||||
"source_catalog_sha256": payload.source.catalog_sha256,
|
||||
"source_bundle_sha256": payload.source.bundle_sha256,
|
||||
"source_capability_manifest_sha256": (payload.source.capability_manifest_sha256),
|
||||
"setup_id": payload.setup.setup_id,
|
||||
"definition_id": payload.setup.definition_id,
|
||||
"definition_version": payload.setup.definition_version,
|
||||
"definition_sha256": payload.setup.definition_sha256,
|
||||
"source_adapter_id": payload.source.adapter.adapter_id,
|
||||
"source_adapter_version": payload.source.adapter.version,
|
||||
"source_adapter_sha256": payload.source.adapter.adapter_sha256,
|
||||
"executor_release_id": payload.executor.release_id,
|
||||
"executor_release_sha256": payload.executor.release_sha256,
|
||||
"executor_image_sha256": payload.executor.image_sha256,
|
||||
"model_release_ids": payload.executor.model_release_ids,
|
||||
"model_manifest_sha256": payload.executor.model_manifest_sha256,
|
||||
"resource_profile_id": payload.executor.resource_profile_id,
|
||||
"resource_profile_sha256": payload.executor.resource_profile_sha256,
|
||||
"checkpoint_policy": payload.checkpoint_policy.mode,
|
||||
"allowed_checkpoints": payload.checkpoint_policy.allowed_checkpoints,
|
||||
}
|
||||
)
|
||||
if payload.identity_sha256 != expected_identity_sha256:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker recorded-job execution identity changed")
|
||||
expected_submission_receipt_sha256 = _sha256_document(
|
||||
{
|
||||
"schema_version": OBSERVATORY_RECORDED_JOB_SCHEMA,
|
||||
"job_id": payload.job_id,
|
||||
"request_sha256": payload.request_sha256,
|
||||
"identity_sha256": payload.identity_sha256,
|
||||
"state": "accepted",
|
||||
"created_at_utc": payload.created_at_utc,
|
||||
}
|
||||
)
|
||||
if payload.submission_receipt_sha256 != expected_submission_receipt_sha256:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker recorded-job submission receipt changed")
|
||||
|
||||
return SealedObservatoryRecordedJob(
|
||||
job_id=payload.job_id,
|
||||
request_sha256=payload.request_sha256,
|
||||
identity_sha256=payload.identity_sha256,
|
||||
source_session_id=payload.source.session_id,
|
||||
source_catalog_sha256=payload.source.catalog_sha256,
|
||||
source_bundle_sha256=payload.source.bundle_sha256,
|
||||
source_capability_manifest_sha256=(payload.source.capability_manifest_sha256),
|
||||
source_adapter_id=payload.source.adapter.adapter_id,
|
||||
source_adapter_version=payload.source.adapter.version,
|
||||
source_adapter_sha256=payload.source.adapter.adapter_sha256,
|
||||
setup_id=payload.setup.setup_id,
|
||||
definition_id=payload.setup.definition_id,
|
||||
definition_version=payload.setup.definition_version,
|
||||
definition_sha256=payload.setup.definition_sha256,
|
||||
executor_release_id=payload.executor.release_id,
|
||||
executor_identity=ObservatoryWorkerExecutorIdentity(
|
||||
release_sha256=payload.executor.release_sha256,
|
||||
image_sha256=payload.executor.image_sha256,
|
||||
model_manifest_sha256=payload.executor.model_manifest_sha256,
|
||||
resource_profile_sha256=payload.executor.resource_profile_sha256,
|
||||
),
|
||||
model_release_ids=tuple(payload.executor.model_release_ids),
|
||||
resource_profile_id=payload.executor.resource_profile_id,
|
||||
checkpoint_policy=payload.checkpoint_policy.mode,
|
||||
allowed_checkpoints=tuple(payload.checkpoint_policy.allowed_checkpoints),
|
||||
claim_generation=payload.claim_generation,
|
||||
restart_from_zero=payload.restart_from_zero,
|
||||
)
|
||||
|
||||
|
||||
def _validate_transition_acknowledgement(
|
||||
payload: Mapping[str, object],
|
||||
*,
|
||||
expected_job: SealedObservatoryRecordedJob,
|
||||
expected_state: RecordedJobWireState | tuple[RecordedJobWireState, ...],
|
||||
) -> _RecordedJobPayload:
|
||||
try:
|
||||
acknowledgement = _RecordedJobPayload.model_validate(dict(payload))
|
||||
sealed = _seal_job(acknowledgement)
|
||||
except ObservatoryWorkerClaimRejectedError:
|
||||
raise
|
||||
except (TypeError, ValueError, ValidationError) as exc:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker transition acknowledgement violates the sealed protocol"
|
||||
) from exc
|
||||
states = (expected_state,) if isinstance(expected_state, str) else expected_state
|
||||
if acknowledgement.state not in states:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker transition acknowledgement has an unexpected state"
|
||||
)
|
||||
if (
|
||||
sealed.job_id != expected_job.job_id
|
||||
or sealed.identity_sha256 != expected_job.identity_sha256
|
||||
or sealed.claim_generation != expected_job.claim_generation
|
||||
):
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker transition acknowledgement changed job identity"
|
||||
)
|
||||
return acknowledgement
|
||||
|
||||
|
||||
def _default_claim_request_id() -> str:
|
||||
return f"worker-006:{uuid4().hex}"
|
||||
|
||||
|
||||
def _bounded_executor_failure(exc: Exception) -> str:
|
||||
detail = " ".join(str(exc).split())
|
||||
message = f"Executor adapter raised {type(exc).__name__}."
|
||||
if detail:
|
||||
message = f"{message} {detail}"
|
||||
return message[:MAX_EXECUTOR_FAILURE_MESSAGE_LENGTH]
|
||||
|
||||
|
||||
def _sha256_document(document: Mapping[str, object]) -> str:
|
||||
payload = json.dumps(
|
||||
document,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
+99
-25
@@ -47,10 +47,20 @@ from k1link.observatory.m49_queue_binding import (
|
||||
M49QueueBindingError,
|
||||
M49RecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinitionRegistry,
|
||||
PortableRunDefinitionRegistryError,
|
||||
)
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
PORTABLE_LAB_V1_SETUP_ID,
|
||||
PortableLabV1SetupProjector,
|
||||
PortableSetupProjectionError,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueError,
|
||||
)
|
||||
from k1link.observatory.source_admission import RecordedK1SourceAdmissionService
|
||||
from k1link.sessions import (
|
||||
MaterializedRecording,
|
||||
RecordedCameraFrameService,
|
||||
@@ -160,6 +170,11 @@ from k1link.web.map_api import (
|
||||
)
|
||||
from k1link.web.map_view_api import build_map_view_router
|
||||
from k1link.web.observatory_api import build_observatory_router
|
||||
from k1link.web.observatory_worker_api import (
|
||||
ObservatoryWorkerAuthentication,
|
||||
build_observatory_worker_router,
|
||||
load_observatory_worker_authentication,
|
||||
)
|
||||
from k1link.web.plugin_catalog import DevicePluginCatalog, PluginCatalogError
|
||||
from k1link.web.plugin_runtime import (
|
||||
STATE_READ_ACTION_ID,
|
||||
@@ -245,6 +260,25 @@ plugin_environment = load_installed_device_plugins(REPOSITORY_ROOT)
|
||||
plugin_catalog: DevicePluginCatalog = plugin_environment.catalog
|
||||
plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher
|
||||
session_store = SessionStore(REPOSITORY_ROOT)
|
||||
|
||||
|
||||
def _load_optional_observatory_worker_authentication(
|
||||
recorded_job_queue: ObservatoryRecordedJobQueue | None,
|
||||
*,
|
||||
token_path: Path,
|
||||
) -> tuple[ObservatoryWorkerAuthentication | None, str | None]:
|
||||
"""Load the optional Worker credential without widening app startup risk."""
|
||||
|
||||
if recorded_job_queue is None:
|
||||
return None, "Observatory recorded-job queue is unavailable"
|
||||
try:
|
||||
return load_observatory_worker_authentication(token_path), None
|
||||
except ValueError as exc:
|
||||
# Worker pull transport is optional. A missing or unsafe credential
|
||||
# disables only this router; K1, Simulation and legacy LAB still start.
|
||||
return None, str(exc)
|
||||
|
||||
|
||||
OBSERVATORY_RUN_PREPARATION_LEDGER: ObservatoryRunPreparationLedger | None
|
||||
OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR: str | None
|
||||
(
|
||||
@@ -262,9 +296,7 @@ try:
|
||||
session_store=session_store,
|
||||
setup_registry=OBSERVATORY_LABORATORY_SETUP_REGISTRY,
|
||||
config=M49QueueBindingConfig.from_file(
|
||||
REPOSITORY_ROOT
|
||||
/ "config"
|
||||
/ "observatory-m49-recorded-queue-binding.json"
|
||||
REPOSITORY_ROOT / "config" / "observatory-m49-recorded-queue-binding.json"
|
||||
),
|
||||
)
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE = ObservatoryRecordedJobQueue(
|
||||
@@ -279,6 +311,17 @@ except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError
|
||||
OBSERVATORY_RECORDED_BINDING_SERVICE = None
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE = None
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = str(exc)
|
||||
OBSERVATORY_WORKER_TOKEN_PATH = session_store.data_dir / "worker-auth" / "observatory-worker.token"
|
||||
OBSERVATORY_WORKER_CLAIM_LEASE_READY = False
|
||||
OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = False
|
||||
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED = False
|
||||
OBSERVATORY_WORKER_AUTHENTICATION: ObservatoryWorkerAuthentication | None
|
||||
OBSERVATORY_WORKER_API_ERROR: str | None
|
||||
OBSERVATORY_WORKER_AUTHENTICATION = None
|
||||
OBSERVATORY_WORKER_API_ERROR = (
|
||||
"Worker pull API is hard-disabled until claim leases and a verified "
|
||||
"Observatory result publisher are implemented and accepted"
|
||||
)
|
||||
simulation_project_store = SimulationProjectStore(session_store.data_dir)
|
||||
simulation_project_service = SimulationProjectService(simulation_project_store)
|
||||
session_artifact_gateway = configured_artifact_gateway(session_store.data_dir)
|
||||
@@ -293,6 +336,39 @@ session_recording_materializer = SessionRecordingMaterializer(
|
||||
session_recorded_media_inspector = RecordedMediaInspector(
|
||||
session_store.data_dir / "recorded-media-preparations"
|
||||
)
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableLabV1SetupProjector | None
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
|
||||
try:
|
||||
portable_definition_registry = PortableRunDefinitionRegistry.from_file(
|
||||
REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
)
|
||||
portable_lab_v1_definition = next(
|
||||
definition
|
||||
for definition in portable_definition_registry.definitions
|
||||
if definition.setup_id == PORTABLE_LAB_V1_SETUP_ID
|
||||
)
|
||||
portable_source_capability_service = RecordedK1SourceAdmissionService(
|
||||
data_dir=session_store.data_dir,
|
||||
session_store=session_store,
|
||||
media_inspector=session_recorded_media_inspector,
|
||||
requirements=portable_lab_v1_definition.to_source_admission_requirements(),
|
||||
)
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableLabV1SetupProjector(
|
||||
registry=portable_definition_registry,
|
||||
capability_probe=portable_source_capability_service,
|
||||
)
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None
|
||||
except (
|
||||
PortableRunDefinitionRegistryError,
|
||||
PortableSetupProjectionError,
|
||||
OSError,
|
||||
StopIteration,
|
||||
ValueError,
|
||||
) as exc:
|
||||
# Portable LAB V1 is an optional observation-only slice. A drifted
|
||||
# registry cannot affect K1, Simulation, legacy LAB, or the exact M49 queue.
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = None
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = str(exc)
|
||||
_ffmpeg = _resolve_media_tool("ffmpeg")
|
||||
_ffprobe = _resolve_media_tool("ffprobe")
|
||||
session_recorded_camera_frame_service = (
|
||||
@@ -765,8 +841,23 @@ app.include_router(
|
||||
recorded_binding_service=OBSERVATORY_RECORDED_BINDING_SERVICE,
|
||||
recorded_job_queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
recorded_job_queue_error=OBSERVATORY_RECORDED_JOB_QUEUE_ERROR,
|
||||
portable_setup_projector=OBSERVATORY_PORTABLE_SETUP_PROJECTOR,
|
||||
portable_setup_projector_error=OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR,
|
||||
)
|
||||
)
|
||||
if (
|
||||
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED
|
||||
and OBSERVATORY_WORKER_CLAIM_LEASE_READY
|
||||
and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY
|
||||
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||
and OBSERVATORY_WORKER_AUTHENTICATION is not None
|
||||
):
|
||||
app.include_router(
|
||||
build_observatory_worker_router(
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
authentication=OBSERVATORY_WORKER_AUTHENTICATION,
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_environment_router(root_provider=lambda: session_store.data_dir / "ui-environment")
|
||||
)
|
||||
@@ -1066,10 +1157,7 @@ app.include_router(
|
||||
spatial_evidence_provider=m48_raw_evidence_reader,
|
||||
evaluation_runner=LABORATORY_RUNNER,
|
||||
evaluation_receipt_root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "laboratory-run-receipts"
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "laboratory-run-receipts"
|
||||
),
|
||||
)
|
||||
)
|
||||
@@ -1093,33 +1181,21 @@ app.include_router(
|
||||
app.include_router(
|
||||
build_m49_tgs_fail_closed_router(
|
||||
root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "m49"
|
||||
/ "tgs-fail-closed-results"
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m49" / "tgs-fail-closed-results"
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_m49_tgs_full_shadow_router(
|
||||
root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "m49"
|
||||
/ "tgs-full-shadow-results"
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m49" / "tgs-full-shadow-results"
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_vegetation_shadow_lab_router(
|
||||
root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "lab-v1-vegetation"
|
||||
/ "results"
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "lab-v1-vegetation" / "results"
|
||||
),
|
||||
canonical_recording_provider=_canonical_lab_recording_source,
|
||||
camera_frame_provider=(
|
||||
@@ -1128,9 +1204,7 @@ app.include_router(
|
||||
else None
|
||||
),
|
||||
jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs",
|
||||
rerun_overlay_cache_root=(
|
||||
session_store.data_dir / "laboratory-rerun-overlays"
|
||||
),
|
||||
rerun_overlay_cache_root=(session_store.data_dir / "laboratory-rerun-overlays"),
|
||||
ffmpeg_path=_ffmpeg,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -22,6 +22,10 @@ from k1link.observatory.m49_queue_binding import (
|
||||
M49QueueBindingIntegrityError,
|
||||
M49RecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
PortableLabV1SetupProjector,
|
||||
PortableSetupProjectionError,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueCapacityError,
|
||||
@@ -32,24 +36,24 @@ from k1link.observatory.recorded_jobs import (
|
||||
from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore
|
||||
from k1link.sessions.models import SessionSummary
|
||||
|
||||
OBSERVATORY_PROJECTION_SCHEMA: Literal[
|
||||
OBSERVATORY_PROJECTION_SCHEMA: Literal["missioncore.observatory-lab-projection/v1"] = (
|
||||
"missioncore.observatory-lab-projection/v1"
|
||||
] = "missioncore.observatory-lab-projection/v1"
|
||||
OBSERVATORY_RENAME_SCHEMA: Literal[
|
||||
)
|
||||
OBSERVATORY_RENAME_SCHEMA: Literal["missioncore.observatory-lab-projection-rename/v1"] = (
|
||||
"missioncore.observatory-lab-projection-rename/v1"
|
||||
] = "missioncore.observatory-lab-projection-rename/v1"
|
||||
)
|
||||
OBSERVATORY_RUN_PREFLIGHT_REQUEST_SCHEMA: Literal[
|
||||
"missioncore.observatory-run-preflight-request/v1"
|
||||
] = "missioncore.observatory-run-preflight-request/v1"
|
||||
OBSERVATORY_RUN_PREFLIGHT_SCHEMA: Literal[
|
||||
OBSERVATORY_RUN_PREFLIGHT_SCHEMA: Literal["missioncore.observatory-run-preflight/v1"] = (
|
||||
"missioncore.observatory-run-preflight/v1"
|
||||
] = "missioncore.observatory-run-preflight/v1"
|
||||
)
|
||||
OBSERVATORY_RECORDED_RUN_SUBMIT_SCHEMA: Literal[
|
||||
"missioncore.observatory-recorded-run-submit/v1"
|
||||
] = "missioncore.observatory-recorded-run-submit/v1"
|
||||
OBSERVATORY_RECORDED_JOB_LIST_SCHEMA: Literal[
|
||||
OBSERVATORY_RECORDED_JOB_LIST_SCHEMA: Literal["missioncore.observatory-recorded-job-list/v1"] = (
|
||||
"missioncore.observatory-recorded-job-list/v1"
|
||||
] = "missioncore.observatory-recorded-job-list/v1"
|
||||
)
|
||||
|
||||
_OBSERVATION_ONLY_AUTHORITY: dict[str, bool] = {
|
||||
"commands_enabled": False,
|
||||
@@ -64,9 +68,7 @@ class _StrictApiModel(BaseModel):
|
||||
|
||||
|
||||
class ObservatoryProjectionRenameRequest(_StrictApiModel):
|
||||
schema_version: Literal[
|
||||
"missioncore.observatory-lab-projection-rename/v1"
|
||||
]
|
||||
schema_version: Literal["missioncore.observatory-lab-projection-rename/v1"]
|
||||
display_name: str = Field(min_length=1, max_length=160)
|
||||
|
||||
|
||||
@@ -92,9 +94,7 @@ class ObservatoryRunPreflightRequest(_StrictApiModel):
|
||||
|
||||
|
||||
class ObservatoryRunPreparationRequest(_StrictApiModel):
|
||||
schema_version: Literal[
|
||||
"missioncore.observatory-run-preparation-request/v1"
|
||||
]
|
||||
schema_version: Literal["missioncore.observatory-run-preparation-request/v1"]
|
||||
idempotency_key: str = Field(
|
||||
min_length=1,
|
||||
max_length=160,
|
||||
@@ -142,6 +142,8 @@ def build_observatory_router(
|
||||
recorded_binding_service: M49RecordedQueueBindingService | None = None,
|
||||
recorded_job_queue: ObservatoryRecordedJobQueue | None = None,
|
||||
recorded_job_queue_error: str | None = None,
|
||||
portable_setup_projector: PortableLabV1SetupProjector | None = None,
|
||||
portable_setup_projector_error: str | None = None,
|
||||
) -> APIRouter:
|
||||
"""Build bounded catalog-only mutations for typed Observatory projections."""
|
||||
|
||||
@@ -212,6 +214,41 @@ def build_observatory_router(
|
||||
available.add(result_id)
|
||||
return frozenset(available)
|
||||
|
||||
if portable_setup_projector is not None:
|
||||
|
||||
@router.get("/api/v1/observatory/portable-laboratory-setups")
|
||||
def list_observatory_portable_laboratory_setups(
|
||||
source_session_id: str = Query(
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||
),
|
||||
) -> dict[str, object]:
|
||||
source = source_summary(source_session_id)
|
||||
try:
|
||||
return portable_setup_projector.catalog(source)
|
||||
except PortableSetupProjectionError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Portable-каталог LAB V1 нарушил контракт целостности.",
|
||||
) from exc
|
||||
|
||||
elif portable_setup_projector_error is not None:
|
||||
|
||||
@router.get("/api/v1/observatory/portable-laboratory-setups")
|
||||
def unavailable_observatory_portable_laboratory_setups(
|
||||
source_session_id: str = Query(
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||
),
|
||||
) -> None:
|
||||
del source_session_id
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Portable-каталог LAB V1 недоступен.",
|
||||
)
|
||||
|
||||
if setup_registry is not None:
|
||||
|
||||
@router.get("/api/v1/observatory/laboratory-setups")
|
||||
@@ -225,9 +262,7 @@ def build_observatory_router(
|
||||
source = source_summary(source_session_id)
|
||||
return setup_registry.catalog(
|
||||
source,
|
||||
available_observatory_result_ids=available_observatory_results(
|
||||
source_session_id
|
||||
),
|
||||
available_observatory_result_ids=available_observatory_results(source_session_id),
|
||||
)
|
||||
|
||||
@router.post("/api/v1/observatory/run-preflights")
|
||||
@@ -254,9 +289,7 @@ def build_observatory_router(
|
||||
)
|
||||
definition = projected["run_definition"]
|
||||
expected_digest = (
|
||||
definition.get("definition_sha256")
|
||||
if isinstance(definition, dict)
|
||||
else None
|
||||
definition.get("definition_sha256") if isinstance(definition, dict) else None
|
||||
)
|
||||
if request.definition_sha256 != expected_digest:
|
||||
raise HTTPException(
|
||||
@@ -276,8 +309,7 @@ def build_observatory_router(
|
||||
binding_service = recorded_binding_service
|
||||
exact_queue_setup = (
|
||||
binding_service is not None
|
||||
and request.setup_id
|
||||
== binding_service.config.setup.setup_id
|
||||
and request.setup_id == binding_service.config.setup.setup_id
|
||||
)
|
||||
if (
|
||||
compatible
|
||||
@@ -303,9 +335,7 @@ def build_observatory_router(
|
||||
{
|
||||
"check_id": "source-compatibility",
|
||||
"outcome": "pass" if compatible else "fail",
|
||||
"reason_code": (
|
||||
"source-compatible" if compatible else "source-incompatible"
|
||||
),
|
||||
"reason_code": ("source-compatible" if compatible else "source-incompatible"),
|
||||
"message": (
|
||||
"Источник точно совместим с сохранённым сетапом."
|
||||
if compatible
|
||||
@@ -323,11 +353,7 @@ def build_observatory_router(
|
||||
{
|
||||
"check_id": "executor",
|
||||
"outcome": (
|
||||
"not-applicable"
|
||||
if existing
|
||||
else "pass"
|
||||
if queue_binding_ready
|
||||
else "fail"
|
||||
"not-applicable" if existing else "pass" if queue_binding_ready else "fail"
|
||||
),
|
||||
"reason_code": (
|
||||
"existing-result-does-not-require-executor"
|
||||
@@ -349,13 +375,7 @@ def build_observatory_router(
|
||||
},
|
||||
{
|
||||
"check_id": "durable-queue",
|
||||
"outcome": (
|
||||
"not-applicable"
|
||||
if existing
|
||||
else "pass"
|
||||
if queueable
|
||||
else "fail"
|
||||
),
|
||||
"outcome": ("not-applicable" if existing else "pass" if queueable else "fail"),
|
||||
"reason_code": (
|
||||
"existing-result-does-not-require-queue"
|
||||
if existing
|
||||
@@ -378,9 +398,7 @@ def build_observatory_router(
|
||||
"source_session_id": request.source_session_id,
|
||||
"setup_id": request.setup_id,
|
||||
"definition_sha256": expected_digest,
|
||||
"outcome": (
|
||||
"existing" if existing else "queueable" if queueable else "blocked"
|
||||
),
|
||||
"outcome": ("existing" if existing else "queueable" if queueable else "blocked"),
|
||||
"submission_allowed": queueable,
|
||||
"checks": checks,
|
||||
"existing_result_ids": preflight.get("existing_result_ids", []),
|
||||
@@ -421,9 +439,7 @@ def build_observatory_router(
|
||||
request: ObservatoryRunPreparationRequest,
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
existing = run_preparation_ledger.get_by_idempotency_key(
|
||||
request.idempotency_key
|
||||
)
|
||||
existing = run_preparation_ledger.get_by_idempotency_key(request.idempotency_key)
|
||||
request_sha256 = observatory_run_preparation_request_sha256(
|
||||
idempotency_key=request.idempotency_key,
|
||||
source_session_id=request.source_session_id,
|
||||
@@ -634,9 +650,7 @@ def build_observatory_router(
|
||||
request: ObservatoryRecordedRunSubmitRequest,
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
existing_job = recorded_job_queue.get_by_idempotency_key(
|
||||
request.idempotency_key
|
||||
)
|
||||
existing_job = recorded_job_queue.get_by_idempotency_key(request.idempotency_key)
|
||||
except ObservatoryRecordedQueueNotFoundError:
|
||||
existing_job = None
|
||||
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
"""Authenticated pull transport for the durable Observatory recorded-job queue.
|
||||
|
||||
The transport is deliberately narrower than an execution API. Worker callers
|
||||
can claim a server-sealed RunDefinition and advance its durable state, but they
|
||||
cannot supply commands, paths, environment variables, container images, or
|
||||
priority. Those execution identities remain part of the queue-owned job.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Final, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Response
|
||||
from fastapi import Path as ApiPath
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedCheckpointError,
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedPreemptionError,
|
||||
ObservatoryRecordedQueueBusyError,
|
||||
ObservatoryRecordedQueueCapacityError,
|
||||
ObservatoryRecordedQueueConflictError,
|
||||
ObservatoryRecordedQueueError,
|
||||
ObservatoryRecordedQueueIntegrityError,
|
||||
ObservatoryRecordedQueueNotFoundError,
|
||||
ObservatoryRecordedQueueStaleClaimError,
|
||||
)
|
||||
|
||||
OBSERVATORY_WORKER_CLAIM_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-claim-request/v1"
|
||||
OBSERVATORY_WORKER_START_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-start-request/v1"
|
||||
OBSERVATORY_WORKER_CHECKPOINT_REQUEST_SCHEMA: Final = (
|
||||
"missioncore.observatory-worker-checkpoint-request/v1"
|
||||
)
|
||||
OBSERVATORY_WORKER_SUCCEED_REQUEST_SCHEMA: Final = (
|
||||
"missioncore.observatory-worker-succeed-request/v1"
|
||||
)
|
||||
OBSERVATORY_WORKER_FAIL_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-fail-request/v1"
|
||||
OBSERVATORY_WORKER_CONTOUR_HEADER: Final = "X-Mission-Core-Contour-Id"
|
||||
|
||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_JOB_ID_PATTERN = r"^observatory-run-[a-f0-9]{32}$"
|
||||
_CLAIM_TOKEN_PATTERN = r"^[a-f0-9]{64}$"
|
||||
_CLAIM_REQUEST_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$"
|
||||
_SESSION_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"
|
||||
_IDENTIFIER_PATTERN = r"^[a-z][a-z0-9-]{2,95}$"
|
||||
_WORKER_BEARER = HTTPBearer(auto_error=False)
|
||||
_TOKEN = re.compile(r"^[A-Za-z0-9._:-]{32,512}$")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerAuthentication:
|
||||
"""Server-owned Worker identity and the SHA-256 of its bearer secret."""
|
||||
|
||||
bearer_token_sha256: str
|
||||
contour_id: str = "worker-006"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if _SHA256.fullmatch(self.bearer_token_sha256) is None:
|
||||
raise ValueError("Worker bearer token SHA-256 is invalid")
|
||||
if _IDENTIFIER.fullmatch(self.contour_id) is None:
|
||||
raise ValueError("Worker contour id is invalid")
|
||||
|
||||
|
||||
def load_observatory_worker_authentication(
|
||||
token_path: Path,
|
||||
*,
|
||||
contour_id: str = "worker-006",
|
||||
) -> ObservatoryWorkerAuthentication:
|
||||
"""Load one local Worker credential without retaining its plaintext.
|
||||
|
||||
The credential file is an operator/deployment concern. Mission Core only
|
||||
retains its SHA-256 in the router configuration and refuses symlinks,
|
||||
non-regular files, or group/other permissions.
|
||||
"""
|
||||
|
||||
candidate = token_path.expanduser().absolute()
|
||||
descriptor: int | None = None
|
||||
try:
|
||||
descriptor = os.open(
|
||||
candidate,
|
||||
os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0),
|
||||
)
|
||||
metadata = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(metadata.st_mode):
|
||||
raise ValueError("Worker bearer credential must be a regular file")
|
||||
if metadata.st_mode & 0o077:
|
||||
raise ValueError("Worker bearer credential permissions are too broad")
|
||||
if not 32 <= metadata.st_size <= 512:
|
||||
raise ValueError("Worker bearer credential format is invalid")
|
||||
with os.fdopen(descriptor, "rb") as stream:
|
||||
descriptor = None
|
||||
payload = stream.read(513)
|
||||
except ValueError:
|
||||
raise
|
||||
except OSError as exc:
|
||||
raise ValueError("Worker bearer credential is unavailable") from exc
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
try:
|
||||
token = payload.decode("ascii")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ValueError("Worker bearer credential is not ASCII") from exc
|
||||
if _TOKEN.fullmatch(token) is None:
|
||||
raise ValueError("Worker bearer credential format is invalid")
|
||||
return ObservatoryWorkerAuthentication(
|
||||
bearer_token_sha256=hashlib.sha256(payload).hexdigest(),
|
||||
contour_id=contour_id,
|
||||
)
|
||||
|
||||
|
||||
class _StrictWorkerRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
|
||||
class ObservatoryWorkerClaimRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-claim-request/v1"]
|
||||
claim_request_id: str = Field(
|
||||
min_length=1,
|
||||
max_length=160,
|
||||
pattern=_CLAIM_REQUEST_ID_PATTERN,
|
||||
)
|
||||
|
||||
|
||||
class ObservatoryWorkerStartRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-start-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
|
||||
|
||||
class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
checkpoint_id: str = Field(
|
||||
min_length=3,
|
||||
max_length=96,
|
||||
pattern=_IDENTIFIER_PATTERN,
|
||||
)
|
||||
|
||||
|
||||
class ObservatoryWorkerSucceedRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-succeed-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
result_id: str = Field(
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=_SESSION_ID_PATTERN,
|
||||
)
|
||||
result_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
|
||||
class ObservatoryWorkerFailRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-fail-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
error_code: str = Field(
|
||||
min_length=3,
|
||||
max_length=96,
|
||||
pattern=_IDENTIFIER_PATTERN,
|
||||
)
|
||||
message: str = Field(min_length=1, max_length=1_000)
|
||||
|
||||
|
||||
def build_observatory_worker_router(
|
||||
queue: ObservatoryRecordedJobQueue,
|
||||
*,
|
||||
authentication: ObservatoryWorkerAuthentication,
|
||||
) -> APIRouter:
|
||||
"""Build the bounded Worker pull/state-transition router.
|
||||
|
||||
``authentication`` contains only a token digest. The plaintext bearer
|
||||
secret exists transiently while FastAPI parses one request, is immediately
|
||||
hashed, and is compared to the configured digest in constant time.
|
||||
"""
|
||||
|
||||
def require_configured_worker(
|
||||
credentials: Annotated[
|
||||
HTTPAuthorizationCredentials | None,
|
||||
Depends(_WORKER_BEARER),
|
||||
],
|
||||
contour_id: Annotated[
|
||||
str | None,
|
||||
Header(alias=OBSERVATORY_WORKER_CONTOUR_HEADER),
|
||||
] = None,
|
||||
) -> None:
|
||||
if credentials is None or credentials.scheme.lower() != "bearer":
|
||||
raise _unauthorized()
|
||||
token = credentials.credentials
|
||||
if not token or len(token) > 512:
|
||||
raise _unauthorized()
|
||||
supplied_sha256 = hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
if not hmac.compare_digest(
|
||||
supplied_sha256,
|
||||
authentication.bearer_token_sha256,
|
||||
):
|
||||
raise _unauthorized()
|
||||
if contour_id is None or not hmac.compare_digest(
|
||||
contour_id,
|
||||
authentication.contour_id,
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Worker contour identity was rejected.",
|
||||
)
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/v1/worker/observatory",
|
||||
tags=["observatory-worker"],
|
||||
dependencies=[Depends(require_configured_worker)],
|
||||
)
|
||||
|
||||
@router.post("/recorded-jobs/claims", response_model=None)
|
||||
def claim_next(
|
||||
request: ObservatoryWorkerClaimRequest,
|
||||
) -> dict[str, object] | Response:
|
||||
claim = _queue_call(
|
||||
lambda: queue.claim_next(
|
||||
claimant_id=authentication.contour_id,
|
||||
claim_request_id=request.claim_request_id,
|
||||
)
|
||||
)
|
||||
if claim is None:
|
||||
return Response(status_code=204)
|
||||
return claim.as_dict()
|
||||
|
||||
@router.get("/recorded-jobs/{job_id}")
|
||||
def get_job(
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(lambda: queue.get(job_id)).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/start")
|
||||
def start_job(
|
||||
request: ObservatoryWorkerStartRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(lambda: queue.start(job_id, claim_token=request.claim_token)).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/checkpoint")
|
||||
def checkpoint_job(
|
||||
request: ObservatoryWorkerCheckpointRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(
|
||||
lambda: queue.checkpoint(
|
||||
job_id,
|
||||
claim_token=request.claim_token,
|
||||
checkpoint_id=request.checkpoint_id,
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/succeed")
|
||||
def succeed_job(
|
||||
request: ObservatoryWorkerSucceedRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(
|
||||
lambda: queue.succeed(
|
||||
job_id,
|
||||
claim_token=request.claim_token,
|
||||
result_id=request.result_id,
|
||||
result_sha256=request.result_sha256,
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/fail")
|
||||
def fail_job(
|
||||
request: ObservatoryWorkerFailRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(
|
||||
lambda: queue.fail(
|
||||
job_id,
|
||||
claim_token=request.claim_token,
|
||||
error_code=request.error_code,
|
||||
message=request.message,
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
return router
|
||||
|
||||
|
||||
def _unauthorized() -> HTTPException:
|
||||
return HTTPException(
|
||||
status_code=401,
|
||||
detail="Worker bearer credential was rejected.",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
|
||||
def _queue_call[T](operation: Callable[[], T]) -> T:
|
||||
try:
|
||||
return operation()
|
||||
except ObservatoryRecordedQueueNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="Recorded job was not found.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueStaleClaimError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Recorded-job claim is stale.",
|
||||
) from exc
|
||||
except ObservatoryRecordedCheckpointError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Recorded-job checkpoint was rejected.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueConflictError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Recorded-job transition conflicts with durable state.",
|
||||
) from exc
|
||||
except (ObservatoryRecordedQueueBusyError, ObservatoryRecordedPreemptionError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Recorded-job resources are reserved for live K1 work.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueCapacityError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Recorded-job queue capacity is unavailable.",
|
||||
headers={"Retry-After": "5"},
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueIntegrityError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Recorded-job queue integrity is unavailable.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Recorded-job queue is unavailable.",
|
||||
) from exc
|
||||
@@ -0,0 +1,574 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.portable_queue_binding import (
|
||||
PortableQueueBindingIntegrityError,
|
||||
PortableQueueBindingStaleCheckError,
|
||||
PortableRecordedQueueBindingService,
|
||||
PortableRecordedRunPreparation,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableExecutorAvailability,
|
||||
PortableRunDefinitionRegistry,
|
||||
PortableRunDefinitionUnavailableError,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import ObservatoryRecordedJobQueue
|
||||
from k1link.observatory.source_admission import (
|
||||
PORTABLE_SOURCE_DOCUMENT_DIRECTORY,
|
||||
PortableRecordedSourceAdmission,
|
||||
PortableSourceNotPreparedError,
|
||||
RecordedK1SourceAdmissionService,
|
||||
)
|
||||
from k1link.sessions.media import (
|
||||
CAMERA_ARCHIVE_SCHEMA,
|
||||
RecordedMediaEpoch,
|
||||
RecordedMediaManifest,
|
||||
RecordedMediaSegment,
|
||||
)
|
||||
from k1link.sessions.models import (
|
||||
RecordedMediaArtifact,
|
||||
ReplayArtifact,
|
||||
ReplayCommand,
|
||||
SessionArtifact,
|
||||
SessionDetail,
|
||||
SessionSource,
|
||||
SessionSummary,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
SESSION_ID = "20260831T000000Z_viewer_live"
|
||||
CATALOG_SHA256 = "1" * 64
|
||||
RAW_SHA256 = "2" * 64
|
||||
GENERATION_SHA256 = "3" * 64
|
||||
SEGMENT_SHA256 = "4" * 64
|
||||
INIT_SHA256 = "e2279963e16d84c91d68e7dbb1f7efed840533387dfeb844b7398bff45fbde38"
|
||||
|
||||
|
||||
def _blocked_registry() -> PortableRunDefinitionRegistry:
|
||||
return PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||
|
||||
|
||||
def _ready_registry() -> PortableRunDefinitionRegistry:
|
||||
blocked = _blocked_registry().definitions[0]
|
||||
executor = PortableExecutorAvailability(
|
||||
contour_id="worker-006",
|
||||
state="ready",
|
||||
release_id="lab-v1-eomt-ddrnet-executor-v1",
|
||||
release_sha256="5" * 64,
|
||||
image_sha256="6" * 64,
|
||||
reason_code=None,
|
||||
reason=None,
|
||||
)
|
||||
identity = blocked.identity_document()
|
||||
identity["executor"] = executor.identity_document()
|
||||
ready = replace(
|
||||
blocked,
|
||||
executor=executor,
|
||||
definition_sha256=canonical_sha256(identity),
|
||||
)
|
||||
return PortableRunDefinitionRegistry((ready,))
|
||||
|
||||
|
||||
def _detail(*, display_name: str = "arbitrary-operator-label") -> SessionDetail:
|
||||
return SessionDetail(
|
||||
summary=SessionSummary(
|
||||
session_id=SESSION_ID,
|
||||
display_name=display_name,
|
||||
status="ready",
|
||||
started_at_utc="2026-08-31T00:00:00Z",
|
||||
completed_at_utc="2026-08-31T00:10:00Z",
|
||||
duration_seconds=600.0,
|
||||
modalities=("point-cloud", "trajectory", "video"),
|
||||
source_count=3,
|
||||
total_bytes=120,
|
||||
replayable=True,
|
||||
origin="xgrids-k1.viewer-live.evidence",
|
||||
),
|
||||
sources=(
|
||||
SessionSource(
|
||||
source_id="sensor.camera.right",
|
||||
semantic_channel_id="camera.video.recorded",
|
||||
modality="video",
|
||||
status="recorded",
|
||||
seekable=True,
|
||||
artifact_id="recorded-video-right",
|
||||
),
|
||||
SessionSource(
|
||||
source_id="sensor.lidar.primary",
|
||||
semantic_channel_id="spatial.point-cloud.recorded",
|
||||
modality="point-cloud",
|
||||
status="recorded",
|
||||
seekable=True,
|
||||
artifact_id="raw-transport-primary",
|
||||
),
|
||||
SessionSource(
|
||||
source_id="spatial.trajectory",
|
||||
semantic_channel_id="spatial.pose.recorded",
|
||||
modality="trajectory",
|
||||
status="recorded",
|
||||
seekable=True,
|
||||
artifact_id="raw-transport-primary",
|
||||
),
|
||||
),
|
||||
artifacts=(
|
||||
SessionArtifact(
|
||||
artifact_id="raw-transport-primary",
|
||||
kind="raw-transport",
|
||||
media_type="application/x-nodedc-k1mqtt",
|
||||
byte_length=100,
|
||||
sha256=RAW_SHA256,
|
||||
integrity_status="verified",
|
||||
),
|
||||
SessionArtifact(
|
||||
artifact_id="recorded-video-right",
|
||||
kind="recorded-video",
|
||||
media_type="video/mp4",
|
||||
byte_length=20,
|
||||
sha256=None,
|
||||
integrity_status="validated-structure",
|
||||
),
|
||||
),
|
||||
plugin_id="nodedc.device.xgrids-lixelkity-k1",
|
||||
archive_id="xgrids-k1.viewer-live.evidence",
|
||||
)
|
||||
|
||||
|
||||
def _replay(root: Path) -> ReplayCommand:
|
||||
return ReplayCommand(
|
||||
session_id=SESSION_ID,
|
||||
plugin_id="nodedc.device.xgrids-lixelkity-k1",
|
||||
allowed_root=root,
|
||||
session_root=root / SESSION_ID,
|
||||
primary_artifact_id="raw-transport-primary",
|
||||
artifacts=(
|
||||
ReplayArtifact(
|
||||
artifact_id="raw-transport-primary",
|
||||
path=root / "mqtt.raw.k1mqtt",
|
||||
media_type="application/x-nodedc-k1mqtt",
|
||||
file_byte_length=100,
|
||||
replay_byte_length=100,
|
||||
expected_sha256=RAW_SHA256,
|
||||
),
|
||||
),
|
||||
timeline_origin_epoch_ns=1,
|
||||
timeline_origin_monotonic_ns=2,
|
||||
speed=1.0,
|
||||
loop=False,
|
||||
)
|
||||
|
||||
|
||||
def _media_artifact(root: Path) -> RecordedMediaArtifact:
|
||||
return RecordedMediaArtifact(
|
||||
session_id=SESSION_ID,
|
||||
public_source_id="recorded.camera.right",
|
||||
artifact_id="recorded-video-right",
|
||||
source_path=root / "media" / "sensor.camera.right",
|
||||
byte_length=20,
|
||||
)
|
||||
|
||||
|
||||
def _manifest(root: Path) -> RecordedMediaManifest:
|
||||
epoch = RecordedMediaEpoch(
|
||||
ordinal=1,
|
||||
path=root / "epoch-1",
|
||||
init_path=root / "epoch-1" / "init.mp4",
|
||||
init_byte_length=10,
|
||||
init_sha256=INIT_SHA256,
|
||||
media_type='video/mp4; codecs="avc1.641028"',
|
||||
timeline_start_seconds=10.0,
|
||||
timeline_end_seconds=10.1,
|
||||
segments=(
|
||||
RecordedMediaSegment(
|
||||
sequence=1,
|
||||
path=root / "epoch-1" / "segments" / "1.m4s",
|
||||
byte_length=10,
|
||||
sha256=SEGMENT_SHA256,
|
||||
random_access=True,
|
||||
end_time_seconds=10.1,
|
||||
),
|
||||
),
|
||||
)
|
||||
return RecordedMediaManifest(
|
||||
session_id=SESSION_ID,
|
||||
public_source_id="recorded.camera.right",
|
||||
artifact_id="recorded-video-right",
|
||||
synchronization="host-arrival-best-effort",
|
||||
generation_sha256=GENERATION_SHA256,
|
||||
timeline_start_seconds=10.0,
|
||||
timeline_end_seconds=10.1,
|
||||
byte_length=20,
|
||||
epochs=(epoch,),
|
||||
)
|
||||
|
||||
|
||||
class _Store:
|
||||
def __init__(
|
||||
self,
|
||||
root: Path,
|
||||
*,
|
||||
catalogs: tuple[str, ...] = (CATALOG_SHA256,),
|
||||
) -> None:
|
||||
self.data_dir = root.resolve()
|
||||
self.detail = _detail()
|
||||
self.replay = _replay(root)
|
||||
self.media = (_media_artifact(root),)
|
||||
self.catalogs = catalogs
|
||||
self.catalog_reads = 0
|
||||
self.prepare_replay_calls = 0
|
||||
self.recorded_media_reads = 0
|
||||
|
||||
def get_session_with_catalog_snapshot(
|
||||
self,
|
||||
session_id: str,
|
||||
) -> tuple[SessionDetail, str]:
|
||||
assert session_id == SESSION_ID
|
||||
index = min(self.catalog_reads, len(self.catalogs) - 1)
|
||||
self.catalog_reads += 1
|
||||
return self.detail, self.catalogs[index]
|
||||
|
||||
def prepare_replay(self, session_id: str) -> ReplayCommand:
|
||||
assert session_id == SESSION_ID
|
||||
self.prepare_replay_calls += 1
|
||||
return self.replay
|
||||
|
||||
def list_recorded_media(
|
||||
self,
|
||||
session_id: str,
|
||||
) -> tuple[RecordedMediaArtifact, ...]:
|
||||
assert session_id == SESSION_ID
|
||||
self.recorded_media_reads += 1
|
||||
return self.media
|
||||
|
||||
|
||||
class _Inspector:
|
||||
def __init__(self, manifest: RecordedMediaManifest | None) -> None:
|
||||
self.manifest = manifest
|
||||
self.inspect_calls = 0
|
||||
self.restore_calls = 0
|
||||
|
||||
def inspect(
|
||||
self,
|
||||
artifact: RecordedMediaArtifact,
|
||||
replay: ReplayCommand,
|
||||
) -> RecordedMediaManifest:
|
||||
assert artifact.session_id == replay.session_id
|
||||
self.inspect_calls += 1
|
||||
assert self.manifest is not None
|
||||
return self.manifest
|
||||
|
||||
def restore_prepared(
|
||||
self,
|
||||
artifact: RecordedMediaArtifact,
|
||||
replay: ReplayCommand,
|
||||
) -> RecordedMediaManifest | None:
|
||||
assert artifact.session_id == replay.session_id
|
||||
self.restore_calls += 1
|
||||
return self.manifest
|
||||
|
||||
|
||||
def _write_probe_summary(root: Path, *, segment_count: int = 11) -> None:
|
||||
epoch = root / "media" / "sensor.camera.right" / "epoch-1"
|
||||
epoch.mkdir(parents=True)
|
||||
(epoch / "summary.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": CAMERA_ARCHIVE_SCHEMA,
|
||||
"source_id": "sensor.camera.right",
|
||||
"codec_epoch": 1,
|
||||
"status": "complete",
|
||||
"segment_count": segment_count,
|
||||
"entry_count": segment_count,
|
||||
"media_segment_count": segment_count,
|
||||
"init_sha256": INIT_SHA256,
|
||||
"synchronization": "host-arrival-best-effort",
|
||||
"commit_policy": "per-segment-fsync",
|
||||
"failure_code": None,
|
||||
"artifacts": {
|
||||
"init": "init.mp4",
|
||||
"segments": "segments",
|
||||
"index": "index.jsonl",
|
||||
},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _service(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
registry: PortableRunDefinitionRegistry,
|
||||
store: _Store | None = None,
|
||||
inspector: _Inspector | None = None,
|
||||
with_queue: bool = True,
|
||||
) -> tuple[
|
||||
PortableRecordedQueueBindingService,
|
||||
_Store,
|
||||
ObservatoryRecordedJobQueue | None,
|
||||
_Inspector,
|
||||
]:
|
||||
active_store = store or _Store(tmp_path)
|
||||
queue = None
|
||||
if with_queue:
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=registry.to_recorded_registry(),
|
||||
clock=lambda: "2026-08-31T00:00:00.000Z",
|
||||
)
|
||||
active_inspector = inspector or _Inspector(_manifest(tmp_path))
|
||||
service = PortableRecordedQueueBindingService(
|
||||
data_dir=tmp_path,
|
||||
session_store=active_store, # type: ignore[arg-type]
|
||||
media_inspector=active_inspector, # type: ignore[arg-type]
|
||||
definitions=registry,
|
||||
queue=queue,
|
||||
)
|
||||
return service, active_store, queue, active_inspector
|
||||
|
||||
|
||||
def _check(
|
||||
service: PortableRecordedQueueBindingService,
|
||||
registry: PortableRunDefinitionRegistry,
|
||||
) -> PortableRecordedRunPreparation:
|
||||
definition = registry.definitions[0]
|
||||
return service.check(
|
||||
source_session_id=SESSION_ID,
|
||||
setup_id=definition.setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
)
|
||||
|
||||
|
||||
def test_not_installed_definition_fails_before_source_or_queue_writes(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry = _blocked_registry()
|
||||
service, store, queue, _inspector = _service(
|
||||
tmp_path,
|
||||
registry=registry,
|
||||
with_queue=False,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
PortableRunDefinitionUnavailableError,
|
||||
match="not sealed or installed",
|
||||
):
|
||||
_check(service, registry)
|
||||
|
||||
assert store.catalog_reads == 0
|
||||
assert queue is None
|
||||
assert not (tmp_path / PORTABLE_SOURCE_DOCUMENT_DIRECTORY).exists()
|
||||
assert not (tmp_path / "observatory-recorded-jobs.sqlite3").exists()
|
||||
|
||||
|
||||
def test_probe_is_bounded_and_does_not_enter_replay_or_media_inspector(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry = _ready_registry()
|
||||
_write_probe_summary(tmp_path, segment_count=31)
|
||||
service, store, queue, inspector = _service(tmp_path, registry=registry)
|
||||
definition = registry.definitions[0]
|
||||
|
||||
capability = service.probe(
|
||||
source_session_id=SESSION_ID,
|
||||
setup_id=definition.setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
)
|
||||
|
||||
assert capability.source_session_id == SESSION_ID
|
||||
assert capability.source_catalog_sha256 == CATALOG_SHA256
|
||||
assert capability.source_adapter_sha256 == definition.source_adapter.contract_sha256
|
||||
assert capability.camera_segment_count == 31
|
||||
assert store.catalog_reads == 1
|
||||
assert store.recorded_media_reads == 1
|
||||
assert store.prepare_replay_calls == 0
|
||||
assert inspector.inspect_calls == 0
|
||||
assert inspector.restore_calls == 0
|
||||
assert not (tmp_path / "media" / "sensor.camera.right" / "epoch-1" / "segments").exists()
|
||||
assert not (tmp_path / PORTABLE_SOURCE_DOCUMENT_DIRECTORY).exists()
|
||||
assert queue is not None
|
||||
assert queue.list_jobs() == ()
|
||||
|
||||
|
||||
def test_check_fails_typed_when_media_was_never_prepared(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry = _ready_registry()
|
||||
empty_inspector = _Inspector(None)
|
||||
service, _, queue, inspector = _service(
|
||||
tmp_path,
|
||||
registry=registry,
|
||||
inspector=empty_inspector,
|
||||
)
|
||||
|
||||
with pytest.raises(PortableSourceNotPreparedError, match="not been prepared"):
|
||||
_check(service, registry)
|
||||
|
||||
assert inspector.inspect_calls == 0
|
||||
assert inspector.restore_calls == 1
|
||||
assert not (tmp_path / PORTABLE_SOURCE_DOCUMENT_DIRECTORY).exists()
|
||||
assert queue is not None
|
||||
assert queue.list_jobs() == ()
|
||||
|
||||
|
||||
def test_check_is_path_free_read_only_and_builds_exact_queue_intent(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry = _ready_registry()
|
||||
service, _, queue, inspector = _service(tmp_path, registry=registry)
|
||||
|
||||
checked = _check(service, registry)
|
||||
intent = checked.intent(idempotency_key="portable-run-001")
|
||||
|
||||
assert intent.source_session_id == SESSION_ID
|
||||
assert intent.source_catalog_sha256 == checked.source.source_catalog_sha256
|
||||
assert intent.source_bundle_sha256 == checked.source.source_bundle_sha256
|
||||
assert intent.source_capability_manifest_sha256 == (
|
||||
checked.source.source_capability_manifest_sha256
|
||||
)
|
||||
assert intent.setup_id == registry.definitions[0].setup_id
|
||||
assert intent.definition_sha256 == registry.definitions[0].definition_sha256
|
||||
assert checked.source.source_adapter_sha256 == (
|
||||
registry.definitions[0].source_adapter.contract_sha256
|
||||
)
|
||||
assert "path" not in str(checked.as_dict()).lower()
|
||||
assert not (tmp_path / PORTABLE_SOURCE_DOCUMENT_DIRECTORY).exists()
|
||||
assert queue is not None
|
||||
assert queue.list_jobs() == ()
|
||||
assert inspector.inspect_calls == 0
|
||||
assert inspector.restore_calls == 1
|
||||
|
||||
|
||||
def test_admit_persists_exact_documents_and_submit_queues_same_identity(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry = _ready_registry()
|
||||
service, _, queue, _inspector = _service(tmp_path, registry=registry)
|
||||
checked = _check(service, registry)
|
||||
definition = registry.definitions[0]
|
||||
|
||||
admitted = service.admit(
|
||||
source_session_id=SESSION_ID,
|
||||
setup_id=definition.setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
expected_check_sha256=checked.check_sha256,
|
||||
)
|
||||
job, created = service.submit(
|
||||
source_session_id=SESSION_ID,
|
||||
setup_id=definition.setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
expected_check_sha256=checked.check_sha256,
|
||||
idempotency_key="portable-run-001",
|
||||
)
|
||||
|
||||
root = tmp_path / PORTABLE_SOURCE_DOCUMENT_DIRECTORY
|
||||
assert sorted(path.name for path in root.iterdir()) == sorted(
|
||||
(
|
||||
f"{admitted.source.source_bundle_sha256}.json",
|
||||
f"{admitted.source.source_capability_manifest_sha256}.json",
|
||||
)
|
||||
)
|
||||
assert created is True
|
||||
assert job.state == "queued"
|
||||
assert job.source_catalog_sha256 == admitted.source.source_catalog_sha256
|
||||
assert job.source_bundle_sha256 == admitted.source.source_bundle_sha256
|
||||
assert job.source_capability_manifest_sha256 == (
|
||||
admitted.source.source_capability_manifest_sha256
|
||||
)
|
||||
assert queue is not None
|
||||
assert queue.list_jobs() == (job,)
|
||||
|
||||
|
||||
def test_admit_rejects_catalog_change_since_check_without_writes_or_job(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry = _ready_registry()
|
||||
store = _Store(tmp_path, catalogs=(CATALOG_SHA256, "7" * 64))
|
||||
service, _, queue, _inspector = _service(
|
||||
tmp_path,
|
||||
registry=registry,
|
||||
store=store,
|
||||
)
|
||||
checked = _check(service, registry)
|
||||
definition = registry.definitions[0]
|
||||
|
||||
with pytest.raises(PortableQueueBindingStaleCheckError, match="changed"):
|
||||
service.submit(
|
||||
source_session_id=SESSION_ID,
|
||||
setup_id=definition.setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
expected_check_sha256=checked.check_sha256,
|
||||
idempotency_key="portable-run-stale-001",
|
||||
)
|
||||
|
||||
assert not (tmp_path / PORTABLE_SOURCE_DOCUMENT_DIRECTORY).exists()
|
||||
assert queue is not None
|
||||
assert queue.list_jobs() == ()
|
||||
|
||||
|
||||
def test_admit_rejects_change_during_commit_before_source_documents(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry = _ready_registry()
|
||||
store = _Store(
|
||||
tmp_path,
|
||||
catalogs=(
|
||||
CATALOG_SHA256,
|
||||
CATALOG_SHA256,
|
||||
CATALOG_SHA256,
|
||||
"8" * 64,
|
||||
),
|
||||
)
|
||||
service, _, queue, _inspector = _service(
|
||||
tmp_path,
|
||||
registry=registry,
|
||||
store=store,
|
||||
)
|
||||
checked = _check(service, registry)
|
||||
definition = registry.definitions[0]
|
||||
|
||||
with pytest.raises(PortableQueueBindingStaleCheckError, match="changed"):
|
||||
service.submit(
|
||||
source_session_id=SESSION_ID,
|
||||
setup_id=definition.setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
expected_check_sha256=checked.check_sha256,
|
||||
idempotency_key="portable-run-racing-001",
|
||||
)
|
||||
|
||||
assert not (tmp_path / PORTABLE_SOURCE_DOCUMENT_DIRECTORY).exists()
|
||||
assert queue is not None
|
||||
assert queue.list_jobs() == ()
|
||||
|
||||
|
||||
def test_registry_and_source_admission_adapter_digests_must_match(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
registry = _ready_registry()
|
||||
service, _, queue, _inspector = _service(tmp_path, registry=registry)
|
||||
original_check = RecordedK1SourceAdmissionService.check
|
||||
|
||||
def corrupted_check(
|
||||
source_service: RecordedK1SourceAdmissionService,
|
||||
source_session_id: str,
|
||||
) -> PortableRecordedSourceAdmission:
|
||||
return replace(
|
||||
original_check(source_service, source_session_id),
|
||||
source_adapter_sha256="9" * 64,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(RecordedK1SourceAdmissionService, "check", corrupted_check)
|
||||
|
||||
with pytest.raises(PortableQueueBindingIntegrityError, match="adapter"):
|
||||
_check(service, registry)
|
||||
|
||||
assert not (tmp_path / PORTABLE_SOURCE_DOCUMENT_DIRECTORY).exists()
|
||||
assert queue is not None
|
||||
assert queue.list_jobs() == ()
|
||||
@@ -0,0 +1,322 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
from dataclasses import FrozenInstanceError, replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PORTABLE_MODEL_MANIFEST_SCHEMA,
|
||||
PortableExecutorAvailability,
|
||||
PortableRunDefinitionRegistry,
|
||||
PortableRunDefinitionRegistryError,
|
||||
PortableRunDefinitionUnavailableError,
|
||||
canonical_sha256,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
DEFINITION_SHA256 = "57bf8f0859e10e54e30322c9a8aa28b427699f6fe6b5267e279ec3390fa78466"
|
||||
MODEL_MANIFEST_SHA256 = "3fd2d43af73bd73f89d9ffae95d8770cfdeb46033ec967509124fac6ae4afe56"
|
||||
|
||||
|
||||
def _registry() -> PortableRunDefinitionRegistry:
|
||||
return PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||
|
||||
|
||||
def _document() -> dict[str, object]:
|
||||
return json.loads(REGISTRY_PATH.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _write(tmp_path: Path, document: object) -> Path:
|
||||
path = tmp_path / "portable-definitions.json"
|
||||
path.write_text(json.dumps(document), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def _first(document: dict[str, object]) -> dict[str, object]:
|
||||
definitions = document["definitions"]
|
||||
assert isinstance(definitions, list)
|
||||
first = definitions[0]
|
||||
assert isinstance(first, dict)
|
||||
return first
|
||||
|
||||
|
||||
def test_production_definition_is_source_independent_and_requirements_are_immutable() -> None:
|
||||
definition = _registry().definitions[0]
|
||||
requirements = definition.source_requirements
|
||||
|
||||
assert definition.setup_id == "lab-v1-eomt-ddrnet-portable-v1"
|
||||
assert requirements.plugin_id == "nodedc.device.xgrids-lixelkity-k1"
|
||||
assert requirements.archive_id == "xgrids-k1.viewer-live.evidence"
|
||||
assert requirements.required_modalities == ("point-cloud", "trajectory", "video")
|
||||
assert requirements.camera_source_id == "sensor.camera.right"
|
||||
assert requirements.camera_semantic_channel_id == "camera.video.recorded"
|
||||
assert requirements.recorded_media_type == 'video/mp4; codecs="avc1.641028"'
|
||||
assert requirements.recorded_media_init_sha256 == (
|
||||
"e2279963e16d84c91d68e7dbb1f7efed840533387dfeb844b7398bff45fbde38"
|
||||
)
|
||||
assert requirements.camera_width == 800
|
||||
assert requirements.camera_height == 600
|
||||
assert requirements.calibration_slot == "camera_1"
|
||||
assert requirements.calibration_identity_sha256 == (
|
||||
"05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
)
|
||||
assert requirements.exactly_one_media_epoch is True
|
||||
assert requirements.seekable is True
|
||||
assert not hasattr(requirements, "session_id")
|
||||
assert not hasattr(requirements, "label")
|
||||
|
||||
identity = json.dumps(definition.identity_document(), sort_keys=True)
|
||||
assert "RAVNOVES" not in identity
|
||||
assert "20260828T130511Z_viewer_live" not in identity
|
||||
assert "historical_results" not in identity
|
||||
assert not hasattr(definition, "historical_results")
|
||||
assert "session_id" not in identity
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
requirements.plugin_id = "changed" # type: ignore[misc]
|
||||
|
||||
|
||||
def test_source_requirements_map_exactly_to_admission_contract() -> None:
|
||||
definition = _registry().definitions[0]
|
||||
|
||||
admission = definition.to_source_admission_requirements()
|
||||
|
||||
assert admission.plugin_id == definition.source_requirements.plugin_id
|
||||
assert admission.archive_id == definition.source_requirements.archive_id
|
||||
assert admission.expected_width == definition.source_requirements.camera_width
|
||||
assert admission.expected_height == definition.source_requirements.camera_height
|
||||
assert admission.calibration_slot == definition.source_requirements.calibration_slot
|
||||
assert admission.require_single_camera_epoch is True
|
||||
assert admission.adapter_document() == definition.source_adapter.identity_document(
|
||||
definition.source_requirements
|
||||
)
|
||||
assert admission.adapter_sha256 == definition.source_adapter.contract_sha256
|
||||
|
||||
|
||||
def test_all_canonical_identities_are_recomputed_from_typed_content() -> None:
|
||||
definition = _registry().definitions[0]
|
||||
|
||||
assert definition.definition_sha256 == DEFINITION_SHA256
|
||||
assert canonical_sha256(definition.identity_document()) == DEFINITION_SHA256
|
||||
assert definition.source_adapter.contract_sha256 == canonical_sha256(
|
||||
definition.source_adapter.identity_document(definition.source_requirements)
|
||||
)
|
||||
assert definition.resource_profile.profile_sha256 == canonical_sha256(
|
||||
definition.resource_profile.identity_document()
|
||||
)
|
||||
assert definition.result_contract.contract_sha256 == canonical_sha256(
|
||||
definition.result_contract.identity_document()
|
||||
)
|
||||
assert definition.model_manifest_sha256 == MODEL_MANIFEST_SHA256
|
||||
assert definition.model_manifest_sha256 == canonical_sha256(
|
||||
{
|
||||
"schema_version": PORTABLE_MODEL_MANIFEST_SCHEMA,
|
||||
"models": [model.as_dict() for model in definition.models],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_registry_rejects_legacy_result_metadata(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
document = _document()
|
||||
first = _first(document)
|
||||
first["historical_results"] = []
|
||||
|
||||
with pytest.raises(PortableRunDefinitionRegistryError, match="fields are invalid"):
|
||||
PortableRunDefinitionRegistry.from_file(_write(tmp_path, document))
|
||||
|
||||
|
||||
def test_component_or_model_mutation_without_a_new_digest_is_rejected(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
document = _document()
|
||||
first = _first(document)
|
||||
models = first["models"]
|
||||
assert isinstance(models, list)
|
||||
model = models[0]
|
||||
assert isinstance(model, dict)
|
||||
artifacts = model["artifacts"]
|
||||
assert isinstance(artifacts, list)
|
||||
artifact = artifacts[1]
|
||||
assert isinstance(artifact, dict)
|
||||
artifact["sha256"] = "0" * 64
|
||||
|
||||
with pytest.raises(
|
||||
PortableRunDefinitionRegistryError,
|
||||
match="portable definition digest changed",
|
||||
):
|
||||
PortableRunDefinitionRegistry.from_file(_write(tmp_path, document))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"forbidden_key",
|
||||
[
|
||||
"source_session_id",
|
||||
"source_label",
|
||||
"runner_path",
|
||||
"command",
|
||||
"environment",
|
||||
"user_priority",
|
||||
],
|
||||
)
|
||||
def test_registry_rejects_source_binding_and_executable_or_priority_fields(
|
||||
tmp_path: Path,
|
||||
forbidden_key: str,
|
||||
) -> None:
|
||||
document = _document()
|
||||
first = _first(document)
|
||||
requirements = first["source_requirements"]
|
||||
assert isinstance(requirements, dict)
|
||||
requirements[forbidden_key] = "caller-owned"
|
||||
|
||||
with pytest.raises(PortableRunDefinitionRegistryError, match="forbids"):
|
||||
PortableRunDefinitionRegistry.from_file(_write(tmp_path, document))
|
||||
|
||||
|
||||
def test_registry_rejects_authority_escalation_and_calibration_disagreement(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
authority_escalation = _document()
|
||||
first = _first(authority_escalation)
|
||||
authority = first["authority"]
|
||||
assert isinstance(authority, dict)
|
||||
authority["commands_enabled"] = True
|
||||
with pytest.raises(PortableRunDefinitionRegistryError, match="observation-only"):
|
||||
PortableRunDefinitionRegistry.from_file(_write(tmp_path, authority_escalation))
|
||||
|
||||
calibration_drift = _document()
|
||||
first = _first(calibration_drift)
|
||||
requirements = first["source_requirements"]
|
||||
assert isinstance(requirements, dict)
|
||||
requirements["calibration_identity_sha256"] = "a" * 64
|
||||
adapter = first["source_adapter"]
|
||||
assert isinstance(adapter, dict)
|
||||
adapter_document = (
|
||||
_registry()
|
||||
.definitions[0]
|
||||
.source_adapter.identity_document(_registry().definitions[0].source_requirements)
|
||||
)
|
||||
calibration = adapter_document["calibration"]
|
||||
assert isinstance(calibration, dict)
|
||||
calibration["sha256"] = "a" * 64
|
||||
adapter["contract_sha256"] = canonical_sha256(adapter_document)
|
||||
with pytest.raises(PortableRunDefinitionRegistryError, match="calibration"):
|
||||
PortableRunDefinitionRegistry.from_file(_write(tmp_path, calibration_drift))
|
||||
|
||||
|
||||
def test_duplicate_definition_and_incomplete_ready_executor_are_rejected(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
duplicate = _document()
|
||||
definitions = duplicate["definitions"]
|
||||
assert isinstance(definitions, list)
|
||||
definitions.append(copy.deepcopy(definitions[0]))
|
||||
with pytest.raises(PortableRunDefinitionRegistryError, match="setup IDs"):
|
||||
PortableRunDefinitionRegistry.from_file(_write(tmp_path, duplicate))
|
||||
|
||||
incomplete = _document()
|
||||
first = _first(incomplete)
|
||||
executor = first["executor"]
|
||||
assert isinstance(executor, dict)
|
||||
executor["state"] = "ready"
|
||||
executor["reason_code"] = None
|
||||
executor["reason"] = None
|
||||
with pytest.raises(PortableRunDefinitionRegistryError, match="release identity"):
|
||||
PortableRunDefinitionRegistry.from_file(_write(tmp_path, incomplete))
|
||||
|
||||
|
||||
def test_production_definition_is_blocked_until_release_and_image_are_sealed() -> None:
|
||||
registry = _registry()
|
||||
definition = registry.definitions[0]
|
||||
|
||||
assert definition.executor.state == "not-installed"
|
||||
assert definition.executor.release_id is None
|
||||
assert definition.executor.release_sha256 is None
|
||||
assert definition.executor.image_sha256 is None
|
||||
with pytest.raises(
|
||||
PortableRunDefinitionUnavailableError,
|
||||
match="not sealed or installed",
|
||||
):
|
||||
definition.to_recorded_run_definition()
|
||||
with pytest.raises(PortableRunDefinitionUnavailableError):
|
||||
registry.to_recorded_registry()
|
||||
|
||||
|
||||
def test_conversion_to_recorded_definition_requires_and_preserves_sealed_identities() -> None:
|
||||
blocked = _registry().definitions[0]
|
||||
ready_executor = PortableExecutorAvailability(
|
||||
contour_id="worker-006",
|
||||
state="ready",
|
||||
release_id="lab-v1-eomt-ddrnet-executor-v1",
|
||||
release_sha256="1" * 64,
|
||||
image_sha256="2" * 64,
|
||||
reason_code=None,
|
||||
reason=None,
|
||||
)
|
||||
identity = blocked.identity_document()
|
||||
identity["executor"] = ready_executor.identity_document()
|
||||
ready = replace(
|
||||
blocked,
|
||||
executor=ready_executor,
|
||||
definition_sha256=canonical_sha256(identity),
|
||||
)
|
||||
|
||||
recorded = ready.to_recorded_run_definition()
|
||||
|
||||
assert recorded.setup_id == ready.setup_id
|
||||
assert recorded.definition_sha256 == ready.definition_sha256
|
||||
assert recorded.source_adapter_sha256 == ready.source_adapter.contract_sha256
|
||||
assert recorded.executor_release_sha256 == "1" * 64
|
||||
assert recorded.executor_image_sha256 == "2" * 64
|
||||
assert recorded.model_release_ids == ready.learned_models
|
||||
assert recorded.model_manifest_sha256 == MODEL_MANIFEST_SHA256
|
||||
assert recorded.resource_profile_sha256 == ready.resource_profile.profile_sha256
|
||||
assert recorded.checkpoint_policy == "non-checkpointable"
|
||||
|
||||
|
||||
def test_production_lab_v1_model_component_and_result_identities_are_exact() -> None:
|
||||
definition = _registry().definitions[0]
|
||||
models = {model.release_id: model for model in definition.models}
|
||||
eomt = models["eomt-cityscapes-large-1024-v1"]
|
||||
ddrnet = models["lab-v1-ddrnet-39-goose-fine-64-v1"]
|
||||
|
||||
assert eomt.model_id == "tue-mps/cityscapes_semantic_eomt_large_1024"
|
||||
assert eomt.revision == "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f"
|
||||
assert {artifact.role: artifact.sha256 for artifact in eomt.artifacts} == {
|
||||
"config-json": ("7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650"),
|
||||
"model-weights": "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782",
|
||||
"preprocessor-config": ("97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7"),
|
||||
}
|
||||
assert ddrnet.model_id == "goose-ddrnet-class-512"
|
||||
assert ddrnet.revision is None
|
||||
assert ddrnet.architecture == "ddrnet_39"
|
||||
assert ddrnet.artifacts[0].sha256 == (
|
||||
"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"
|
||||
)
|
||||
assert ddrnet.artifacts[0].byte_length == 259_419_077
|
||||
|
||||
components = {component.component_id: component for component in definition.components}
|
||||
assert components["eomt-recorded-profile-v1"].sha256 == (
|
||||
"ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875"
|
||||
)
|
||||
assert components["eomt-recorded-runner-v1"].sha256 == (
|
||||
"651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4"
|
||||
)
|
||||
assert components["k1-camera-1-calibration-v1"].sha256 == (
|
||||
"05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
)
|
||||
assert components["k1-valid-fov-identity-v1"].sha256 == (
|
||||
"b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2"
|
||||
)
|
||||
assert components["k1-valid-fov-mask-v1"].sha256 == (
|
||||
"a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63"
|
||||
)
|
||||
|
||||
assert definition.result_contract.result_schema == (
|
||||
"missioncore.recorded-eomt-ddrnet-review/v2"
|
||||
)
|
||||
assert definition.result_contract.result_kind == "recorded-perception-qualification"
|
||||
@@ -0,0 +1,114 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry
|
||||
from k1link.observatory.portable_setup_projection import PortableLabV1SetupProjector
|
||||
from k1link.observatory.source_admission import PortableRecordedSourceCapability
|
||||
from k1link.sessions import SessionNotFoundError
|
||||
from k1link.sessions.models import SessionSummary
|
||||
from k1link.web.observatory_api import build_observatory_router
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
SOURCE_SESSION_ID = "20260831T080000Z_viewer_live"
|
||||
|
||||
|
||||
def _summary() -> SessionSummary:
|
||||
return SessionSummary(
|
||||
session_id=SOURCE_SESSION_ID,
|
||||
display_name="RAVNOVES005",
|
||||
status="ready",
|
||||
started_at_utc="2026-08-31T08:00:00Z",
|
||||
completed_at_utc="2026-08-31T08:10:00Z",
|
||||
duration_seconds=600.0,
|
||||
modalities=("point-cloud", "trajectory", "video"),
|
||||
source_count=3,
|
||||
total_bytes=1,
|
||||
replayable=True,
|
||||
origin="recorded",
|
||||
)
|
||||
|
||||
|
||||
class _Store:
|
||||
def get_session(self, session_id: str) -> SimpleNamespace:
|
||||
if session_id != SOURCE_SESSION_ID:
|
||||
raise SessionNotFoundError(session_id)
|
||||
return SimpleNamespace(summary=_summary())
|
||||
|
||||
|
||||
def _projector() -> PortableLabV1SetupProjector:
|
||||
registry = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||
definition = registry.definitions[0]
|
||||
|
||||
def probe(session_id: str) -> PortableRecordedSourceCapability:
|
||||
return PortableRecordedSourceCapability(
|
||||
source_session_id=session_id,
|
||||
source_catalog_sha256="a" * 64,
|
||||
source_adapter_sha256=definition.source_adapter.contract_sha256,
|
||||
camera_segment_count=600,
|
||||
)
|
||||
|
||||
return PortableLabV1SetupProjector(
|
||||
registry=registry,
|
||||
capability_probe=probe,
|
||||
)
|
||||
|
||||
|
||||
def test_portable_setup_catalog_exposes_capability_and_executor_separately() -> None:
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_observatory_router(
|
||||
_Store(), # type: ignore[arg-type]
|
||||
portable_setup_projector=_projector(),
|
||||
)
|
||||
)
|
||||
|
||||
response = TestClient(app).get(
|
||||
"/api/v1/observatory/portable-laboratory-setups",
|
||||
params={"source_session_id": SOURCE_SESSION_ID},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
document = response.json()
|
||||
assert document["schema_version"] == ("missioncore.observatory-portable-setup-catalog/v2")
|
||||
setup = document["setups"][0]
|
||||
assert setup["origin"] == "portable-definition"
|
||||
assert setup["source_compatibility"]["outcome"] == "pass"
|
||||
assert setup["executor"]["state"] == "not-installed"
|
||||
assert setup["existing_results"] == []
|
||||
assert setup["preflight"]["outcome"] == "blocked"
|
||||
assert setup["preflight"]["submission_allowed"] is False
|
||||
|
||||
|
||||
def test_portable_setup_catalog_preserves_source_and_optional_slice_failures() -> None:
|
||||
ready_app = FastAPI()
|
||||
ready_app.include_router(
|
||||
build_observatory_router(
|
||||
_Store(), # type: ignore[arg-type]
|
||||
portable_setup_projector=_projector(),
|
||||
)
|
||||
)
|
||||
missing = TestClient(ready_app).get(
|
||||
"/api/v1/observatory/portable-laboratory-setups",
|
||||
params={"source_session_id": "missing-session"},
|
||||
)
|
||||
assert missing.status_code == 404
|
||||
|
||||
unavailable_app = FastAPI()
|
||||
unavailable_app.include_router(
|
||||
build_observatory_router(
|
||||
_Store(), # type: ignore[arg-type]
|
||||
portable_setup_projector_error="drifted portable registry",
|
||||
)
|
||||
)
|
||||
unavailable = TestClient(unavailable_app).get(
|
||||
"/api/v1/observatory/portable-laboratory-setups",
|
||||
params={"source_session_id": SOURCE_SESSION_ID},
|
||||
)
|
||||
assert unavailable.status_code == 503
|
||||
assert unavailable.json()["detail"] == "Portable-каталог LAB V1 недоступен."
|
||||
@@ -0,0 +1,244 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableExecutorAvailability,
|
||||
PortableRunDefinitionRegistry,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
PORTABLE_LAB_V1_DISPLAY_NAME,
|
||||
PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA,
|
||||
PortableLabV1SetupProjector,
|
||||
PortableSetupProjectionError,
|
||||
PortableSourceCapabilityProbe,
|
||||
)
|
||||
from k1link.observatory.source_admission import (
|
||||
PortableRecordedSourceCapability,
|
||||
PortableSourceAdmissionIntegrityError,
|
||||
)
|
||||
from k1link.sessions.models import SessionSummary
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
RAV004_SESSION_ID = "20260828T130511Z_viewer_live"
|
||||
NEW_SESSION_ID = "20260831T080000Z_viewer_live"
|
||||
|
||||
|
||||
def _registry() -> PortableRunDefinitionRegistry:
|
||||
return PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||
|
||||
|
||||
def _source(
|
||||
session_id: str,
|
||||
*,
|
||||
display_name: str = "arbitrary operator label",
|
||||
) -> SessionSummary:
|
||||
return SessionSummary(
|
||||
session_id=session_id,
|
||||
display_name=display_name,
|
||||
status="ready",
|
||||
started_at_utc="2026-08-31T08:00:00Z",
|
||||
completed_at_utc="2026-08-31T08:10:00Z",
|
||||
duration_seconds=600.0,
|
||||
modalities=("point-cloud", "trajectory", "video"),
|
||||
source_count=3,
|
||||
total_bytes=1,
|
||||
replayable=True,
|
||||
origin="recorded",
|
||||
)
|
||||
|
||||
|
||||
def _capability(
|
||||
source_session_id: str,
|
||||
*,
|
||||
adapter_sha256: str | None = None,
|
||||
) -> PortableRecordedSourceCapability:
|
||||
definition = _registry().definitions[0]
|
||||
return PortableRecordedSourceCapability(
|
||||
source_session_id=source_session_id,
|
||||
source_catalog_sha256="a" * 64,
|
||||
source_adapter_sha256=(adapter_sha256 or definition.source_adapter.contract_sha256),
|
||||
camera_segment_count=60,
|
||||
)
|
||||
|
||||
|
||||
def _projector(
|
||||
capability_probe: PortableSourceCapabilityProbe,
|
||||
*,
|
||||
registry: PortableRunDefinitionRegistry | None = None,
|
||||
) -> PortableLabV1SetupProjector:
|
||||
return PortableLabV1SetupProjector(
|
||||
registry=registry or _registry(),
|
||||
capability_probe=capability_probe,
|
||||
)
|
||||
|
||||
|
||||
def test_projection_uses_portable_identity_and_exact_model_presentation() -> None:
|
||||
source = _source(NEW_SESSION_ID, display_name="RAVNOVES005")
|
||||
projection = _projector(lambda session_id: _capability(session_id)).catalog(source)
|
||||
setup = projection["setups"][0]
|
||||
|
||||
assert projection["schema_version"] == PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA
|
||||
assert setup["origin"] == "portable-definition"
|
||||
assert setup["display_name"] == PORTABLE_LAB_V1_DISPLAY_NAME
|
||||
assert setup["run_definition"]["models"] == [
|
||||
{
|
||||
"name": "EoMT Cityscapes Large 1024",
|
||||
"release_id": "eomt-cityscapes-large-1024-v1",
|
||||
"model_id": "tue-mps/cityscapes_semantic_eomt_large_1024",
|
||||
"architecture": "EomtForUniversalSegmentation",
|
||||
},
|
||||
{
|
||||
"name": "DDRNet-39",
|
||||
"release_id": "lab-v1-ddrnet-39-goose-fine-64-v1",
|
||||
"model_id": "goose-ddrnet-class-512",
|
||||
"architecture": "ddrnet_39",
|
||||
},
|
||||
]
|
||||
requirements = setup["source_requirements"]
|
||||
assert "source_session_id" not in requirements
|
||||
assert "source_label" not in requirements
|
||||
assert "label" not in requirements
|
||||
assert "RAVNOVES" not in str(requirements)
|
||||
assert setup["run_definition"]["definition_sha256"] == (
|
||||
_registry().definitions[0].definition_sha256
|
||||
)
|
||||
assert setup["run_definition"]["result_schema"] == (
|
||||
_registry().definitions[0].result_contract.result_schema
|
||||
)
|
||||
assert setup["authority"] == {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
|
||||
|
||||
def test_new_compatible_source_passes_capability_but_uninstalled_executor_blocks() -> None:
|
||||
source = _source(NEW_SESSION_ID)
|
||||
setup = _projector(lambda session_id: _capability(session_id)).project(source)
|
||||
|
||||
assert setup["source_compatibility"] == {
|
||||
"outcome": "pass",
|
||||
"compatible": True,
|
||||
"reason": "Запись соответствует требованиям EoMT + DDRNet.",
|
||||
}
|
||||
assert setup["executor"]["state"] == "not-installed"
|
||||
assert setup["executor"]["ready"] is False
|
||||
assert setup["existing_results"] == []
|
||||
assert setup["preflight"] == {
|
||||
"outcome": "blocked",
|
||||
"action": "blocked",
|
||||
"reason": "Вычислительный контур LAB V1 пока недоступен.",
|
||||
"submission_allowed": False,
|
||||
"existing_result_ids": [],
|
||||
}
|
||||
|
||||
|
||||
def test_legacy_vegetation_result_is_never_existing_for_portable_v2() -> None:
|
||||
setup = _projector(lambda session_id: _capability(session_id)).project(
|
||||
_source(RAV004_SESSION_ID)
|
||||
)
|
||||
|
||||
assert setup["run_definition"]["result_schema"] == (
|
||||
"missioncore.recorded-eomt-ddrnet-review/v2"
|
||||
)
|
||||
assert setup["existing_results"] == []
|
||||
assert setup["preflight"] == {
|
||||
"outcome": "blocked",
|
||||
"action": "blocked",
|
||||
"reason": "Вычислительный контур LAB V1 пока недоступен.",
|
||||
"submission_allowed": False,
|
||||
"existing_result_ids": [],
|
||||
}
|
||||
|
||||
|
||||
class _RejectingCapabilityService:
|
||||
def probe(self, source_session_id: str) -> PortableRecordedSourceCapability:
|
||||
del source_session_id
|
||||
raise PortableSourceAdmissionIntegrityError("missing sealed video")
|
||||
|
||||
|
||||
def test_real_capability_probe_rejection_wins_over_summary() -> None:
|
||||
setup = _projector(_RejectingCapabilityService()).project(_source(RAV004_SESSION_ID))
|
||||
|
||||
assert setup["source_compatibility"] == {
|
||||
"outcome": "blocked",
|
||||
"compatible": False,
|
||||
"reason": "Запись не соответствует требованиям EoMT + DDRNet.",
|
||||
}
|
||||
assert setup["existing_results"] == []
|
||||
assert setup["preflight"]["outcome"] == "blocked"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mismatch", ["source", "adapter"])
|
||||
def test_capability_probe_must_match_source_and_definition_adapter(
|
||||
mismatch: str,
|
||||
) -> None:
|
||||
source = _source(NEW_SESSION_ID)
|
||||
|
||||
def probe(source_session_id: str) -> PortableRecordedSourceCapability:
|
||||
if mismatch == "source":
|
||||
return _capability(RAV004_SESSION_ID)
|
||||
return _capability(source_session_id, adapter_sha256="f" * 64)
|
||||
|
||||
with pytest.raises(PortableSetupProjectionError, match=mismatch):
|
||||
_projector(probe).project(source)
|
||||
|
||||
|
||||
class _ProbeOnlyCapabilityService:
|
||||
def probe(self, source_session_id: str) -> PortableRecordedSourceCapability:
|
||||
return _capability(source_session_id)
|
||||
|
||||
def check(self, source_session_id: str) -> None:
|
||||
raise AssertionError(f"full admission check called for {source_session_id}")
|
||||
|
||||
|
||||
def test_projector_uses_lightweight_probe_and_never_full_admission_check() -> None:
|
||||
setup = _projector(_ProbeOnlyCapabilityService()).project(_source(NEW_SESSION_ID))
|
||||
|
||||
assert setup["source_compatibility"]["outcome"] == "pass"
|
||||
|
||||
|
||||
def _ready_registry() -> PortableRunDefinitionRegistry:
|
||||
blocked = _registry().definitions[0]
|
||||
ready_executor = PortableExecutorAvailability(
|
||||
contour_id="worker-006",
|
||||
state="ready",
|
||||
release_id="lab-v1-eomt-ddrnet-executor-v1",
|
||||
release_sha256="1" * 64,
|
||||
image_sha256="2" * 64,
|
||||
reason_code=None,
|
||||
reason=None,
|
||||
)
|
||||
identity = blocked.identity_document()
|
||||
identity["executor"] = ready_executor.identity_document()
|
||||
ready = replace(
|
||||
blocked,
|
||||
executor=ready_executor,
|
||||
definition_sha256=canonical_sha256(identity),
|
||||
)
|
||||
return PortableRunDefinitionRegistry((ready,))
|
||||
|
||||
|
||||
def test_ready_executor_still_blocks_without_a_dispatch_boundary() -> None:
|
||||
setup = _projector(
|
||||
lambda session_id: _capability(session_id),
|
||||
registry=_ready_registry(),
|
||||
).project(_source(NEW_SESSION_ID))
|
||||
|
||||
assert setup["preflight"] == {
|
||||
"outcome": "blocked",
|
||||
"action": "blocked",
|
||||
"reason": (
|
||||
"Server-side проверка definition/check SHA и постановка portable "
|
||||
"LAB V1 в очередь пока недоступны."
|
||||
),
|
||||
"submission_allowed": False,
|
||||
"existing_result_ids": [],
|
||||
}
|
||||
@@ -0,0 +1,558 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.source_admission import (
|
||||
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||
PortableSourceAdmissionIntegrityError,
|
||||
PortableSourceAdmissionStaleError,
|
||||
PortableSourceNotPreparedError,
|
||||
RecordedK1SourceAdmissionService,
|
||||
RecordedK1SourceRequirements,
|
||||
)
|
||||
from k1link.sessions.media import (
|
||||
CAMERA_ARCHIVE_SCHEMA,
|
||||
RecordedMediaEpoch,
|
||||
RecordedMediaManifest,
|
||||
RecordedMediaSegment,
|
||||
)
|
||||
from k1link.sessions.models import (
|
||||
LabReplayCapability,
|
||||
LabSessionBinding,
|
||||
RecordedMediaArtifact,
|
||||
ReplayArtifact,
|
||||
ReplayCommand,
|
||||
SessionArtifact,
|
||||
SessionDetail,
|
||||
SessionSource,
|
||||
SessionSummary,
|
||||
)
|
||||
|
||||
INIT_SHA256 = "e2279963e16d84c91d68e7dbb1f7efed840533387dfeb844b7398bff45fbde38"
|
||||
CALIBRATION_SHA256 = "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
CATALOG_SHA256 = "1" * 64
|
||||
RAW_SHA256 = "2" * 64
|
||||
GENERATION_SHA256 = "3" * 64
|
||||
SEGMENT_SHA256 = "4" * 64
|
||||
|
||||
|
||||
def _requirements() -> RecordedK1SourceRequirements:
|
||||
return RecordedK1SourceRequirements(
|
||||
adapter_id="recorded-k1-right-camera",
|
||||
adapter_version=1,
|
||||
plugin_id="nodedc.device.xgrids-lixelkity-k1",
|
||||
archive_id="xgrids-k1.viewer-live.evidence",
|
||||
required_modalities=("point-cloud", "trajectory", "video"),
|
||||
camera_source_id="sensor.camera.right",
|
||||
camera_semantic_channel_id="camera.video.recorded",
|
||||
camera_media_type='video/mp4; codecs="avc1.641028"',
|
||||
camera_init_sha256=INIT_SHA256,
|
||||
expected_width=800,
|
||||
expected_height=600,
|
||||
calibration_slot="camera_1",
|
||||
calibration_sha256=CALIBRATION_SHA256,
|
||||
)
|
||||
|
||||
|
||||
def _detail(session_id: str, display_name: str) -> SessionDetail:
|
||||
return SessionDetail(
|
||||
summary=SessionSummary(
|
||||
session_id=session_id,
|
||||
display_name=display_name,
|
||||
status="ready",
|
||||
started_at_utc="2026-08-31T00:00:00Z",
|
||||
completed_at_utc="2026-08-31T00:10:00Z",
|
||||
duration_seconds=600.0,
|
||||
modalities=("point-cloud", "trajectory", "video"),
|
||||
source_count=3,
|
||||
total_bytes=123,
|
||||
replayable=True,
|
||||
origin="xgrids-k1.viewer-live.evidence",
|
||||
),
|
||||
sources=(
|
||||
SessionSource(
|
||||
source_id="sensor.camera.right",
|
||||
semantic_channel_id="camera.video.recorded",
|
||||
modality="video",
|
||||
status="recorded",
|
||||
seekable=True,
|
||||
artifact_id="recorded-video-right",
|
||||
),
|
||||
SessionSource(
|
||||
source_id="sensor.lidar.primary",
|
||||
semantic_channel_id="spatial.point-cloud.recorded",
|
||||
modality="point-cloud",
|
||||
status="recorded",
|
||||
seekable=True,
|
||||
artifact_id="raw-transport-primary",
|
||||
),
|
||||
SessionSource(
|
||||
source_id="spatial.trajectory",
|
||||
semantic_channel_id="spatial.pose.recorded",
|
||||
modality="trajectory",
|
||||
status="recorded",
|
||||
seekable=True,
|
||||
artifact_id="raw-transport-primary",
|
||||
),
|
||||
),
|
||||
artifacts=(
|
||||
SessionArtifact(
|
||||
artifact_id="raw-transport-primary",
|
||||
kind="raw-transport",
|
||||
media_type="application/x-nodedc-k1mqtt",
|
||||
byte_length=100,
|
||||
sha256=RAW_SHA256,
|
||||
integrity_status="verified",
|
||||
),
|
||||
SessionArtifact(
|
||||
artifact_id="recorded-video-right",
|
||||
kind="recorded-video",
|
||||
media_type="video/mp4",
|
||||
byte_length=20,
|
||||
sha256=None,
|
||||
integrity_status="validated-structure",
|
||||
),
|
||||
),
|
||||
plugin_id="nodedc.device.xgrids-lixelkity-k1",
|
||||
archive_id="xgrids-k1.viewer-live.evidence",
|
||||
)
|
||||
|
||||
|
||||
def _replay(session_id: str, root: Path) -> ReplayCommand:
|
||||
return ReplayCommand(
|
||||
session_id=session_id,
|
||||
plugin_id="nodedc.device.xgrids-lixelkity-k1",
|
||||
allowed_root=root,
|
||||
session_root=root / session_id,
|
||||
primary_artifact_id="raw-transport-primary",
|
||||
artifacts=(
|
||||
ReplayArtifact(
|
||||
artifact_id="raw-transport-primary",
|
||||
path=root / "mqtt.raw.k1mqtt",
|
||||
media_type="application/x-nodedc-k1mqtt",
|
||||
file_byte_length=100,
|
||||
replay_byte_length=100,
|
||||
expected_sha256=RAW_SHA256,
|
||||
),
|
||||
),
|
||||
timeline_origin_epoch_ns=1,
|
||||
timeline_origin_monotonic_ns=2,
|
||||
speed=1.0,
|
||||
loop=False,
|
||||
)
|
||||
|
||||
|
||||
def _recorded_media(session_id: str, root: Path) -> RecordedMediaArtifact:
|
||||
return RecordedMediaArtifact(
|
||||
session_id=session_id,
|
||||
public_source_id="recorded.camera.right",
|
||||
artifact_id="recorded-video-right",
|
||||
source_path=root / "media" / "sensor.camera.right",
|
||||
byte_length=20,
|
||||
)
|
||||
|
||||
|
||||
def _manifest(session_id: str, root: Path) -> RecordedMediaManifest:
|
||||
segment = RecordedMediaSegment(
|
||||
sequence=1,
|
||||
path=root / "epoch-1" / "segments" / "1.m4s",
|
||||
byte_length=10,
|
||||
sha256=SEGMENT_SHA256,
|
||||
random_access=True,
|
||||
end_time_seconds=0.1,
|
||||
)
|
||||
epoch = RecordedMediaEpoch(
|
||||
ordinal=1,
|
||||
path=root / "epoch-1",
|
||||
init_path=root / "epoch-1" / "init.mp4",
|
||||
init_byte_length=10,
|
||||
init_sha256=INIT_SHA256,
|
||||
media_type='video/mp4; codecs="avc1.641028"',
|
||||
timeline_start_seconds=10.0,
|
||||
timeline_end_seconds=10.1,
|
||||
segments=(segment,),
|
||||
)
|
||||
return RecordedMediaManifest(
|
||||
session_id=session_id,
|
||||
public_source_id="recorded.camera.right",
|
||||
artifact_id="recorded-video-right",
|
||||
synchronization="host-arrival-best-effort",
|
||||
generation_sha256=GENERATION_SHA256,
|
||||
timeline_start_seconds=10.0,
|
||||
timeline_end_seconds=10.1,
|
||||
byte_length=20,
|
||||
epochs=(epoch,),
|
||||
)
|
||||
|
||||
|
||||
class _Store:
|
||||
def __init__(
|
||||
self,
|
||||
root: Path,
|
||||
detail: SessionDetail,
|
||||
*,
|
||||
catalogs: tuple[str, ...] = (CATALOG_SHA256,),
|
||||
) -> None:
|
||||
self.data_dir = root
|
||||
self.detail = detail
|
||||
self.replay = _replay(detail.summary.session_id, root)
|
||||
self.media = (_recorded_media(detail.summary.session_id, root),)
|
||||
self.catalogs = catalogs
|
||||
self.catalog_reads = 0
|
||||
self.prepare_replay_calls = 0
|
||||
self.recorded_media_reads = 0
|
||||
|
||||
def get_session_with_catalog_snapshot(
|
||||
self,
|
||||
session_id: str,
|
||||
) -> tuple[SessionDetail, str]:
|
||||
assert session_id == self.detail.summary.session_id
|
||||
index = min(self.catalog_reads, len(self.catalogs) - 1)
|
||||
self.catalog_reads += 1
|
||||
return self.detail, self.catalogs[index]
|
||||
|
||||
def prepare_replay(self, session_id: str) -> ReplayCommand:
|
||||
assert session_id == self.detail.summary.session_id
|
||||
self.prepare_replay_calls += 1
|
||||
return self.replay
|
||||
|
||||
def list_recorded_media(
|
||||
self,
|
||||
session_id: str,
|
||||
) -> tuple[RecordedMediaArtifact, ...]:
|
||||
assert session_id == self.detail.summary.session_id
|
||||
self.recorded_media_reads += 1
|
||||
return self.media
|
||||
|
||||
|
||||
class _Inspector:
|
||||
def __init__(self, manifest: RecordedMediaManifest | None) -> None:
|
||||
self.manifest = manifest
|
||||
self.inspect_calls = 0
|
||||
self.restore_calls = 0
|
||||
|
||||
def inspect(
|
||||
self,
|
||||
artifact: RecordedMediaArtifact,
|
||||
replay: ReplayCommand,
|
||||
) -> RecordedMediaManifest | None:
|
||||
assert artifact.session_id == replay.session_id
|
||||
self.inspect_calls += 1
|
||||
return self.manifest
|
||||
|
||||
def restore_prepared(
|
||||
self,
|
||||
artifact: RecordedMediaArtifact,
|
||||
replay: ReplayCommand,
|
||||
) -> RecordedMediaManifest | None:
|
||||
assert artifact.session_id == replay.session_id
|
||||
self.restore_calls += 1
|
||||
return self.manifest
|
||||
|
||||
|
||||
def _write_probe_summary(root: Path, *, segment_count: int = 7) -> None:
|
||||
epoch = root / "media" / "sensor.camera.right" / "epoch-1"
|
||||
epoch.mkdir(parents=True)
|
||||
(epoch / "summary.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": CAMERA_ARCHIVE_SCHEMA,
|
||||
"source_id": "sensor.camera.right",
|
||||
"codec_epoch": 1,
|
||||
"status": "complete",
|
||||
"segment_count": segment_count,
|
||||
"entry_count": segment_count,
|
||||
"media_segment_count": segment_count,
|
||||
"init_sha256": INIT_SHA256,
|
||||
"synchronization": "host-arrival-best-effort",
|
||||
"commit_policy": "per-segment-fsync",
|
||||
"failure_code": None,
|
||||
"artifacts": {
|
||||
"init": "init.mp4",
|
||||
"segments": "segments",
|
||||
"index": "index.jsonl",
|
||||
},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _service(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
session_id: str = "20260831T000000Z_viewer_live",
|
||||
display_name: str = "RAVNOVES005",
|
||||
detail: SessionDetail | None = None,
|
||||
manifest: RecordedMediaManifest | None = None,
|
||||
) -> RecordedK1SourceAdmissionService:
|
||||
active_detail = detail or _detail(session_id, display_name)
|
||||
store = _Store(tmp_path, active_detail)
|
||||
return RecordedK1SourceAdmissionService(
|
||||
data_dir=tmp_path,
|
||||
session_store=store, # type: ignore[arg-type]
|
||||
media_inspector=_Inspector(
|
||||
manifest or _manifest(active_detail.summary.session_id, tmp_path)
|
||||
), # type: ignore[arg-type]
|
||||
requirements=_requirements(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("session_id", "display_name"),
|
||||
(
|
||||
("20260720T065719Z_viewer_live", "RAVNOVES00"),
|
||||
("20260828T130511Z_viewer_live", "RAVNOVES004TREE"),
|
||||
("20260831T000000Z_viewer_live", "RAVNOVES005"),
|
||||
),
|
||||
)
|
||||
def test_portable_source_admission_is_independent_from_session_label(
|
||||
tmp_path: Path,
|
||||
session_id: str,
|
||||
display_name: str,
|
||||
) -> None:
|
||||
admission = _service(
|
||||
tmp_path,
|
||||
session_id=session_id,
|
||||
display_name=display_name,
|
||||
).check(session_id)
|
||||
|
||||
assert admission.source_session_id == session_id
|
||||
assert admission.frame_count == 1
|
||||
bundle = json.loads(admission.source_bundle)
|
||||
capability = json.loads(admission.capability_manifest)
|
||||
assert bundle["schema_version"] == PORTABLE_SOURCE_BUNDLE_SCHEMA
|
||||
assert capability["schema_version"] == PORTABLE_SOURCE_CAPABILITY_SCHEMA
|
||||
assert "RAVNOVES" not in admission.source_bundle.decode()
|
||||
assert "path" not in admission.source_bundle.decode()
|
||||
assert capability["camera_profile"]["width"] == 800
|
||||
assert capability["camera_profile"]["height"] == 600
|
||||
|
||||
|
||||
def test_catalog_capability_check_restores_media_without_preparing_it(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
detail = _detail("20260831T000000Z_viewer_live", "RAVNOVES005")
|
||||
store = _Store(tmp_path, detail)
|
||||
inspector = _Inspector(_manifest(detail.summary.session_id, tmp_path))
|
||||
service = RecordedK1SourceAdmissionService(
|
||||
data_dir=tmp_path,
|
||||
session_store=store, # type: ignore[arg-type]
|
||||
media_inspector=inspector, # type: ignore[arg-type]
|
||||
requirements=_requirements(),
|
||||
prepare_media=False,
|
||||
)
|
||||
|
||||
admission = service.check(detail.summary.session_id)
|
||||
|
||||
assert admission.frame_count == 1
|
||||
assert inspector.inspect_calls == 0
|
||||
assert inspector.restore_calls == 1
|
||||
|
||||
|
||||
def test_lightweight_probe_reads_only_catalog_media_handle_and_summary(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
detail = _detail("20260831T000000Z_viewer_live", "ignored-label")
|
||||
store = _Store(tmp_path, detail)
|
||||
inspector = _Inspector(_manifest(detail.summary.session_id, tmp_path))
|
||||
_write_probe_summary(tmp_path, segment_count=23)
|
||||
service = RecordedK1SourceAdmissionService(
|
||||
data_dir=tmp_path,
|
||||
session_store=store, # type: ignore[arg-type]
|
||||
media_inspector=inspector, # type: ignore[arg-type]
|
||||
requirements=_requirements(),
|
||||
)
|
||||
|
||||
capability = service.probe(detail.summary.session_id)
|
||||
|
||||
assert capability.source_session_id == detail.summary.session_id
|
||||
assert capability.source_catalog_sha256 == CATALOG_SHA256
|
||||
assert capability.source_adapter_sha256 == _requirements().adapter_sha256
|
||||
assert capability.camera_segment_count == 23
|
||||
assert store.catalog_reads == 1
|
||||
assert store.recorded_media_reads == 1
|
||||
assert store.prepare_replay_calls == 0
|
||||
assert inspector.inspect_calls == 0
|
||||
assert inspector.restore_calls == 0
|
||||
assert not (tmp_path / "media" / "sensor.camera.right" / "epoch-1" / "segments").exists()
|
||||
assert not (tmp_path / "observatory-portable-source-contracts").exists()
|
||||
|
||||
|
||||
def test_read_only_check_reports_missing_prepared_manifest_without_writes(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
detail = _detail("20260831T000000Z_viewer_live", "ignored-label")
|
||||
store = _Store(tmp_path, detail)
|
||||
inspector = _Inspector(None)
|
||||
service = RecordedK1SourceAdmissionService(
|
||||
data_dir=tmp_path,
|
||||
session_store=store, # type: ignore[arg-type]
|
||||
media_inspector=inspector, # type: ignore[arg-type]
|
||||
requirements=_requirements(),
|
||||
)
|
||||
|
||||
with pytest.raises(PortableSourceNotPreparedError, match="not been prepared"):
|
||||
service.check(detail.summary.session_id)
|
||||
|
||||
assert inspector.inspect_calls == 0
|
||||
assert inspector.restore_calls == 1
|
||||
assert not (tmp_path / "observatory-portable-source-contracts").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mutation", "message"),
|
||||
(
|
||||
("replay-session", "another source contract"),
|
||||
("replay-digest", "replay member disagrees"),
|
||||
("media-session", "artifact identity disagrees"),
|
||||
),
|
||||
)
|
||||
def test_full_check_rejects_cross_bound_replay_and_media(
|
||||
tmp_path: Path,
|
||||
mutation: str,
|
||||
message: str,
|
||||
) -> None:
|
||||
detail = _detail("20260831T000000Z_viewer_live", "ignored-label")
|
||||
store = _Store(tmp_path, detail)
|
||||
if mutation == "replay-session":
|
||||
store.replay = replace(store.replay, session_id="another-session")
|
||||
elif mutation == "replay-digest":
|
||||
store.replay = replace(
|
||||
store.replay,
|
||||
artifacts=(replace(store.replay.artifacts[0], expected_sha256="9" * 64),),
|
||||
)
|
||||
else:
|
||||
store.media = (replace(store.media[0], session_id="another-session"),)
|
||||
inspector = _Inspector(_manifest(detail.summary.session_id, tmp_path))
|
||||
service = RecordedK1SourceAdmissionService(
|
||||
data_dir=tmp_path,
|
||||
session_store=store, # type: ignore[arg-type]
|
||||
media_inspector=inspector, # type: ignore[arg-type]
|
||||
requirements=_requirements(),
|
||||
)
|
||||
|
||||
with pytest.raises(PortableSourceAdmissionIntegrityError, match=message):
|
||||
service.check(detail.summary.session_id)
|
||||
|
||||
assert inspector.inspect_calls == 0
|
||||
assert not (tmp_path / "observatory-portable-source-contracts").exists()
|
||||
|
||||
|
||||
def test_admit_rechecks_catalog_immediately_before_persistence(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
detail = _detail("20260831T000000Z_viewer_live", "ignored-label")
|
||||
store = _Store(tmp_path, detail, catalogs=(CATALOG_SHA256, "7" * 64))
|
||||
service = RecordedK1SourceAdmissionService(
|
||||
data_dir=tmp_path,
|
||||
session_store=store, # type: ignore[arg-type]
|
||||
media_inspector=_Inspector(_manifest(detail.summary.session_id, tmp_path)), # type: ignore[arg-type]
|
||||
requirements=_requirements(),
|
||||
)
|
||||
|
||||
with pytest.raises(PortableSourceAdmissionStaleError, match="changed"):
|
||||
service.admit(detail.summary.session_id)
|
||||
|
||||
assert store.catalog_reads == 2
|
||||
assert not (tmp_path / "observatory-portable-source-contracts").exists()
|
||||
|
||||
|
||||
def test_portable_source_admission_rejects_missing_video(tmp_path: Path) -> None:
|
||||
detail = _detail("20260728T163450Z_viewer_live", "RAVNOVES01")
|
||||
detail = replace(
|
||||
detail,
|
||||
summary=replace(
|
||||
detail.summary,
|
||||
modalities=("point-cloud", "trajectory"),
|
||||
source_count=2,
|
||||
),
|
||||
sources=tuple(source for source in detail.sources if source.modality != "video"),
|
||||
artifacts=tuple(
|
||||
artifact
|
||||
for artifact in detail.artifacts
|
||||
if artifact.artifact_id != "recorded-video-right"
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
PortableSourceAdmissionIntegrityError,
|
||||
match="modalities",
|
||||
):
|
||||
_service(tmp_path, detail=detail).check(detail.summary.session_id)
|
||||
|
||||
|
||||
def test_portable_source_admission_rejects_another_camera_profile(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
session_id = "20260831T000000Z_viewer_live"
|
||||
manifest = _manifest(session_id, tmp_path)
|
||||
manifest = replace(
|
||||
manifest,
|
||||
epochs=(replace(manifest.epochs[0], init_sha256="9" * 64),),
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
PortableSourceAdmissionIntegrityError,
|
||||
match="media profile",
|
||||
):
|
||||
_service(tmp_path, session_id=session_id, manifest=manifest).check(session_id)
|
||||
|
||||
|
||||
def test_portable_source_admission_rejects_lab_chaining(tmp_path: Path) -> None:
|
||||
detail = _detail("20260831T000000Z_viewer_live", "Derived")
|
||||
lab = LabSessionBinding(
|
||||
session_id=detail.summary.session_id,
|
||||
source_session_id="20260828T130511Z_viewer_live",
|
||||
lab_id="LAB V1",
|
||||
result_kind="recorded-perception-qualification",
|
||||
result_id="lab-v1-result",
|
||||
source_result_id=None,
|
||||
config_sha256=None,
|
||||
run_created_at_utc="2026-08-31T00:00:00Z",
|
||||
published_at_utc="2026-08-31T00:00:01Z",
|
||||
replay_capability=LabReplayCapability(
|
||||
schema_version="missioncore.observation-lab-replay-capability/v1",
|
||||
kind="canonical-recorded-rerun",
|
||||
viewer_profile="recorded-session",
|
||||
timeline="session_time",
|
||||
activation="explicit",
|
||||
commands_enabled=False,
|
||||
),
|
||||
provenance={},
|
||||
)
|
||||
detail = replace(detail, summary=replace(detail.summary, lab=lab))
|
||||
|
||||
with pytest.raises(
|
||||
PortableSourceAdmissionIntegrityError,
|
||||
match="not an admitted",
|
||||
):
|
||||
_service(tmp_path, detail=detail).check(detail.summary.session_id)
|
||||
|
||||
|
||||
def test_portable_source_admission_persists_only_content_addressed_documents(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
session_id = "20260831T000000Z_viewer_live"
|
||||
service = _service(tmp_path, session_id=session_id)
|
||||
|
||||
first = service.admit(session_id)
|
||||
repeated = service.admit(session_id)
|
||||
|
||||
assert repeated == first
|
||||
root = tmp_path / "observatory-portable-source-contracts"
|
||||
assert sorted(path.name for path in root.iterdir()) == sorted(
|
||||
(
|
||||
f"{first.source_bundle_sha256}.json",
|
||||
f"{first.source_capability_manifest_sha256}.json",
|
||||
)
|
||||
)
|
||||
assert (root / f"{first.source_bundle_sha256}.json").read_bytes() == first.source_bundle
|
||||
assert (
|
||||
root / f"{first.source_capability_manifest_sha256}.json"
|
||||
).read_bytes() == first.capability_manifest
|
||||
@@ -0,0 +1,416 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from threading import Event, Thread
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobIntent,
|
||||
ObservatoryRecordedJobQueue,
|
||||
RecordedRunDefinition,
|
||||
RecordedRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.worker_agent import (
|
||||
MAX_EXECUTOR_FAILURE_MESSAGE_LENGTH,
|
||||
WORKER_006_CONTOUR_ID,
|
||||
ObservatoryWorkerAgent,
|
||||
ObservatoryWorkerAgentBusyError,
|
||||
ObservatoryWorkerExecutionResult,
|
||||
ObservatoryWorkerExecutorIdentity,
|
||||
ObservatoryWorkerExecutorRegistration,
|
||||
ObservatoryWorkerExecutorRegistry,
|
||||
SealedObservatoryRecordedJob,
|
||||
)
|
||||
|
||||
DEFINITION_SHA = "1" * 64
|
||||
ADAPTER_SHA = "2" * 64
|
||||
EXECUTOR_RELEASE_SHA = "3" * 64
|
||||
EXECUTOR_IMAGE_SHA = "4" * 64
|
||||
MODEL_MANIFEST_SHA = "5" * 64
|
||||
RESOURCE_PROFILE_SHA = "6" * 64
|
||||
CATALOG_SHA = "7" * 64
|
||||
SOURCE_BUNDLE_SHA = "8" * 64
|
||||
SOURCE_CAPABILITY_SHA = "9" * 64
|
||||
RESULT_SHA = "a" * 64
|
||||
|
||||
|
||||
def _definition() -> RecordedRunDefinition:
|
||||
return RecordedRunDefinition(
|
||||
setup_id="portable-lab-v1",
|
||||
definition_id="portable-lab-v1-definition",
|
||||
definition_version=1,
|
||||
definition_sha256=DEFINITION_SHA,
|
||||
source_adapter_id="sealed-session-source",
|
||||
source_adapter_version=1,
|
||||
source_adapter_sha256=ADAPTER_SHA,
|
||||
executor_release_id="portable-lab-v1-worker",
|
||||
executor_release_sha256=EXECUTOR_RELEASE_SHA,
|
||||
executor_image_sha256=EXECUTOR_IMAGE_SHA,
|
||||
model_release_ids=("eomt-cityscapes-large", "ddrnet-39"),
|
||||
model_manifest_sha256=MODEL_MANIFEST_SHA,
|
||||
resource_profile_id="worker006-single-gpu",
|
||||
resource_profile_sha256=RESOURCE_PROFILE_SHA,
|
||||
checkpoint_policy="cooperative",
|
||||
allowed_checkpoints=("semantic-pass",),
|
||||
)
|
||||
|
||||
|
||||
def _identity(
|
||||
*,
|
||||
release_sha256: str = EXECUTOR_RELEASE_SHA,
|
||||
) -> ObservatoryWorkerExecutorIdentity:
|
||||
return ObservatoryWorkerExecutorIdentity(
|
||||
release_sha256=release_sha256,
|
||||
image_sha256=EXECUTOR_IMAGE_SHA,
|
||||
model_manifest_sha256=MODEL_MANIFEST_SHA,
|
||||
resource_profile_sha256=RESOURCE_PROFILE_SHA,
|
||||
)
|
||||
|
||||
|
||||
def _queue(tmp_path: Path) -> ObservatoryRecordedJobQueue:
|
||||
return ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=RecordedRunDefinitionRegistry((_definition(),)),
|
||||
)
|
||||
|
||||
|
||||
def _enqueue(queue: ObservatoryRecordedJobQueue) -> str:
|
||||
job, created = queue.submit(
|
||||
ObservatoryRecordedJobIntent(
|
||||
idempotency_key="worker-agent:test-job",
|
||||
source_session_id="20260831T120000Z_viewer_live",
|
||||
source_catalog_sha256=CATALOG_SHA,
|
||||
source_bundle_sha256=SOURCE_BUNDLE_SHA,
|
||||
source_capability_manifest_sha256=SOURCE_CAPABILITY_SHA,
|
||||
setup_id="portable-lab-v1",
|
||||
definition_sha256=DEFINITION_SHA,
|
||||
),
|
||||
enqueue=True,
|
||||
)
|
||||
assert created is True
|
||||
return job.job_id
|
||||
|
||||
|
||||
ClaimMutator = Callable[[dict[str, object]], dict[str, object]]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FakeTransport:
|
||||
queue: ObservatoryRecordedJobQueue
|
||||
claim_mutator: ClaimMutator | None = None
|
||||
starts: list[str] = field(default_factory=list)
|
||||
successes: list[tuple[str, str, str]] = field(default_factory=list)
|
||||
failures: list[tuple[str, str, str]] = field(default_factory=list)
|
||||
|
||||
def claim_next(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
claim_request_id: str,
|
||||
) -> Mapping[str, object] | None:
|
||||
claim = self.queue.claim_next(
|
||||
claimant_id=claimant_id,
|
||||
claim_request_id=claim_request_id,
|
||||
)
|
||||
if claim is None:
|
||||
return None
|
||||
payload = claim.as_dict()
|
||||
return self.claim_mutator(payload) if self.claim_mutator is not None else payload
|
||||
|
||||
def start(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
) -> Mapping[str, object]:
|
||||
assert claimant_id == WORKER_006_CONTOUR_ID
|
||||
self.starts.append(job_id)
|
||||
return self.queue.start(job_id, claim_token=claim_token).as_dict()
|
||||
|
||||
def succeed(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
result_id: str,
|
||||
result_sha256: str,
|
||||
) -> Mapping[str, object]:
|
||||
assert claimant_id == WORKER_006_CONTOUR_ID
|
||||
self.successes.append((job_id, result_id, result_sha256))
|
||||
return self.queue.succeed(
|
||||
job_id,
|
||||
claim_token=claim_token,
|
||||
result_id=result_id,
|
||||
result_sha256=result_sha256,
|
||||
).as_dict()
|
||||
|
||||
def fail(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
error_code: str,
|
||||
message: str,
|
||||
) -> Mapping[str, object]:
|
||||
assert claimant_id == WORKER_006_CONTOUR_ID
|
||||
self.failures.append((job_id, error_code, message))
|
||||
return self.queue.fail(
|
||||
job_id,
|
||||
claim_token=claim_token,
|
||||
error_code=error_code,
|
||||
message=message,
|
||||
).as_dict()
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FakeExecutor:
|
||||
result: ObservatoryWorkerExecutionResult = field(
|
||||
default_factory=lambda: ObservatoryWorkerExecutionResult(
|
||||
result_id="lab-v1-result",
|
||||
result_sha256=RESULT_SHA,
|
||||
)
|
||||
)
|
||||
error: Exception | None = None
|
||||
jobs: list[SealedObservatoryRecordedJob] = field(default_factory=list)
|
||||
|
||||
def execute(
|
||||
self,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
) -> ObservatoryWorkerExecutionResult:
|
||||
self.jobs.append(job)
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
return self.result
|
||||
|
||||
|
||||
def _agent(
|
||||
transport: FakeTransport,
|
||||
executor: FakeExecutor,
|
||||
*,
|
||||
identity: ObservatoryWorkerExecutorIdentity | None = None,
|
||||
) -> ObservatoryWorkerAgent:
|
||||
return ObservatoryWorkerAgent(
|
||||
transport=transport,
|
||||
executors=ObservatoryWorkerExecutorRegistry(
|
||||
(
|
||||
ObservatoryWorkerExecutorRegistration(
|
||||
identity=identity or _identity(),
|
||||
adapter=executor,
|
||||
),
|
||||
)
|
||||
),
|
||||
claim_request_id_factory=lambda: "worker-006:test-cycle",
|
||||
)
|
||||
|
||||
|
||||
def test_worker_agent_executes_one_sealed_allowlisted_job(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
transport = FakeTransport(queue)
|
||||
executor = FakeExecutor()
|
||||
|
||||
report = _agent(transport, executor).run_once()
|
||||
|
||||
assert report.state == "succeeded"
|
||||
assert report.job_id == job_id
|
||||
assert report.result_id == "lab-v1-result"
|
||||
assert transport.starts == [job_id]
|
||||
assert transport.successes == [(job_id, "lab-v1-result", RESULT_SHA)]
|
||||
assert transport.failures == []
|
||||
assert len(executor.jobs) == 1
|
||||
sealed_job = executor.jobs[0]
|
||||
assert sealed_job.executor_identity == _identity()
|
||||
assert sealed_job.source_bundle_sha256 == SOURCE_BUNDLE_SHA
|
||||
assert not hasattr(sealed_job, "command")
|
||||
assert not hasattr(sealed_job, "path")
|
||||
assert not hasattr(sealed_job, "environment")
|
||||
assert queue.get(job_id).state == "succeeded"
|
||||
|
||||
|
||||
def test_worker_agent_leaves_empty_queue_untouched(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
transport = FakeTransport(queue)
|
||||
executor = FakeExecutor()
|
||||
|
||||
report = _agent(transport, executor).run_once()
|
||||
|
||||
assert report.state == "empty"
|
||||
assert report.job_id is None
|
||||
assert transport.starts == []
|
||||
assert transport.failures == []
|
||||
assert executor.jobs == []
|
||||
|
||||
|
||||
def test_exact_release_mismatch_fails_closed_without_start(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
transport = FakeTransport(queue)
|
||||
executor = FakeExecutor()
|
||||
|
||||
report = _agent(
|
||||
transport,
|
||||
executor,
|
||||
identity=_identity(release_sha256="b" * 64),
|
||||
).run_once()
|
||||
|
||||
assert report.state == "failed"
|
||||
assert report.failure_code == "executor-not-allowlisted"
|
||||
assert transport.starts == []
|
||||
assert transport.successes == []
|
||||
assert transport.failures == [
|
||||
(
|
||||
job_id,
|
||||
"executor-not-allowlisted",
|
||||
"Exact executor identity is not installed on Worker 006.",
|
||||
)
|
||||
]
|
||||
assert executor.jobs == []
|
||||
assert queue.get(job_id).state == "failed"
|
||||
|
||||
|
||||
def _spoof_claimant(payload: dict[str, object]) -> dict[str, object]:
|
||||
changed = deepcopy(payload)
|
||||
changed["claimant_id"] = "worker-007"
|
||||
return changed
|
||||
|
||||
|
||||
def _inject_unknown_execution_payload(payload: dict[str, object]) -> dict[str, object]:
|
||||
changed = deepcopy(payload)
|
||||
job = changed["job"]
|
||||
assert isinstance(job, dict)
|
||||
job["command"] = ["python", "untrusted.py"]
|
||||
return changed
|
||||
|
||||
|
||||
def _corrupt_job_identity(payload: dict[str, object]) -> dict[str, object]:
|
||||
changed = deepcopy(payload)
|
||||
job = changed["job"]
|
||||
assert isinstance(job, dict)
|
||||
source = job["source"]
|
||||
assert isinstance(source, dict)
|
||||
source["bundle_sha256"] = "c" * 64
|
||||
return changed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutator",
|
||||
[_spoof_claimant, _inject_unknown_execution_payload, _corrupt_job_identity],
|
||||
)
|
||||
def test_spoofed_unknown_or_corrupted_claim_is_rejected_without_execution(
|
||||
tmp_path: Path,
|
||||
mutator: ClaimMutator,
|
||||
) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
transport = FakeTransport(queue, claim_mutator=mutator)
|
||||
executor = FakeExecutor()
|
||||
|
||||
report = _agent(transport, executor).run_once()
|
||||
|
||||
assert report.state == "rejected"
|
||||
assert report.failure_code == "claim-rejected"
|
||||
assert transport.starts == []
|
||||
assert transport.successes == []
|
||||
assert transport.failures == []
|
||||
assert executor.jobs == []
|
||||
assert queue.get(job_id).state == "claimed"
|
||||
|
||||
|
||||
def test_executor_error_is_reported_as_bounded_failure(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
transport = FakeTransport(queue)
|
||||
executor = FakeExecutor(error=RuntimeError("unsafe\n" + ("x" * 2_000)))
|
||||
|
||||
report = _agent(transport, executor).run_once()
|
||||
|
||||
assert report.state == "failed"
|
||||
assert report.failure_code == "executor-error"
|
||||
assert transport.starts == [job_id]
|
||||
assert transport.successes == []
|
||||
assert len(transport.failures) == 1
|
||||
failed_job_id, failure_code, message = transport.failures[0]
|
||||
assert failed_job_id == job_id
|
||||
assert failure_code == "executor-error"
|
||||
assert len(message) <= MAX_EXECUTOR_FAILURE_MESSAGE_LENGTH
|
||||
assert "\n" not in message
|
||||
assert queue.get(job_id).state == "failed"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class BlockingEmptyTransport:
|
||||
entered: Event
|
||||
release: Event
|
||||
|
||||
def claim_next(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
claim_request_id: str,
|
||||
) -> Mapping[str, object] | None:
|
||||
assert claimant_id == WORKER_006_CONTOUR_ID
|
||||
assert claim_request_id == "worker-006:overlap-test"
|
||||
self.entered.set()
|
||||
assert self.release.wait(timeout=2)
|
||||
return None
|
||||
|
||||
def start(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
) -> Mapping[str, object]:
|
||||
raise AssertionError("an empty transport cannot start a job")
|
||||
|
||||
def succeed(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
result_id: str,
|
||||
result_sha256: str,
|
||||
) -> Mapping[str, object]:
|
||||
raise AssertionError("an empty transport cannot succeed a job")
|
||||
|
||||
def fail(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
error_code: str,
|
||||
message: str,
|
||||
) -> Mapping[str, object]:
|
||||
raise AssertionError("an empty transport cannot fail a job")
|
||||
|
||||
|
||||
def test_worker_agent_allows_only_one_claim_cycle_at_a_time() -> None:
|
||||
entered = Event()
|
||||
release = Event()
|
||||
transport = BlockingEmptyTransport(entered=entered, release=release)
|
||||
agent = ObservatoryWorkerAgent(
|
||||
transport=transport,
|
||||
executors=ObservatoryWorkerExecutorRegistry(()),
|
||||
claim_request_id_factory=lambda: "worker-006:overlap-test",
|
||||
)
|
||||
reports: list[object] = []
|
||||
|
||||
first = Thread(target=lambda: reports.append(agent.run_once()))
|
||||
first.start()
|
||||
assert entered.wait(timeout=2)
|
||||
with pytest.raises(ObservatoryWorkerAgentBusyError, match="active claim"):
|
||||
agent.run_once()
|
||||
release.set()
|
||||
first.join(timeout=2)
|
||||
|
||||
assert not first.is_alive()
|
||||
assert len(reports) == 1
|
||||
@@ -0,0 +1,408 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobIntent,
|
||||
ObservatoryRecordedJobQueue,
|
||||
RecordedRunDefinition,
|
||||
RecordedRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.web.observatory_worker_api import (
|
||||
OBSERVATORY_WORKER_CONTOUR_HEADER,
|
||||
ObservatoryWorkerAuthentication,
|
||||
build_observatory_worker_router,
|
||||
load_observatory_worker_authentication,
|
||||
)
|
||||
|
||||
WORKER_TOKEN = "worker-006-test-bearer-secret-32bytes"
|
||||
WORKER_TOKEN_SHA256 = hashlib.sha256(WORKER_TOKEN.encode()).hexdigest()
|
||||
WORKER_HEADERS = {
|
||||
"Authorization": f"Bearer {WORKER_TOKEN}",
|
||||
OBSERVATORY_WORKER_CONTOUR_HEADER: "worker-006",
|
||||
}
|
||||
CLAIM_SCHEMA = "missioncore.observatory-worker-claim-request/v1"
|
||||
START_SCHEMA = "missioncore.observatory-worker-start-request/v1"
|
||||
CHECKPOINT_SCHEMA = "missioncore.observatory-worker-checkpoint-request/v1"
|
||||
SUCCEED_SCHEMA = "missioncore.observatory-worker-succeed-request/v1"
|
||||
FAIL_SCHEMA = "missioncore.observatory-worker-fail-request/v1"
|
||||
|
||||
|
||||
def _definition() -> RecordedRunDefinition:
|
||||
return RecordedRunDefinition(
|
||||
setup_id="portable-lab-v1",
|
||||
definition_id="portable-lab-v1-definition",
|
||||
definition_version=1,
|
||||
definition_sha256="1" * 64,
|
||||
source_adapter_id="sealed-session-source",
|
||||
source_adapter_version=1,
|
||||
source_adapter_sha256="2" * 64,
|
||||
executor_release_id="portable-lab-v1-worker",
|
||||
executor_release_sha256="3" * 64,
|
||||
executor_image_sha256="4" * 64,
|
||||
model_release_ids=("eomt-cityscapes-large", "ddrnet-39"),
|
||||
model_manifest_sha256="5" * 64,
|
||||
resource_profile_id="worker006-single-gpu",
|
||||
resource_profile_sha256="6" * 64,
|
||||
checkpoint_policy="cooperative",
|
||||
allowed_checkpoints=("semantic-pass",),
|
||||
)
|
||||
|
||||
|
||||
def _services(tmp_path: Path) -> tuple[TestClient, ObservatoryRecordedJobQueue]:
|
||||
definition = _definition()
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=RecordedRunDefinitionRegistry((definition,)),
|
||||
)
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_observatory_worker_router(
|
||||
queue,
|
||||
authentication=ObservatoryWorkerAuthentication(
|
||||
bearer_token_sha256=WORKER_TOKEN_SHA256,
|
||||
contour_id="worker-006",
|
||||
),
|
||||
)
|
||||
)
|
||||
return TestClient(app), queue
|
||||
|
||||
|
||||
def _enqueue(
|
||||
queue: ObservatoryRecordedJobQueue,
|
||||
*,
|
||||
idempotency_key: str = "worker-api:test-job",
|
||||
) -> str:
|
||||
definition = _definition()
|
||||
job, created = queue.submit(
|
||||
ObservatoryRecordedJobIntent(
|
||||
idempotency_key=idempotency_key,
|
||||
source_session_id="20260831T120000Z_viewer_live",
|
||||
source_catalog_sha256="7" * 64,
|
||||
source_bundle_sha256="8" * 64,
|
||||
source_capability_manifest_sha256="9" * 64,
|
||||
setup_id=definition.setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
),
|
||||
enqueue=True,
|
||||
)
|
||||
assert created is True
|
||||
return job.job_id
|
||||
|
||||
|
||||
def _claim(
|
||||
client: TestClient,
|
||||
*,
|
||||
claim_request_id: str = "worker-006:test-claim",
|
||||
) -> dict[str, Any]:
|
||||
response = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
headers=WORKER_HEADERS,
|
||||
json={
|
||||
"schema_version": CLAIM_SCHEMA,
|
||||
"claim_request_id": claim_request_id,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
return response.json()
|
||||
|
||||
|
||||
def test_worker_authentication_requires_digest_and_configured_contour(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, queue = _services(tmp_path)
|
||||
_enqueue(queue)
|
||||
request = {
|
||||
"schema_version": CLAIM_SCHEMA,
|
||||
"claim_request_id": "worker-006:auth-claim",
|
||||
}
|
||||
|
||||
missing = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
json=request,
|
||||
)
|
||||
wrong_token = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
headers={
|
||||
"Authorization": "Bearer wrong-secret",
|
||||
OBSERVATORY_WORKER_CONTOUR_HEADER: "worker-006",
|
||||
},
|
||||
json=request,
|
||||
)
|
||||
wrong_contour = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
headers={
|
||||
"Authorization": f"Bearer {WORKER_TOKEN}",
|
||||
OBSERVATORY_WORKER_CONTOUR_HEADER: "worker-007",
|
||||
},
|
||||
json=request,
|
||||
)
|
||||
|
||||
assert missing.status_code == 401
|
||||
assert missing.headers["www-authenticate"] == "Bearer"
|
||||
assert wrong_token.status_code == 401
|
||||
assert wrong_contour.status_code == 403
|
||||
assert queue.list_jobs()[0].state == "queued"
|
||||
|
||||
claimed = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
headers=WORKER_HEADERS,
|
||||
json=request,
|
||||
)
|
||||
assert claimed.status_code == 200
|
||||
assert claimed.json()["claimant_id"] == "worker-006"
|
||||
assert WORKER_TOKEN not in claimed.text
|
||||
|
||||
|
||||
def test_claim_is_idempotent_and_request_schema_rejects_execution_inputs(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, queue = _services(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
request = {
|
||||
"schema_version": CLAIM_SCHEMA,
|
||||
"claim_request_id": "worker-006:stable-claim",
|
||||
}
|
||||
|
||||
first = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
headers=WORKER_HEADERS,
|
||||
json=request,
|
||||
)
|
||||
repeated = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
headers=WORKER_HEADERS,
|
||||
json=request,
|
||||
)
|
||||
forbidden_inputs = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/start",
|
||||
headers=WORKER_HEADERS,
|
||||
json={
|
||||
"schema_version": START_SCHEMA,
|
||||
"claim_token": first.json()["claim_token"],
|
||||
"command": ["python", "worker.py"],
|
||||
"working_path": "/tmp/run",
|
||||
"environment": {"CUDA_VISIBLE_DEVICES": "0"},
|
||||
"image": "unsealed:latest",
|
||||
},
|
||||
)
|
||||
|
||||
assert first.status_code == 200
|
||||
assert repeated.status_code == 200
|
||||
assert first.json()["job"]["job_id"] == job_id
|
||||
assert repeated.json() == first.json()
|
||||
assert queue.get(job_id).claim_generation == 1
|
||||
assert forbidden_inputs.status_code == 422
|
||||
assert queue.get(job_id).state == "claimed"
|
||||
|
||||
|
||||
def test_empty_claim_is_204_and_empty_receipt_remains_idempotent(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, queue = _services(tmp_path)
|
||||
request = {
|
||||
"schema_version": CLAIM_SCHEMA,
|
||||
"claim_request_id": "worker-006:empty-poll",
|
||||
}
|
||||
|
||||
first = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
headers=WORKER_HEADERS,
|
||||
json=request,
|
||||
)
|
||||
_enqueue(queue)
|
||||
repeated = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
headers=WORKER_HEADERS,
|
||||
json=request,
|
||||
)
|
||||
fresh = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
headers=WORKER_HEADERS,
|
||||
json={
|
||||
"schema_version": CLAIM_SCHEMA,
|
||||
"claim_request_id": "worker-006:fresh-poll",
|
||||
},
|
||||
)
|
||||
|
||||
assert first.status_code == 204
|
||||
assert first.content == b""
|
||||
assert repeated.status_code == 204
|
||||
assert fresh.status_code == 200
|
||||
|
||||
|
||||
def test_worker_can_start_checkpoint_and_read_job_but_stale_token_fails_closed(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, queue = _services(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
claim = _claim(client)
|
||||
claim_token = claim["claim_token"]
|
||||
|
||||
stale = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/start",
|
||||
headers=WORKER_HEADERS,
|
||||
json={"schema_version": START_SCHEMA, "claim_token": "a" * 64},
|
||||
)
|
||||
started = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/start",
|
||||
headers=WORKER_HEADERS,
|
||||
json={"schema_version": START_SCHEMA, "claim_token": claim_token},
|
||||
)
|
||||
checkpointed = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/checkpoint",
|
||||
headers=WORKER_HEADERS,
|
||||
json={
|
||||
"schema_version": CHECKPOINT_SCHEMA,
|
||||
"claim_token": claim_token,
|
||||
"checkpoint_id": "semantic-pass",
|
||||
},
|
||||
)
|
||||
fetched = client.get(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}",
|
||||
headers=WORKER_HEADERS,
|
||||
)
|
||||
|
||||
assert stale.status_code == 409
|
||||
assert started.status_code == 200
|
||||
assert started.json()["state"] == "running"
|
||||
assert checkpointed.status_code == 200
|
||||
assert checkpointed.json()["state"] == "running"
|
||||
assert checkpointed.json()["checkpoint_policy"]["last_checkpoint_id"] == ("semantic-pass")
|
||||
assert fetched.status_code == 200
|
||||
assert fetched.json()["job_id"] == job_id
|
||||
assert "active_claim_token" not in fetched.json()
|
||||
|
||||
|
||||
def test_worker_can_publish_success_idempotently(tmp_path: Path) -> None:
|
||||
client, queue = _services(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
claim_token = _claim(client)["claim_token"]
|
||||
client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/start",
|
||||
headers=WORKER_HEADERS,
|
||||
json={"schema_version": START_SCHEMA, "claim_token": claim_token},
|
||||
)
|
||||
request = {
|
||||
"schema_version": SUCCEED_SCHEMA,
|
||||
"claim_token": claim_token,
|
||||
"result_id": "lab-v1-worker-result-001",
|
||||
"result_sha256": "b" * 64,
|
||||
}
|
||||
|
||||
succeeded = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/succeed",
|
||||
headers=WORKER_HEADERS,
|
||||
json=request,
|
||||
)
|
||||
repeated = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/succeed",
|
||||
headers=WORKER_HEADERS,
|
||||
json=request,
|
||||
)
|
||||
conflicting_failure = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/fail",
|
||||
headers=WORKER_HEADERS,
|
||||
json={
|
||||
"schema_version": FAIL_SCHEMA,
|
||||
"claim_token": claim_token,
|
||||
"error_code": "executor-failed",
|
||||
"message": "Must not replace sealed success.",
|
||||
},
|
||||
)
|
||||
|
||||
assert succeeded.status_code == 200
|
||||
assert succeeded.json()["state"] == "succeeded"
|
||||
assert succeeded.json()["result"] == {
|
||||
"result_id": "lab-v1-worker-result-001",
|
||||
"sha256": "b" * 64,
|
||||
}
|
||||
assert repeated.status_code == 200
|
||||
assert repeated.json() == succeeded.json()
|
||||
assert conflicting_failure.status_code == 409
|
||||
|
||||
|
||||
def test_worker_can_fail_claimed_job_idempotently(tmp_path: Path) -> None:
|
||||
client, queue = _services(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
claim_token = _claim(client)["claim_token"]
|
||||
request = {
|
||||
"schema_version": FAIL_SCHEMA,
|
||||
"claim_token": claim_token,
|
||||
"error_code": "model-unavailable",
|
||||
"message": "Sealed model release is not installed.",
|
||||
}
|
||||
|
||||
failed = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/fail",
|
||||
headers=WORKER_HEADERS,
|
||||
json=request,
|
||||
)
|
||||
repeated = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/fail",
|
||||
headers=WORKER_HEADERS,
|
||||
json=request,
|
||||
)
|
||||
|
||||
assert failed.status_code == 200
|
||||
assert failed.json()["state"] == "failed"
|
||||
assert failed.json()["terminal"] == {
|
||||
"code": "model-unavailable",
|
||||
"message": "Sealed model release is not installed.",
|
||||
}
|
||||
assert repeated.status_code == 200
|
||||
assert repeated.json() == failed.json()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bearer_token_sha256,contour_id",
|
||||
[("not-a-digest", "worker-006"), ("a" * 64, "WORKER 006")],
|
||||
)
|
||||
def test_worker_authentication_configuration_is_strict(
|
||||
bearer_token_sha256: str,
|
||||
contour_id: str,
|
||||
) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
ObservatoryWorkerAuthentication(
|
||||
bearer_token_sha256=bearer_token_sha256,
|
||||
contour_id=contour_id,
|
||||
)
|
||||
|
||||
|
||||
def test_worker_authentication_loads_private_regular_token_file(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
token_path = tmp_path / "observatory-worker.token"
|
||||
token_path.write_text(WORKER_TOKEN, encoding="ascii")
|
||||
token_path.chmod(0o600)
|
||||
|
||||
authentication = load_observatory_worker_authentication(token_path)
|
||||
|
||||
assert authentication == ObservatoryWorkerAuthentication(
|
||||
bearer_token_sha256=WORKER_TOKEN_SHA256,
|
||||
contour_id="worker-006",
|
||||
)
|
||||
assert WORKER_TOKEN not in repr(authentication)
|
||||
|
||||
|
||||
def test_worker_authentication_rejects_broad_permissions_and_symlinks(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
token_path = tmp_path / "observatory-worker.token"
|
||||
token_path.write_text(WORKER_TOKEN, encoding="ascii")
|
||||
token_path.chmod(0o644)
|
||||
|
||||
with pytest.raises(ValueError, match="permissions"):
|
||||
load_observatory_worker_authentication(token_path)
|
||||
|
||||
token_path.chmod(0o600)
|
||||
link = tmp_path / "linked.token"
|
||||
link.symlink_to(token_path)
|
||||
with pytest.raises(ValueError, match="credential"):
|
||||
load_observatory_worker_authentication(link)
|
||||
@@ -0,0 +1,59 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.web import app as app_module
|
||||
|
||||
WORKER_ROUTE_PREFIX = "/api/v1/worker/observatory"
|
||||
|
||||
|
||||
def test_worker_router_is_hard_disabled_until_lease_and_publisher_exist() -> None:
|
||||
assert app_module.OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||
assert app_module.OBSERVATORY_WORKER_CLAIM_LEASE_READY is False
|
||||
assert app_module.OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY is False
|
||||
assert app_module.OBSERVATORY_WORKER_PRODUCTION_API_ENABLED is False
|
||||
assert app_module.OBSERVATORY_WORKER_AUTHENTICATION is None
|
||||
assert app_module.OBSERVATORY_WORKER_API_ERROR is not None
|
||||
assert "hard-disabled" in app_module.OBSERVATORY_WORKER_API_ERROR
|
||||
assert not any(
|
||||
getattr(route, "path", "").startswith(WORKER_ROUTE_PREFIX)
|
||||
for route in app_module.app.routes
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_worker_credential_disables_only_optional_authentication(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
token_path = tmp_path / "observatory-worker.token"
|
||||
token_path.write_text("worker-006-test-bearer-secret-32bytes", encoding="ascii")
|
||||
token_path.chmod(0o644)
|
||||
|
||||
authentication, error = app_module._load_optional_observatory_worker_authentication(
|
||||
app_module.OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
token_path=token_path,
|
||||
)
|
||||
|
||||
assert authentication is None
|
||||
assert error is not None and "permissions" in error
|
||||
|
||||
|
||||
def test_valid_worker_credential_cannot_enable_production_router(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
token_path = tmp_path / "observatory-worker.token"
|
||||
token_path.write_text("worker-006-test-bearer-secret-32bytes", encoding="ascii")
|
||||
token_path.chmod(0o600)
|
||||
|
||||
authentication, error = app_module._load_optional_observatory_worker_authentication(
|
||||
app_module.OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
token_path=token_path,
|
||||
)
|
||||
|
||||
assert authentication is not None
|
||||
assert error is None
|
||||
assert app_module.OBSERVATORY_WORKER_PRODUCTION_API_ENABLED is False
|
||||
assert app_module.OBSERVATORY_WORKER_AUTHENTICATION is None
|
||||
assert not any(
|
||||
getattr(route, "path", "").startswith(WORKER_ROUTE_PREFIX)
|
||||
for route in app_module.app.routes
|
||||
)
|
||||
Reference in New Issue
Block a user