feat(observatory): add portable calculation profiles
This commit is contained in:
@@ -0,0 +1,140 @@
|
|||||||
|
export const OBSERVATION_LAB_CALCULATION_PROFILE_SCHEMA =
|
||||||
|
"missioncore.observatory-calculation-profile/v1" as const;
|
||||||
|
|
||||||
|
export type ObservationLabCalculationProfileOrigin =
|
||||||
|
| "archived-definition"
|
||||||
|
| "existing-result";
|
||||||
|
|
||||||
|
export interface ObservationLabCalculationProfile {
|
||||||
|
readonly schemaVersion: typeof OBSERVATION_LAB_CALCULATION_PROFILE_SCHEMA;
|
||||||
|
readonly setupId: string;
|
||||||
|
readonly displayName: string;
|
||||||
|
readonly origin: ObservationLabCalculationProfileOrigin;
|
||||||
|
readonly definitionId: string | null;
|
||||||
|
readonly definitionVersion: number | null;
|
||||||
|
readonly definitionSha256: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ObservationLabCalculationProfileContractError extends Error {
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message);
|
||||||
|
this.name = "ObservationLabCalculationProfileContractError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const PROFILE_KEYS = new Set([
|
||||||
|
"schema_version",
|
||||||
|
"setup_id",
|
||||||
|
"display_name",
|
||||||
|
"origin",
|
||||||
|
"definition_id",
|
||||||
|
"definition_version",
|
||||||
|
"definition_sha256",
|
||||||
|
]);
|
||||||
|
const IDENTIFIER = /^[a-z][a-z0-9-]{2,95}$/;
|
||||||
|
const SHA256 = /^[a-f0-9]{64}$/;
|
||||||
|
|
||||||
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireText(value: unknown, field: string, maximum: number): string {
|
||||||
|
if (
|
||||||
|
typeof value !== "string"
|
||||||
|
|| value.length === 0
|
||||||
|
|| value.length > maximum
|
||||||
|
|| value !== value.trim()
|
||||||
|
) {
|
||||||
|
throw new ObservationLabCalculationProfileContractError(
|
||||||
|
`Поле calculation_profile.${field} должно быть непустой строкой.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireIdentifier(value: unknown, field: string): string {
|
||||||
|
const identifier = requireText(value, field, 96);
|
||||||
|
if (!IDENTIFIER.test(identifier)) {
|
||||||
|
throw new ObservationLabCalculationProfileContractError(
|
||||||
|
`Поле calculation_profile.${field} содержит некорректный идентификатор.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return identifier;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function decodeObservationLabCalculationProfile(
|
||||||
|
value: unknown,
|
||||||
|
sessionId: string,
|
||||||
|
): ObservationLabCalculationProfile | null {
|
||||||
|
if (value === null) return null;
|
||||||
|
if (!isRecord(value) || Object.keys(value).some((key) => !PROFILE_KEYS.has(key))) {
|
||||||
|
throw new ObservationLabCalculationProfileContractError(
|
||||||
|
`Профиль расчёта LAB-сессии ${sessionId} нарушил контракт полей.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
Object.keys(value).length !== PROFILE_KEYS.size
|
||||||
|
|| value.schema_version !== OBSERVATION_LAB_CALCULATION_PROFILE_SCHEMA
|
||||||
|
) {
|
||||||
|
throw new ObservationLabCalculationProfileContractError(
|
||||||
|
`Профиль расчёта LAB-сессии ${sessionId} имеет неизвестную версию.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const origin = value.origin;
|
||||||
|
if (origin !== "archived-definition" && origin !== "existing-result") {
|
||||||
|
throw new ObservationLabCalculationProfileContractError(
|
||||||
|
`Профиль расчёта LAB-сессии ${sessionId} имеет неизвестное происхождение.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const definitionId = value.definition_id === null
|
||||||
|
? null
|
||||||
|
: requireIdentifier(value.definition_id, "definition_id");
|
||||||
|
const definitionVersion = value.definition_version === null
|
||||||
|
? null
|
||||||
|
: value.definition_version;
|
||||||
|
if (
|
||||||
|
definitionVersion !== null
|
||||||
|
&& (
|
||||||
|
typeof definitionVersion !== "number"
|
||||||
|
|| !Number.isInteger(definitionVersion)
|
||||||
|
|| definitionVersion < 1
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
throw new ObservationLabCalculationProfileContractError(
|
||||||
|
`Профиль расчёта LAB-сессии ${sessionId} содержит некорректную версию определения.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const definitionSha256 = value.definition_sha256 === null
|
||||||
|
? null
|
||||||
|
: requireText(value.definition_sha256, "definition_sha256", 64);
|
||||||
|
if (definitionSha256 !== null && !SHA256.test(definitionSha256)) {
|
||||||
|
throw new ObservationLabCalculationProfileContractError(
|
||||||
|
`Профиль расчёта LAB-сессии ${sessionId} не содержит SHA-256 определения.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const definitionFieldCount = [
|
||||||
|
definitionId,
|
||||||
|
definitionVersion,
|
||||||
|
definitionSha256,
|
||||||
|
].filter((entry) => entry !== null).length;
|
||||||
|
if (
|
||||||
|
(origin === "existing-result" && definitionFieldCount !== 0)
|
||||||
|
|| (origin === "archived-definition" && definitionFieldCount !== 3)
|
||||||
|
) {
|
||||||
|
throw new ObservationLabCalculationProfileContractError(
|
||||||
|
`Профиль расчёта LAB-сессии ${sessionId} содержит неполную идентичность определения.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
schemaVersion: OBSERVATION_LAB_CALCULATION_PROFILE_SCHEMA,
|
||||||
|
setupId: requireIdentifier(value.setup_id, "setup_id"),
|
||||||
|
displayName: requireText(value.display_name, "display_name", 256),
|
||||||
|
origin,
|
||||||
|
definitionId,
|
||||||
|
definitionVersion,
|
||||||
|
definitionSha256,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -7,8 +7,14 @@ import {
|
|||||||
ObservationLabReplayCapabilityContractError,
|
ObservationLabReplayCapabilityContractError,
|
||||||
type ObservationLabReplayCapability,
|
type ObservationLabReplayCapability,
|
||||||
} from "./labReplayCapability";
|
} from "./labReplayCapability";
|
||||||
|
import {
|
||||||
|
decodeObservationLabCalculationProfile,
|
||||||
|
ObservationLabCalculationProfileContractError,
|
||||||
|
type ObservationLabCalculationProfile,
|
||||||
|
} from "./labCalculationProfile";
|
||||||
|
|
||||||
export type { ObservationLabReplayCapability } from "./labReplayCapability";
|
export type { ObservationLabReplayCapability } from "./labReplayCapability";
|
||||||
|
export type { ObservationLabCalculationProfile } from "./labCalculationProfile";
|
||||||
|
|
||||||
export type ObservationSessionStatus =
|
export type ObservationSessionStatus =
|
||||||
| "recording"
|
| "recording"
|
||||||
@@ -29,6 +35,7 @@ export interface ObservationLabInstance {
|
|||||||
runCreatedAtUtc: string;
|
runCreatedAtUtc: string;
|
||||||
publishedAtUtc: string;
|
publishedAtUtc: string;
|
||||||
replayCapability: ObservationLabReplayCapability | null;
|
replayCapability: ObservationLabReplayCapability | null;
|
||||||
|
calculationProfile: ObservationLabCalculationProfile | null;
|
||||||
provenance: Readonly<Record<string, unknown>>;
|
provenance: Readonly<Record<string, unknown>>;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -175,7 +182,11 @@ const LEGACY_LAB_KEYS = new Set([
|
|||||||
"published_at_utc",
|
"published_at_utc",
|
||||||
"provenance",
|
"provenance",
|
||||||
]);
|
]);
|
||||||
const LAB_KEYS = new Set([...LEGACY_LAB_KEYS, "replay_capability"]);
|
const LAB_V2_KEYS = new Set([...LEGACY_LAB_KEYS, "replay_capability"]);
|
||||||
|
const LAB_V3_KEYS = new Set([
|
||||||
|
...LAB_V2_KEYS,
|
||||||
|
"calculation_profile",
|
||||||
|
]);
|
||||||
const CATALOG_PREPARATION_KEYS = new Set([
|
const CATALOG_PREPARATION_KEYS = new Set([
|
||||||
"preparation_id",
|
"preparation_id",
|
||||||
"state",
|
"state",
|
||||||
@@ -408,9 +419,17 @@ function decodeLabInstance(
|
|||||||
value,
|
value,
|
||||||
"replay_capability",
|
"replay_capability",
|
||||||
);
|
);
|
||||||
|
const hasCalculationProfile = Object.prototype.hasOwnProperty.call(
|
||||||
|
value,
|
||||||
|
"calculation_profile",
|
||||||
|
);
|
||||||
assertExactKeys(
|
assertExactKeys(
|
||||||
value,
|
value,
|
||||||
hasTypedCapability ? LAB_KEYS : LEGACY_LAB_KEYS,
|
hasCalculationProfile
|
||||||
|
? LAB_V3_KEYS
|
||||||
|
: hasTypedCapability
|
||||||
|
? LAB_V2_KEYS
|
||||||
|
: LEGACY_LAB_KEYS,
|
||||||
`LAB-привязка сессии ${sessionId}`,
|
`LAB-привязка сессии ${sessionId}`,
|
||||||
);
|
);
|
||||||
const labId = requireString(value.lab_id, `lab(${sessionId}).lab_id`, 36);
|
const labId = requireString(value.lab_id, `lab(${sessionId}).lab_id`, 36);
|
||||||
@@ -459,6 +478,17 @@ function decodeLabInstance(
|
|||||||
}
|
}
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
|
let calculationProfile: ObservationLabCalculationProfile | null;
|
||||||
|
try {
|
||||||
|
calculationProfile = hasCalculationProfile
|
||||||
|
? decodeObservationLabCalculationProfile(value.calculation_profile, sessionId)
|
||||||
|
: null;
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof ObservationLabCalculationProfileContractError) {
|
||||||
|
throw new ObservationSessionContractError(error.message);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
labId,
|
labId,
|
||||||
sourceSessionId,
|
sourceSessionId,
|
||||||
@@ -475,6 +505,7 @@ function decodeLabInstance(
|
|||||||
`lab(${sessionId}).published_at_utc`,
|
`lab(${sessionId}).published_at_utc`,
|
||||||
),
|
),
|
||||||
replayCapability,
|
replayCapability,
|
||||||
|
calculationProfile,
|
||||||
provenance: value.provenance,
|
provenance: value.provenance,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -1211,7 +1242,7 @@ export async function fetchObservationSessionCatalog({
|
|||||||
queryParameters.set("limit", String(Number(limit)));
|
queryParameters.set("limit", String(Number(limit)));
|
||||||
}
|
}
|
||||||
if (scope !== "all") queryParameters.set("scope", scope);
|
if (scope !== "all") queryParameters.set("scope", scope);
|
||||||
if (scope === "laboratory") queryParameters.set("lab_contract", "v2");
|
if (scope === "laboratory") queryParameters.set("lab_contract", "v3");
|
||||||
const serializedQuery = queryParameters.toString();
|
const serializedQuery = queryParameters.toString();
|
||||||
const query = serializedQuery ? `?${serializedQuery}` : "";
|
const query = serializedQuery ? `?${serializedQuery}` : "";
|
||||||
let response: Response;
|
let response: Response;
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
import { preflightObservatoryPortableLaboratorySetup } from "./portableLaboratorySetups";
|
|
||||||
|
|
||||||
export { fetchObservatoryPortableLaboratorySetups } from "./portableLaboratorySetups";
|
export { fetchObservatoryPortableLaboratorySetups } from "./portableLaboratorySetups";
|
||||||
|
|
||||||
const CATALOG_SCHEMA = "missioncore.observatory-laboratory-setup-catalog/v1";
|
const CATALOG_SCHEMA = "missioncore.observatory-laboratory-setup-catalog/v1";
|
||||||
@@ -14,6 +12,7 @@ export type ObservatoryLaboratorySetupOrigin =
|
|||||||
export type ObservatoryLaboratorySetupAction =
|
export type ObservatoryLaboratorySetupAction =
|
||||||
| "open-existing"
|
| "open-existing"
|
||||||
| "open-legacy"
|
| "open-legacy"
|
||||||
|
| "check"
|
||||||
| "blocked";
|
| "blocked";
|
||||||
|
|
||||||
export interface ObservatoryLaboratoryRunDefinition {
|
export interface ObservatoryLaboratoryRunDefinition {
|
||||||
@@ -62,7 +61,7 @@ export interface ObservatoryLaboratorySetup {
|
|||||||
};
|
};
|
||||||
readonly preservedResults: readonly ObservatoryLaboratoryPreservedResult[];
|
readonly preservedResults: readonly ObservatoryLaboratoryPreservedResult[];
|
||||||
readonly preflight: {
|
readonly preflight: {
|
||||||
readonly outcome: "existing" | "blocked";
|
readonly outcome: "existing" | "ready" | "blocked";
|
||||||
readonly action: ObservatoryLaboratorySetupAction;
|
readonly action: ObservatoryLaboratorySetupAction;
|
||||||
readonly reason: string;
|
readonly reason: string;
|
||||||
readonly submissionAllowed: boolean;
|
readonly submissionAllowed: boolean;
|
||||||
@@ -79,6 +78,7 @@ export interface ObservatoryLaboratoryRunPreflight {
|
|||||||
readonly sourceSessionId: string;
|
readonly sourceSessionId: string;
|
||||||
readonly setupId: string;
|
readonly setupId: string;
|
||||||
readonly definitionSha256: string | null;
|
readonly definitionSha256: string | null;
|
||||||
|
readonly checkSha256: string | null;
|
||||||
readonly outcome: "existing" | "queueable" | "blocked";
|
readonly outcome: "existing" | "queueable" | "blocked";
|
||||||
readonly submissionAllowed: boolean;
|
readonly submissionAllowed: boolean;
|
||||||
readonly checks: readonly {
|
readonly checks: readonly {
|
||||||
@@ -142,9 +142,6 @@ export async function preflightObservatoryLaboratorySetup(
|
|||||||
fetcher?: ObservatoryLaboratorySetupFetch;
|
fetcher?: ObservatoryLaboratorySetupFetch;
|
||||||
} = {},
|
} = {},
|
||||||
): Promise<ObservatoryLaboratoryRunPreflight> {
|
): Promise<ObservatoryLaboratoryRunPreflight> {
|
||||||
if (setup.origin === "portable-definition") {
|
|
||||||
return preflightObservatoryPortableLaboratorySetup(sourceSessionId, setup);
|
|
||||||
}
|
|
||||||
const response = await request(
|
const response = await request(
|
||||||
fetcher,
|
fetcher,
|
||||||
"/api/v1/observatory/run-preflights",
|
"/api/v1/observatory/run-preflights",
|
||||||
@@ -297,29 +294,55 @@ function decodePreservedResult(value: unknown): ObservatoryLaboratoryPreservedRe
|
|||||||
|
|
||||||
function decodePreflight(value: unknown): ObservatoryLaboratoryRunPreflight {
|
function decodePreflight(value: unknown): ObservatoryLaboratoryRunPreflight {
|
||||||
const row = record(value, "preflight");
|
const row = record(value, "preflight");
|
||||||
exactKeys(row, [
|
const baseKeys = [
|
||||||
"authority", "checks", "definition_sha256", "executor", "existing_result_ids",
|
"authority", "checks", "definition_sha256", "executor", "existing_result_ids",
|
||||||
"outcome", "schema_version", "setup_id", "source_session_id", "submission_allowed",
|
"outcome", "schema_version", "setup_id", "source_session_id", "submission_allowed",
|
||||||
], "preflight");
|
] as const;
|
||||||
|
const hasPortableCheck = Object.hasOwn(row, "check_sha256");
|
||||||
|
exactKeys(
|
||||||
|
row,
|
||||||
|
hasPortableCheck ? [...baseKeys, "check_sha256"] : baseKeys,
|
||||||
|
"preflight",
|
||||||
|
);
|
||||||
exact(row.schema_version, PREFLIGHT_SCHEMA, "preflight schema");
|
exact(row.schema_version, PREFLIGHT_SCHEMA, "preflight schema");
|
||||||
observationAuthority(row.authority);
|
observationAuthority(row.authority);
|
||||||
const digest = row.definition_sha256;
|
const digest = row.definition_sha256;
|
||||||
if (digest !== null && (typeof digest !== "string" || !SHA256.test(digest))) {
|
if (digest !== null && (typeof digest !== "string" || !SHA256.test(digest))) {
|
||||||
throw new ObservatoryLaboratorySetupContractError("Некорректный digest preflight.");
|
throw new ObservatoryLaboratorySetupContractError("Некорректный digest preflight.");
|
||||||
}
|
}
|
||||||
|
const checkDigest = hasPortableCheck ? row.check_sha256 : null;
|
||||||
|
if (
|
||||||
|
checkDigest !== null
|
||||||
|
&& (typeof checkDigest !== "string" || !SHA256.test(checkDigest))
|
||||||
|
) {
|
||||||
|
throw new ObservatoryLaboratorySetupContractError(
|
||||||
|
"Некорректный check digest preflight.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const outcome = oneOf(
|
||||||
|
row.outcome,
|
||||||
|
["existing", "queueable", "blocked"] as const,
|
||||||
|
"preflight outcome",
|
||||||
|
);
|
||||||
|
const submissionAllowed = boolean(
|
||||||
|
row.submission_allowed,
|
||||||
|
"preflight submission_allowed",
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
submissionAllowed !== (outcome === "queueable")
|
||||||
|
|| (hasPortableCheck && outcome === "queueable" && checkDigest === null)
|
||||||
|
) {
|
||||||
|
throw new ObservatoryLaboratorySetupContractError(
|
||||||
|
"Preflight содержит противоречивое разрешение постановки в очередь.",
|
||||||
|
);
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
sourceSessionId: text(row.source_session_id, "preflight source_session_id"),
|
sourceSessionId: text(row.source_session_id, "preflight source_session_id"),
|
||||||
setupId: text(row.setup_id, "preflight setup_id"),
|
setupId: text(row.setup_id, "preflight setup_id"),
|
||||||
definitionSha256: digest,
|
definitionSha256: digest,
|
||||||
outcome: oneOf(
|
checkSha256: checkDigest,
|
||||||
row.outcome,
|
outcome,
|
||||||
["existing", "queueable", "blocked"] as const,
|
submissionAllowed,
|
||||||
"preflight outcome",
|
|
||||||
),
|
|
||||||
submissionAllowed: boolean(
|
|
||||||
row.submission_allowed,
|
|
||||||
"preflight submission_allowed",
|
|
||||||
),
|
|
||||||
checks: array(row.checks, "preflight checks").map((item) => {
|
checks: array(row.checks, "preflight checks").map((item) => {
|
||||||
const check = record(item, "preflight check");
|
const check = record(item, "preflight check");
|
||||||
exactKeys(check, ["check_id", "message", "outcome", "reason_code"], "preflight check");
|
exactKeys(check, ["check_id", "message", "outcome", "reason_code"], "preflight check");
|
||||||
|
|||||||
@@ -55,7 +55,11 @@ function decodePortableSetup(value: unknown): ObservatoryLaboratorySetup {
|
|||||||
const compatibilityReason = text(compatibility.reason, "portable compatibility reason");
|
const compatibilityReason = text(compatibility.reason, "portable compatibility reason");
|
||||||
|
|
||||||
const executor = record(row.executor, "portable executor");
|
const executor = record(row.executor, "portable executor");
|
||||||
exactKeys(executor, ["contour_id", "ready", "reason", "state"], "portable executor");
|
exactKeys(
|
||||||
|
executor,
|
||||||
|
["contour_id", "ready", "reason", "reason_code", "state"],
|
||||||
|
"portable executor",
|
||||||
|
);
|
||||||
const executorState = oneOf(
|
const executorState = oneOf(
|
||||||
executor.state,
|
executor.state,
|
||||||
["not-installed", "ready"] as const,
|
["not-installed", "ready"] as const,
|
||||||
@@ -70,20 +74,43 @@ function decodePortableSetup(value: unknown): ObservatoryLaboratorySetup {
|
|||||||
const executorReason = executor.reason === null
|
const executorReason = executor.reason === null
|
||||||
? "Исполнитель установлен."
|
? "Исполнитель установлен."
|
||||||
: text(executor.reason, "portable executor reason");
|
: text(executor.reason, "portable executor reason");
|
||||||
|
const executorReasonCode = executor.reason_code === null
|
||||||
|
? "portable-executor-ready"
|
||||||
|
: text(executor.reason_code, "portable executor reason_code");
|
||||||
|
if (
|
||||||
|
(executorReady && (executor.reason !== null || executor.reason_code !== null))
|
||||||
|
|| (!executorReady && (executor.reason === null || executor.reason_code === null))
|
||||||
|
) {
|
||||||
|
throw new ObservatoryPortableSetupDecodeError(
|
||||||
|
"Portable executor: причина недоступности противоречит состоянию.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const preflight = record(row.preflight, "portable preflight");
|
const preflight = record(row.preflight, "portable preflight");
|
||||||
exactKeys(preflight, [
|
exactKeys(preflight, [
|
||||||
"action", "existing_result_ids", "outcome", "reason", "submission_allowed",
|
"action", "existing_result_ids", "outcome", "reason", "submission_allowed",
|
||||||
], "portable preflight");
|
], "portable preflight");
|
||||||
exact(preflight.outcome, "blocked", "portable preflight outcome");
|
const preflightOutcome = oneOf(
|
||||||
exact(preflight.action, "blocked", "portable preflight action");
|
preflight.outcome,
|
||||||
|
["ready", "blocked"] as const,
|
||||||
|
"portable preflight outcome",
|
||||||
|
);
|
||||||
|
const preflightAction = oneOf(
|
||||||
|
preflight.action,
|
||||||
|
["check", "blocked"] as const,
|
||||||
|
"portable preflight action",
|
||||||
|
);
|
||||||
const submissionAllowed = boolean(
|
const submissionAllowed = boolean(
|
||||||
preflight.submission_allowed,
|
preflight.submission_allowed,
|
||||||
"portable preflight submission_allowed",
|
"portable preflight submission_allowed",
|
||||||
);
|
);
|
||||||
if (submissionAllowed) {
|
if (
|
||||||
|
submissionAllowed !== (preflightOutcome === "ready")
|
||||||
|
|| (preflightOutcome === "ready" && preflightAction !== "check")
|
||||||
|
|| (preflightOutcome === "blocked" && preflightAction !== "blocked")
|
||||||
|
) {
|
||||||
throw new ObservatoryPortableSetupDecodeError(
|
throw new ObservatoryPortableSetupDecodeError(
|
||||||
"Portable preflight: постановка в очередь ещё не поддерживается.",
|
"Portable preflight: состояние запуска противоречиво.",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const existingResults = array(row.existing_results, "portable existing_results");
|
const existingResults = array(row.existing_results, "portable existing_results");
|
||||||
@@ -112,15 +139,13 @@ function decodePortableSetup(value: unknown): ObservatoryLaboratorySetup {
|
|||||||
executor: {
|
executor: {
|
||||||
contourId: text(executor.contour_id, "portable executor contour_id"),
|
contourId: text(executor.contour_id, "portable executor contour_id"),
|
||||||
state: executorState,
|
state: executorState,
|
||||||
reasonCode: executorReady
|
reasonCode: executorReasonCode,
|
||||||
? "portable-executor-ready"
|
|
||||||
: "portable-executor-not-installed",
|
|
||||||
reason: executorReason,
|
reason: executorReason,
|
||||||
},
|
},
|
||||||
preservedResults: [],
|
preservedResults: [],
|
||||||
preflight: {
|
preflight: {
|
||||||
outcome: "blocked",
|
outcome: preflightOutcome,
|
||||||
action: "blocked",
|
action: preflightAction,
|
||||||
reason: text(preflight.reason, "portable preflight reason"),
|
reason: text(preflight.reason, "portable preflight reason"),
|
||||||
submissionAllowed,
|
submissionAllowed,
|
||||||
existingResultIds: [],
|
existingResultIds: [],
|
||||||
|
|||||||
@@ -1,7 +1,5 @@
|
|||||||
import { decodePortableCatalog } from "./portableLaboratorySetupDecoder";
|
import { decodePortableCatalog } from "./portableLaboratorySetupDecoder";
|
||||||
import type {
|
import type {
|
||||||
ObservatoryLaboratoryRunPreflight,
|
|
||||||
ObservatoryLaboratorySetup,
|
|
||||||
ObservatoryLaboratorySetupCatalog,
|
ObservatoryLaboratorySetupCatalog,
|
||||||
} from "./laboratorySetups";
|
} from "./laboratorySetups";
|
||||||
|
|
||||||
@@ -46,61 +44,6 @@ export async function fetchObservatoryPortableLaboratorySetups(
|
|||||||
return catalog;
|
return catalog;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function preflightObservatoryPortableLaboratorySetup(
|
|
||||||
sourceSessionId: string,
|
|
||||||
setup: ObservatoryLaboratorySetup,
|
|
||||||
): ObservatoryLaboratoryRunPreflight {
|
|
||||||
const definitionSha256 = setup.runDefinition?.definitionSha256 ?? null;
|
|
||||||
if (setup.origin !== "portable-definition" || definitionSha256 === null) {
|
|
||||||
throw new ObservatoryPortableLaboratorySetupContractError(
|
|
||||||
"Portable-сетап не содержит RunDefinition.",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
setup.preflight.outcome !== "blocked"
|
|
||||||
|| setup.preflight.action !== "blocked"
|
|
||||||
|| setup.preflight.existingResultIds.length > 0
|
|
||||||
|| setup.preservedResults.length > 0
|
|
||||||
) {
|
|
||||||
throw new ObservatoryPortableLaboratorySetupContractError(
|
|
||||||
"Portable-result ещё не имеет проверяемой привязки к RunDefinition.",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
sourceSessionId,
|
|
||||||
setupId: setup.setupId,
|
|
||||||
definitionSha256,
|
|
||||||
outcome: "blocked",
|
|
||||||
submissionAllowed: false,
|
|
||||||
checks: [
|
|
||||||
{
|
|
||||||
checkId: "source-compatibility",
|
|
||||||
outcome: setup.compatibility.compatible ? "pass" : "fail",
|
|
||||||
reasonCode: setup.compatibility.compatible
|
|
||||||
? "source-capability-admitted"
|
|
||||||
: "source-capability-blocked",
|
|
||||||
message: setup.compatibility.compatible
|
|
||||||
? "Запись соответствует portable-профилю LAB V1."
|
|
||||||
: setup.compatibility.reasons[0]?.message
|
|
||||||
?? "Запись не соответствует portable-профилю LAB V1.",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
checkId: "executor",
|
|
||||||
outcome: setup.executor.state === "ready" ? "pass" : "fail",
|
|
||||||
reasonCode: setup.executor.reasonCode,
|
|
||||||
message: setup.executor.reason,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
checkId: "durable-queue",
|
|
||||||
outcome: "fail",
|
|
||||||
reasonCode: "portable-dispatch-unavailable",
|
|
||||||
message: setup.preflight.reason,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
existingResultIds: [],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function request(
|
async function request(
|
||||||
fetcher: ObservatoryPortableLaboratorySetupFetch,
|
fetcher: ObservatoryPortableLaboratorySetupFetch,
|
||||||
input: string,
|
input: string,
|
||||||
|
|||||||
@@ -79,6 +79,10 @@ export async function submitObservatoryRecordedJob(
|
|||||||
sourceSessionId: string,
|
sourceSessionId: string,
|
||||||
setupId: string,
|
setupId: string,
|
||||||
idempotencyKey: string,
|
idempotencyKey: string,
|
||||||
|
portableBinding: {
|
||||||
|
readonly definitionSha256: string;
|
||||||
|
readonly checkSha256: string;
|
||||||
|
} | null,
|
||||||
{
|
{
|
||||||
signal,
|
signal,
|
||||||
fetcher = globalThis.fetch,
|
fetcher = globalThis.fetch,
|
||||||
@@ -95,6 +99,10 @@ export async function submitObservatoryRecordedJob(
|
|||||||
idempotency_key: idempotencyKey,
|
idempotency_key: idempotencyKey,
|
||||||
source_session_id: sourceSessionId,
|
source_session_id: sourceSessionId,
|
||||||
setup_id: setupId,
|
setup_id: setupId,
|
||||||
|
...(portableBinding === null ? {} : {
|
||||||
|
definition_sha256: portableBinding.definitionSha256,
|
||||||
|
check_sha256: portableBinding.checkSha256,
|
||||||
|
}),
|
||||||
}),
|
}),
|
||||||
signal,
|
signal,
|
||||||
});
|
});
|
||||||
@@ -112,7 +120,7 @@ export async function submitObservatoryRecordedJob(
|
|||||||
function decodeJob(value: unknown): ObservatoryRecordedJob {
|
function decodeJob(value: unknown): ObservatoryRecordedJob {
|
||||||
const row = record(value, "расчёт");
|
const row = record(value, "расчёт");
|
||||||
exactKeys(row, [
|
exactKeys(row, [
|
||||||
"authority", "checkpoint_policy", "claim_generation", "created_at_utc", "executor",
|
"authority", "checkpoint_policy", "claim_generation", "claim_lease", "created_at_utc", "executor",
|
||||||
"idempotency_key", "identity_sha256", "job_id", "preemption_receipt_sha256",
|
"idempotency_key", "identity_sha256", "job_id", "preemption_receipt_sha256",
|
||||||
"preemption_requested", "priority", "request_sha256", "restart_from_zero", "result",
|
"preemption_requested", "priority", "request_sha256", "restart_from_zero", "result",
|
||||||
"schema_version", "setup", "source", "state", "submission_receipt_sha256", "terminal",
|
"schema_version", "setup", "source", "state", "submission_receipt_sha256", "terminal",
|
||||||
@@ -137,6 +145,18 @@ function decodeJob(value: unknown): ObservatoryRecordedJob {
|
|||||||
"accepted", "queued", "claimed", "running", "paused", "preemption-pending",
|
"accepted", "queued", "claimed", "running", "paused", "preemption-pending",
|
||||||
"succeeded", "failed", "reconciliation-required",
|
"succeeded", "failed", "reconciliation-required",
|
||||||
] as const, "state");
|
] as const, "state");
|
||||||
|
if (row.claim_lease !== null) {
|
||||||
|
const lease = record(row.claim_lease, "claim_lease");
|
||||||
|
exactKeys(
|
||||||
|
lease,
|
||||||
|
["claimed_at_utc", "expires_at_utc", "heartbeat_at_utc", "renewal_count"],
|
||||||
|
"claim_lease",
|
||||||
|
);
|
||||||
|
text(lease.claimed_at_utc, "claim_lease.claimed_at_utc");
|
||||||
|
text(lease.expires_at_utc, "claim_lease.expires_at_utc");
|
||||||
|
text(lease.heartbeat_at_utc, "claim_lease.heartbeat_at_utc");
|
||||||
|
nonNegativeInteger(lease.renewal_count, "claim_lease.renewal_count");
|
||||||
|
}
|
||||||
const result = row.result === null ? null : record(row.result, "result");
|
const result = row.result === null ? null : record(row.result, "result");
|
||||||
if (result !== null) exactKeys(result, ["result_id", "sha256"], "result");
|
if (result !== null) exactKeys(result, ["result_id", "sha256"], "result");
|
||||||
const terminal = row.terminal === null ? null : record(row.terminal, "terminal");
|
const terminal = row.terminal === null ? null : record(row.terminal, "terminal");
|
||||||
@@ -220,6 +240,13 @@ function boolean(value: unknown, label: string): boolean {
|
|||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function nonNegativeInteger(value: unknown, label: string): number {
|
||||||
|
if (!Number.isInteger(value) || Number(value) < 0) {
|
||||||
|
throw new ObservatoryRecordedJobContractError(`${label}: ожидалось целое число.`);
|
||||||
|
}
|
||||||
|
return Number(value);
|
||||||
|
}
|
||||||
|
|
||||||
function exact<T>(value: unknown, expected: T, label: string): T {
|
function exact<T>(value: unknown, expected: T, label: string): T {
|
||||||
if (value !== expected) throw new ObservatoryRecordedJobContractError(`${label}: значение изменилось.`);
|
if (value !== expected) throw new ObservatoryRecordedJobContractError(`${label}: значение изменилось.`);
|
||||||
return expected;
|
return expected;
|
||||||
|
|||||||
@@ -73,7 +73,7 @@ export function useObservatoryLaboratorySetups(sourceSessionId: string) {
|
|||||||
publishSetupCatalog(legacyCatalog, setCatalog, setSelectedSetupId);
|
publishSetupCatalog(legacyCatalog, setCatalog, setSelectedSetupId);
|
||||||
setError(catalogErrorMessage(
|
setError(catalogErrorMessage(
|
||||||
caught,
|
caught,
|
||||||
"Portable-каталог LAB V1 нарушил локальный контракт.",
|
"Portable-каталог профилей нарушил локальный контракт.",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
@@ -81,7 +81,7 @@ export function useObservatoryLaboratorySetups(sourceSessionId: string) {
|
|||||||
publishSetupCatalog(legacyCatalog, setCatalog, setSelectedSetupId);
|
publishSetupCatalog(legacyCatalog, setCatalog, setSelectedSetupId);
|
||||||
setError(catalogErrorMessage(
|
setError(catalogErrorMessage(
|
||||||
optionalPortable.reason,
|
optionalPortable.reason,
|
||||||
"Portable-каталог LAB V1 недоступен.",
|
"Portable-каталог профилей недоступен.",
|
||||||
));
|
));
|
||||||
})
|
})
|
||||||
.catch((caught: unknown) => {
|
.catch((caught: unknown) => {
|
||||||
@@ -195,7 +195,15 @@ function mergeSetupCatalogs(
|
|||||||
if (legacy.sourceSessionId !== portable.sourceSessionId) {
|
if (legacy.sourceSessionId !== portable.sourceSessionId) {
|
||||||
throw new Error("Каталоги сетапов относятся к разным исходным сессиям.");
|
throw new Error("Каталоги сетапов относятся к разным исходным сессиям.");
|
||||||
}
|
}
|
||||||
const setups = [...legacy.setups, ...portable.setups];
|
const portableProfileNames = new Set(
|
||||||
|
portable.setups.map((setup) => setup.displayName),
|
||||||
|
);
|
||||||
|
const setups = [
|
||||||
|
...legacy.setups.filter(
|
||||||
|
(setup) => !portableProfileNames.has(setup.displayName),
|
||||||
|
),
|
||||||
|
...portable.setups,
|
||||||
|
];
|
||||||
if (new Set(setups.map((setup) => setup.setupId)).size !== setups.length) {
|
if (new Set(setups.map((setup) => setup.setupId)).size !== setups.length) {
|
||||||
throw new Error("Каталоги сетапов содержат повторяющиеся идентификаторы.");
|
throw new Error("Каталоги сетапов содержат повторяющиеся идентификаторы.");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -68,7 +68,12 @@ export function useObservatoryRecordedJobs(sourceSessionId: string, setupId: str
|
|||||||
|
|
||||||
const refresh = useCallback(() => setRevision((value) => value + 1), []);
|
const refresh = useCallback(() => setRevision((value) => value + 1), []);
|
||||||
|
|
||||||
const submit = useCallback(async (): Promise<ObservatoryRecordedJob | null> => {
|
const submit = useCallback(async (
|
||||||
|
portableBinding: {
|
||||||
|
readonly definitionSha256: string;
|
||||||
|
readonly checkSha256: string;
|
||||||
|
} | null = null,
|
||||||
|
): Promise<ObservatoryRecordedJob | null> => {
|
||||||
if (!sourceSessionId || !setupId || state === "submitting") return null;
|
if (!sourceSessionId || !setupId || state === "submitting") return null;
|
||||||
if (activeJob) return activeJob;
|
if (activeJob) return activeJob;
|
||||||
requestRef.current?.abort();
|
requestRef.current?.abort();
|
||||||
@@ -80,9 +85,13 @@ export function useObservatoryRecordedJobs(sourceSessionId: string, setupId: str
|
|||||||
setState("submitting");
|
setState("submitting");
|
||||||
setError(null);
|
setError(null);
|
||||||
try {
|
try {
|
||||||
const job = await submitObservatoryRecordedJob(sourceSessionId, setupId, key, {
|
const job = await submitObservatoryRecordedJob(
|
||||||
signal: request.signal,
|
sourceSessionId,
|
||||||
});
|
setupId,
|
||||||
|
key,
|
||||||
|
portableBinding,
|
||||||
|
{ signal: request.signal },
|
||||||
|
);
|
||||||
if (request.signal.aborted || requestRef.current !== request) return null;
|
if (request.signal.aborted || requestRef.current !== request) return null;
|
||||||
setJobs((current) => [job, ...current.filter((candidate) => candidate.jobId !== job.jobId)]);
|
setJobs((current) => [job, ...current.filter((candidate) => candidate.jobId !== job.jobId)]);
|
||||||
setState("ready");
|
setState("ready");
|
||||||
|
|||||||
@@ -150,6 +150,11 @@ function mutationErrorMessage(error: unknown): string {
|
|||||||
: "Не удалось изменить лабораторный результат в Обсерватории.";
|
: "Не удалось изменить лабораторный результат в Обсерватории.";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function evidenceResultSubtitle(evidence: ObservatoryEvidence): string {
|
||||||
|
const profileName = evidence.lab.calculationProfile?.displayName;
|
||||||
|
return profileName ? `${evidence.label} · ${profileName}` : evidence.label;
|
||||||
|
}
|
||||||
|
|
||||||
export function ObservatoryWorkspace({
|
export function ObservatoryWorkspace({
|
||||||
definition,
|
definition,
|
||||||
}: {
|
}: {
|
||||||
@@ -214,7 +219,7 @@ export function ObservatoryWorkspace({
|
|||||||
? "Готовый результат"
|
? "Готовый результат"
|
||||||
: setup.origin === "portable-definition"
|
: setup.origin === "portable-definition"
|
||||||
? setup.executor.state === "ready"
|
? setup.executor.state === "ready"
|
||||||
? "Запись совместима · Worker установлен, запуск закрыт"
|
? "Запись совместима · Worker готов к проверке"
|
||||||
: "Запись совместима · Worker не установлен"
|
: "Запись совместима · Worker не установлен"
|
||||||
: "Совместимый архивный сетап"
|
: "Совместимый архивный сетап"
|
||||||
: "Несовместим с выбранной сессией",
|
: "Несовместим с выбранной сессией",
|
||||||
@@ -512,8 +517,8 @@ export function ObservatoryWorkspace({
|
|||||||
>
|
>
|
||||||
{setupController.selectedSetup.compatibility.compatible
|
{setupController.selectedSetup.compatibility.compatible
|
||||||
? setupController.selectedSetup.executor.state === "not-installed"
|
? setupController.selectedSetup.executor.state === "not-installed"
|
||||||
? "Worker LAB V1 не установлен"
|
? "Worker-профиль не установлен"
|
||||||
: "Запуск LAB V1 недоступен"
|
: "Запуск профиля недоступен"
|
||||||
: "Запись несовместима"}
|
: "Запись несовместима"}
|
||||||
</StatusBadge>
|
</StatusBadge>
|
||||||
) : null}
|
) : null}
|
||||||
@@ -521,7 +526,18 @@ export function ObservatoryWorkspace({
|
|||||||
<Button
|
<Button
|
||||||
size="compact"
|
size="compact"
|
||||||
variant="primary"
|
variant="primary"
|
||||||
onClick={() => { void recordedJobsController.submit(); }}
|
onClick={() => {
|
||||||
|
void recordedJobsController.submit(
|
||||||
|
setupController.selectedSetup?.origin === "portable-definition"
|
||||||
|
&& runPreflight?.definitionSha256
|
||||||
|
&& runPreflight.checkSha256
|
||||||
|
? {
|
||||||
|
definitionSha256: runPreflight.definitionSha256,
|
||||||
|
checkSha256: runPreflight.checkSha256,
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
);
|
||||||
|
}}
|
||||||
>
|
>
|
||||||
Рассчитать
|
Рассчитать
|
||||||
</Button>
|
</Button>
|
||||||
@@ -631,7 +647,7 @@ export function ObservatoryWorkspace({
|
|||||||
</span>
|
</span>
|
||||||
<div className="observatory-evidence-card__copy">
|
<div className="observatory-evidence-card__copy">
|
||||||
<strong>{evidence.lab.labId}</strong>
|
<strong>{evidence.lab.labId}</strong>
|
||||||
<span>{evidence.label}</span>
|
<span>{evidenceResultSubtitle(evidence)}</span>
|
||||||
<small>
|
<small>
|
||||||
{evidence.lab.resultKind} · {formatTimestamp(evidence.publishedAtUtc)}
|
{evidence.lab.resultKind} · {formatTimestamp(evidence.publishedAtUtc)}
|
||||||
</small>
|
</small>
|
||||||
|
|||||||
@@ -101,6 +101,19 @@ function canonicalLab(overrides = {}) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function legacyCalculationProfile(overrides = {}) {
|
||||||
|
return {
|
||||||
|
schema_version: "missioncore.observatory-calculation-profile/v1",
|
||||||
|
setup_id: "lab-v1-ravnoves004tree-final",
|
||||||
|
display_name: "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||||
|
origin: "existing-result",
|
||||||
|
definition_id: null,
|
||||||
|
definition_version: null,
|
||||||
|
definition_sha256: null,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function canonicalProjectionProvenance(resultId, replayCapability) {
|
function canonicalProjectionProvenance(resultId, replayCapability) {
|
||||||
return {
|
return {
|
||||||
schema_version: "missioncore.canonical-recorded-lab-projection/v1",
|
schema_version: "missioncore.canonical-recorded-lab-projection/v1",
|
||||||
@@ -260,6 +273,69 @@ test("session catalog strictly decodes the explicit recorded LAB replay capabili
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("session catalog decodes a typed calculation profile without inferring unknown results", () => {
|
||||||
|
const resultId = `lab-v1-vegetation-shadow-${"8".repeat(64)}`;
|
||||||
|
const decoded = decodeObservationSessionCatalog({
|
||||||
|
items: [session({
|
||||||
|
id: resultId,
|
||||||
|
lab: canonicalLab({ calculation_profile: legacyCalculationProfile() }),
|
||||||
|
})],
|
||||||
|
}).items[0].lab;
|
||||||
|
|
||||||
|
assert.deepEqual(decoded.calculationProfile, {
|
||||||
|
schemaVersion: "missioncore.observatory-calculation-profile/v1",
|
||||||
|
setupId: "lab-v1-ravnoves004tree-final",
|
||||||
|
displayName: "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||||
|
origin: "existing-result",
|
||||||
|
definitionId: null,
|
||||||
|
definitionVersion: null,
|
||||||
|
definitionSha256: null,
|
||||||
|
});
|
||||||
|
assert.equal(
|
||||||
|
decodeObservationSessionCatalog({
|
||||||
|
items: [session({ id: resultId, lab: canonicalLab() })],
|
||||||
|
}).items[0].lab.calculationProfile,
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
decodeObservationSessionCatalog({
|
||||||
|
items: [session({
|
||||||
|
id: resultId,
|
||||||
|
lab: canonicalLab({ calculation_profile: null }),
|
||||||
|
})],
|
||||||
|
}).items[0].lab.calculationProfile,
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => decodeObservationSessionCatalog({
|
||||||
|
items: [session({
|
||||||
|
id: resultId,
|
||||||
|
lab: canonicalLab({
|
||||||
|
calculation_profile: legacyCalculationProfile({
|
||||||
|
origin: "archived-definition",
|
||||||
|
definition_id: "lab-v1-portable",
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
})],
|
||||||
|
}),
|
||||||
|
ObservationSessionContractError,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => decodeObservationSessionCatalog({
|
||||||
|
items: [session({
|
||||||
|
id: resultId,
|
||||||
|
lab: canonicalLab({
|
||||||
|
calculation_profile: {
|
||||||
|
...legacyCalculationProfile(),
|
||||||
|
guessed_from_provenance: true,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
})],
|
||||||
|
}),
|
||||||
|
ObservationSessionContractError,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
test("opened archive is named in the scene header and trash hover has no pill", async () => {
|
test("opened archive is named in the scene header and trash hover has no pill", async () => {
|
||||||
const appSource = await readFile(new URL("../src/App.tsx", import.meta.url), "utf8");
|
const appSource = await readFile(new URL("../src/App.tsx", import.meta.url), "utf8");
|
||||||
const styles = await readFile(
|
const styles = await readFile(
|
||||||
@@ -365,7 +441,7 @@ test("source and laboratory catalogs are requested as disjoint backend projectio
|
|||||||
|
|
||||||
assert.deepEqual(calls, [
|
assert.deepEqual(calls, [
|
||||||
"/api/v1/observation-sessions?limit=100&scope=source",
|
"/api/v1/observation-sessions?limit=100&scope=source",
|
||||||
"/api/v1/observation-sessions?limit=100&scope=laboratory&lab_contract=v2",
|
"/api/v1/observation-sessions?limit=100&scope=laboratory&lab_contract=v3",
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ function evidence(id, sourceSessionId, publishedAtUtc) {
|
|||||||
runCreatedAtUtc: publishedAtUtc,
|
runCreatedAtUtc: publishedAtUtc,
|
||||||
publishedAtUtc,
|
publishedAtUtc,
|
||||||
replayCapability: null,
|
replayCapability: null,
|
||||||
|
calculationProfile: null,
|
||||||
provenance: { verdict: "must-not-be-inferred" },
|
provenance: { verdict: "must-not-be-inferred" },
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
@@ -174,7 +175,7 @@ test("Observatory fetches disjoint read-only source and laboratory projections",
|
|||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
calls.map(({ input }) => input).sort(),
|
calls.map(({ input }) => input).sort(),
|
||||||
[
|
[
|
||||||
"/api/v1/observation-sessions?limit=50&scope=laboratory&lab_contract=v2",
|
"/api/v1/observation-sessions?limit=50&scope=laboratory&lab_contract=v3",
|
||||||
"/api/v1/observation-sessions?limit=50&scope=source",
|
"/api/v1/observation-sessions?limit=50&scope=source",
|
||||||
],
|
],
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -122,6 +122,7 @@ function portableSetup() {
|
|||||||
contour_id: "worker-006",
|
contour_id: "worker-006",
|
||||||
state: "not-installed",
|
state: "not-installed",
|
||||||
ready: false,
|
ready: false,
|
||||||
|
reason_code: "eomt-executor-release-unsealed",
|
||||||
reason: "Immutable executor release не установлен.",
|
reason: "Immutable executor release не установлен.",
|
||||||
},
|
},
|
||||||
existing_results: [],
|
existing_results: [],
|
||||||
@@ -136,6 +137,26 @@ function portableSetup() {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function portableM49Setup() {
|
||||||
|
const profile = portableSetup();
|
||||||
|
profile.setup_id = "m49-tgs-portable-v2";
|
||||||
|
profile.display_name = "M4.9T5 · TRAVEL TGS · CPU-only, без ML";
|
||||||
|
profile.description = "Динамический TGS-разбор записанной K1-сессии без ML; только наблюдение.";
|
||||||
|
profile.run_definition = {
|
||||||
|
definition_id: "m49-tgs-portable",
|
||||||
|
version: 2,
|
||||||
|
definition_sha256: "4".repeat(64),
|
||||||
|
result_schema: "missioncore.recorded-travel-tgs-review/v2",
|
||||||
|
result_kind: "recorded-source-paced-tgs-shadow",
|
||||||
|
models: [],
|
||||||
|
};
|
||||||
|
profile.source_compatibility.reason = "Запись соответствует требованиям TRAVEL TGS.";
|
||||||
|
profile.executor.reason_code = "m49-executor-release-unsealed";
|
||||||
|
profile.executor.reason = "Immutable executor release не установлен.";
|
||||||
|
profile.preflight.reason = "Переносимый вычислительный контур M4.9T5 пока недоступен.";
|
||||||
|
return profile;
|
||||||
|
}
|
||||||
|
|
||||||
before(async () => {
|
before(async () => {
|
||||||
server = await createServer({
|
server = await createServer({
|
||||||
appType: "custom",
|
appType: "custom",
|
||||||
@@ -178,7 +199,7 @@ test("Observatory setup catalog keeps definition identity separate from executor
|
|||||||
assert.equal(calls[0].init.method, "GET");
|
assert.equal(calls[0].init.method, "GET");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("portable LAB V1 reports compatible source separately from unavailable Worker", async () => {
|
test("portable profiles report compatible source separately from unavailable Worker", async () => {
|
||||||
const calls = [];
|
const calls = [];
|
||||||
const selected = (await fetchObservatoryPortableLaboratorySetups("source-a", {
|
const selected = (await fetchObservatoryPortableLaboratorySetups("source-a", {
|
||||||
fetcher: async (input, init) => {
|
fetcher: async (input, init) => {
|
||||||
@@ -186,7 +207,7 @@ test("portable LAB V1 reports compatible source separately from unavailable Work
|
|||||||
return new Response(JSON.stringify({
|
return new Response(JSON.stringify({
|
||||||
schema_version: "missioncore.observatory-portable-setup-catalog/v2",
|
schema_version: "missioncore.observatory-portable-setup-catalog/v2",
|
||||||
source_session_id: "source-a",
|
source_session_id: "source-a",
|
||||||
setups: [portableSetup()],
|
setups: [portableSetup(), portableM49Setup()],
|
||||||
authority,
|
authority,
|
||||||
}), { status: 200 });
|
}), { status: 200 });
|
||||||
},
|
},
|
||||||
@@ -200,52 +221,81 @@ test("portable LAB V1 reports compatible source separately from unavailable Work
|
|||||||
"EoMT Cityscapes Large 1024",
|
"EoMT Cityscapes Large 1024",
|
||||||
"DDRNet-39",
|
"DDRNet-39",
|
||||||
]);
|
]);
|
||||||
|
const m49 = (await fetchObservatoryPortableLaboratorySetups("source-a", {
|
||||||
|
fetcher: async () => new Response(JSON.stringify({
|
||||||
|
schema_version: "missioncore.observatory-portable-setup-catalog/v2",
|
||||||
|
source_session_id: "source-a",
|
||||||
|
setups: [portableSetup(), portableM49Setup()],
|
||||||
|
authority,
|
||||||
|
}), { status: 200 }),
|
||||||
|
})).setups[1];
|
||||||
|
assert.equal(m49.displayName, "M4.9T5 · TRAVEL TGS · CPU-only, без ML");
|
||||||
|
assert.deepEqual(m49.runDefinition.models, []);
|
||||||
assert.equal(
|
assert.equal(
|
||||||
calls[0].input,
|
calls[0].input,
|
||||||
"/api/v1/observatory/portable-laboratory-setups?source_session_id=source-a",
|
"/api/v1/observatory/portable-laboratory-setups?source_session_id=source-a",
|
||||||
);
|
);
|
||||||
|
|
||||||
let unexpectedNetworkCall = false;
|
let preflightRequest;
|
||||||
const preflight = await preflightObservatoryLaboratorySetup("source-a", selected, {
|
const preflight = await preflightObservatoryLaboratorySetup("source-a", selected, {
|
||||||
fetcher: async () => {
|
fetcher: async (input, init) => {
|
||||||
unexpectedNetworkCall = true;
|
preflightRequest = { input: String(input), init };
|
||||||
throw new Error("portable preflight must use its server projection");
|
return new Response(JSON.stringify({
|
||||||
|
schema_version: "missioncore.observatory-run-preflight/v1",
|
||||||
|
source_session_id: "source-a",
|
||||||
|
setup_id: selected.setupId,
|
||||||
|
definition_sha256: "3".repeat(64),
|
||||||
|
check_sha256: null,
|
||||||
|
outcome: "blocked",
|
||||||
|
submission_allowed: false,
|
||||||
|
checks: [{
|
||||||
|
check_id: "executor",
|
||||||
|
outcome: "fail",
|
||||||
|
reason_code: "eomt-executor-release-unsealed",
|
||||||
|
message: "Immutable executor release не установлен.",
|
||||||
|
}],
|
||||||
|
existing_result_ids: [],
|
||||||
|
executor: portableSetup().executor,
|
||||||
|
authority,
|
||||||
|
}), { status: 200 });
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
assert.equal(unexpectedNetworkCall, false);
|
assert.equal(preflightRequest.input, "/api/v1/observatory/run-preflights");
|
||||||
|
assert.equal(preflightRequest.init.method, "POST");
|
||||||
assert.equal(preflight.outcome, "blocked");
|
assert.equal(preflight.outcome, "blocked");
|
||||||
assert.equal(preflight.submissionAllowed, false);
|
assert.equal(preflight.submissionAllowed, false);
|
||||||
assert.equal(preflight.checks[0].outcome, "pass");
|
assert.equal(preflight.checkSha256, null);
|
||||||
assert.equal(preflight.checks[1].outcome, "fail");
|
assert.equal(preflight.checks[0].outcome, "fail");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("portable LAB V1 rejects a premature enqueue projection", async () => {
|
test("portable profile accepts a consistent ready projection", async () => {
|
||||||
const premature = portableSetup();
|
const ready = portableSetup();
|
||||||
premature.executor = {
|
ready.executor = {
|
||||||
contour_id: "worker-006",
|
contour_id: "worker-006",
|
||||||
state: "ready",
|
state: "ready",
|
||||||
ready: true,
|
ready: true,
|
||||||
|
reason_code: null,
|
||||||
reason: null,
|
reason: null,
|
||||||
};
|
};
|
||||||
premature.preflight = {
|
ready.preflight = {
|
||||||
outcome: "ready",
|
outcome: "ready",
|
||||||
action: "enqueue",
|
action: "check",
|
||||||
reason: "Запись готова к постановке в очередь.",
|
reason: "Запись готова к постановке в очередь.",
|
||||||
submission_allowed: true,
|
submission_allowed: true,
|
||||||
existing_result_ids: [],
|
existing_result_ids: [],
|
||||||
};
|
};
|
||||||
|
|
||||||
await assert.rejects(
|
const catalog = await fetchObservatoryPortableLaboratorySetups("source-a", {
|
||||||
fetchObservatoryPortableLaboratorySetups("source-a", {
|
fetcher: async () => new Response(JSON.stringify({
|
||||||
fetcher: async () => new Response(JSON.stringify({
|
schema_version: "missioncore.observatory-portable-setup-catalog/v2",
|
||||||
schema_version: "missioncore.observatory-portable-setup-catalog/v2",
|
source_session_id: "source-a",
|
||||||
source_session_id: "source-a",
|
setups: [ready],
|
||||||
setups: [premature],
|
authority,
|
||||||
authority,
|
}), { status: 200 }),
|
||||||
}), { status: 200 }),
|
});
|
||||||
}),
|
assert.equal(catalog.setups[0].executor.state, "ready");
|
||||||
/значение изменилось|значение не поддерживается|постановка в очередь ещё не поддерживается/,
|
assert.equal(catalog.setups[0].preflight.outcome, "ready");
|
||||||
);
|
assert.equal(catalog.setups[0].preflight.submissionAllowed, true);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("portable LAB V1 rejects an unbound existing result projection", async () => {
|
test("portable LAB V1 rejects an unbound existing result projection", async () => {
|
||||||
@@ -272,7 +322,7 @@ test("portable LAB V1 rejects an unbound existing result projection", async () =
|
|||||||
authority,
|
authority,
|
||||||
}), { status: 200 }),
|
}), { status: 200 }),
|
||||||
}),
|
}),
|
||||||
/значение изменилось|проверяемая привязка результата/,
|
/значение изменилось|значение не поддерживается|проверяемая привязка результата/,
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -315,6 +365,7 @@ test("Observatory preflight sends the exact selected definition and never submit
|
|||||||
source_session_id: "source-a",
|
source_session_id: "source-a",
|
||||||
setup_id: selected.setupId,
|
setup_id: selected.setupId,
|
||||||
definition_sha256: "b".repeat(64),
|
definition_sha256: "b".repeat(64),
|
||||||
|
check_sha256: null,
|
||||||
outcome: "blocked",
|
outcome: "blocked",
|
||||||
submission_allowed: false,
|
submission_allowed: false,
|
||||||
checks: [{
|
checks: [{
|
||||||
@@ -340,6 +391,7 @@ test("Observatory preflight sends the exact selected definition and never submit
|
|||||||
});
|
});
|
||||||
assert.equal(preflight.outcome, "blocked");
|
assert.equal(preflight.outcome, "blocked");
|
||||||
assert.equal(preflight.submissionAllowed, false);
|
assert.equal(preflight.submissionAllowed, false);
|
||||||
|
assert.equal(preflight.checkSha256, null);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("Observatory dynamic preflight admits only an explicit queueable response", async () => {
|
test("Observatory dynamic preflight admits only an explicit queueable response", async () => {
|
||||||
@@ -374,6 +426,7 @@ test("Observatory dynamic preflight admits only an explicit queueable response",
|
|||||||
|
|
||||||
assert.equal(preflight.outcome, "queueable");
|
assert.equal(preflight.outcome, "queueable");
|
||||||
assert.equal(preflight.submissionAllowed, true);
|
assert.equal(preflight.submissionAllowed, true);
|
||||||
|
assert.equal(preflight.checkSha256, null);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("Observatory setup contract rejects authority escalation and response drift", async () => {
|
test("Observatory setup contract rejects authority escalation and response drift", async () => {
|
||||||
|
|||||||
@@ -61,6 +61,7 @@ function job(state = "queued") {
|
|||||||
restart_from_zero: state === "paused",
|
restart_from_zero: state === "paused",
|
||||||
preemption_receipt_sha256: null,
|
preemption_receipt_sha256: null,
|
||||||
claim_generation: 0,
|
claim_generation: 0,
|
||||||
|
claim_lease: null,
|
||||||
result: state === "succeeded"
|
result: state === "succeeded"
|
||||||
? { result_id: "m49-result", sha256: "d".repeat(64) }
|
? { result_id: "m49-result", sha256: "d".repeat(64) }
|
||||||
: null,
|
: null,
|
||||||
@@ -132,6 +133,7 @@ test("Observatory submits only public identities and accepts every durable state
|
|||||||
"source-a",
|
"source-a",
|
||||||
"m49-tgs",
|
"m49-tgs",
|
||||||
"observatory-ui:source-a:m49-tgs:request-a",
|
"observatory-ui:source-a:m49-tgs:request-a",
|
||||||
|
null,
|
||||||
{
|
{
|
||||||
fetcher: async (input, init) => {
|
fetcher: async (input, init) => {
|
||||||
request = { input: String(input), init };
|
request = { input: String(input), init };
|
||||||
@@ -172,8 +174,41 @@ test("Observatory queue contract rejects authority escalation and response drift
|
|||||||
"source-a",
|
"source-a",
|
||||||
"m49-tgs",
|
"m49-tgs",
|
||||||
"observatory-ui:source-a:m49-tgs:request-a",
|
"observatory-ui:source-a:m49-tgs:request-a",
|
||||||
|
null,
|
||||||
{ fetcher: async () => new Response(JSON.stringify(drifted), { status: 200 }) },
|
{ fetcher: async () => new Response(JSON.stringify(drifted), { status: 200 }) },
|
||||||
),
|
),
|
||||||
ObservatoryRecordedJobContractError,
|
ObservatoryRecordedJobContractError,
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("portable submission carries the exact definition/check fence", async () => {
|
||||||
|
let request;
|
||||||
|
await submitObservatoryRecordedJob(
|
||||||
|
"source-a",
|
||||||
|
"lab-v1-eomt-ddrnet-portable-v1",
|
||||||
|
"observatory-ui:source-a:lab-v1:request-a",
|
||||||
|
{
|
||||||
|
definitionSha256: "e".repeat(64),
|
||||||
|
checkSha256: "f".repeat(64),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
fetcher: async (input, init) => {
|
||||||
|
request = { input: String(input), init };
|
||||||
|
const response = job("queued");
|
||||||
|
response.setup.setup_id = "lab-v1-eomt-ddrnet-portable-v1";
|
||||||
|
response.setup.definition_sha256 = "e".repeat(64);
|
||||||
|
response.source.session_id = "source-a";
|
||||||
|
return new Response(JSON.stringify(response), { status: 202 });
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.deepEqual(JSON.parse(request.init.body), {
|
||||||
|
schema_version: "missioncore.observatory-recorded-run-submit/v1",
|
||||||
|
idempotency_key: "observatory-ui:source-a:lab-v1:request-a",
|
||||||
|
source_session_id: "source-a",
|
||||||
|
setup_id: "lab-v1-eomt-ddrnet-portable-v1",
|
||||||
|
definition_sha256: "e".repeat(64),
|
||||||
|
check_sha256: "f".repeat(64),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -84,6 +84,14 @@ test("Observatory mounts the one shared canonical replay only after explicit adm
|
|||||||
assert.match(workspace, /observatory-session-stack/);
|
assert.match(workspace, /observatory-session-stack/);
|
||||||
assert.match(workspace, /observatory-session-summary__facts/);
|
assert.match(workspace, /observatory-session-summary__facts/);
|
||||||
assert.match(workspace, /observatory-evidence-card__copy/);
|
assert.match(workspace, /observatory-evidence-card__copy/);
|
||||||
|
assert.match(
|
||||||
|
workspace,
|
||||||
|
/function evidenceResultSubtitle[\s\S]*calculationProfile\?\.displayName[\s\S]*`\$\{evidence\.label\} · \$\{profileName\}`[\s\S]*: evidence\.label/,
|
||||||
|
);
|
||||||
|
assert.match(
|
||||||
|
workspace,
|
||||||
|
/<span>\{evidenceResultSubtitle\(evidence\)\}<\/span>/,
|
||||||
|
);
|
||||||
assert.match(
|
assert.match(
|
||||||
workspace,
|
workspace,
|
||||||
/evidence\.recordedRun \? \([\s\S]*name="trash"[\s\S]*name="edit"[\s\S]*Открыть визуальный разбор:[\s\S]*name="eye"/,
|
/evidence\.recordedRun \? \([\s\S]*name="trash"[\s\S]*name="edit"[\s\S]*Открыть визуальный разбор:[\s\S]*name="eye"/,
|
||||||
@@ -242,8 +250,12 @@ test("Observatory keeps one compact selector axis without the obsolete setup det
|
|||||||
assert.doesNotMatch(setupHook, /Promise\.allSettled/);
|
assert.doesNotMatch(setupHook, /Promise\.allSettled/);
|
||||||
assert.match(setupHook, /publishSetupCatalog\(\s*legacyCatalog/);
|
assert.match(setupHook, /publishSetupCatalog\(\s*legacyCatalog/);
|
||||||
assert.match(setupHook, /preserveUnknownSelection: true/);
|
assert.match(setupHook, /preserveUnknownSelection: true/);
|
||||||
|
assert.match(
|
||||||
|
setupHook,
|
||||||
|
/portableProfileNames[\s\S]*legacy\.setups\.filter\([\s\S]*!portableProfileNames\.has\(setup\.displayName\)[\s\S]*\.\.\.portable\.setups/,
|
||||||
|
);
|
||||||
assert.match(setupHook, /selectedSetupId, sourceSessionId/);
|
assert.match(setupHook, /selectedSetupId, sourceSessionId/);
|
||||||
assert.match(workspace, /Worker установлен, запуск закрыт/);
|
assert.match(workspace, /Worker готов к проверке/);
|
||||||
assert.match(
|
assert.match(
|
||||||
workspace,
|
workspace,
|
||||||
/preflightCandidate\.definitionSha256[\s\S]*selectedSetup\?\.runDefinition\?\.definitionSha256/,
|
/preflightCandidate\.definitionSha256[\s\S]*selectedSetup\?\.runDefinition\?\.definitionSha256/,
|
||||||
|
|||||||
@@ -4,8 +4,8 @@
|
|||||||
{
|
{
|
||||||
"setup_id": "lab-v1-eomt-ddrnet-portable-v1",
|
"setup_id": "lab-v1-eomt-ddrnet-portable-v1",
|
||||||
"definition_id": "lab-v1-eomt-ddrnet-portable",
|
"definition_id": "lab-v1-eomt-ddrnet-portable",
|
||||||
"version": 1,
|
"version": 2,
|
||||||
"definition_sha256": "57bf8f0859e10e54e30322c9a8aa28b427699f6fe6b5267e279ec3390fa78466",
|
"definition_sha256": "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2",
|
||||||
"source_requirements": {
|
"source_requirements": {
|
||||||
"plugin_id": "nodedc.device.xgrids-lixelkity-k1",
|
"plugin_id": "nodedc.device.xgrids-lixelkity-k1",
|
||||||
"archive_id": "xgrids-k1.viewer-live.evidence",
|
"archive_id": "xgrids-k1.viewer-live.evidence",
|
||||||
@@ -32,9 +32,9 @@
|
|||||||
},
|
},
|
||||||
"components": [
|
"components": [
|
||||||
{
|
{
|
||||||
"component_id": "ddrnet-full-route-runtime-config-v1",
|
"component_id": "ddrnet-portable-runtime-config-v2",
|
||||||
"kind": "configuration",
|
"kind": "configuration",
|
||||||
"sha256": "ec7464c2818a707c79aadaa6494625b76d45fb4fe59b74b5208ff9bbd7c9006a"
|
"sha256": "c0ff657dadc86607d77d232e84d041fbf2d8b63e86d02319e9cd607220d00f21"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"component_id": "eomt-recorded-dependency-set-v1",
|
"component_id": "eomt-recorded-dependency-set-v1",
|
||||||
@@ -145,8 +145,85 @@
|
|||||||
"release_id": null,
|
"release_id": null,
|
||||||
"release_sha256": null,
|
"release_sha256": null,
|
||||||
"image_sha256": null,
|
"image_sha256": null,
|
||||||
"reason_code": "eomt-executor-release-unsealed",
|
"reason_code": "lab-v1-portable-v2-uninstalled",
|
||||||
"reason": "Immutable EoMT plus DDRNet executor release and image are not sealed or installed on Worker 006."
|
"reason": "Portable LAB V1 v2 is not sealed or installed; a commit-bound combined image and Worker 006 receipt are required."
|
||||||
|
},
|
||||||
|
"authority": {
|
||||||
|
"commands_enabled": false,
|
||||||
|
"actuation_allowed": false,
|
||||||
|
"navigation_or_safety_accepted": false,
|
||||||
|
"production_accepted": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"setup_id": "m49-tgs-portable-v2",
|
||||||
|
"definition_id": "m49-tgs-portable",
|
||||||
|
"version": 2,
|
||||||
|
"definition_sha256": "73611f24d70319ea1edca428726d6538a3cbad012a415cc0c1a7ecb7d9b4d910",
|
||||||
|
"source_requirements": {
|
||||||
|
"plugin_id": "nodedc.device.xgrids-lixelkity-k1",
|
||||||
|
"archive_id": "xgrids-k1.viewer-live.evidence",
|
||||||
|
"required_modalities": [
|
||||||
|
"point-cloud",
|
||||||
|
"trajectory",
|
||||||
|
"video"
|
||||||
|
],
|
||||||
|
"camera_source_id": "sensor.camera.right",
|
||||||
|
"camera_semantic_channel_id": "camera.video.recorded",
|
||||||
|
"recorded_media_type": "video/mp4; codecs=\"avc1.641028\"",
|
||||||
|
"recorded_media_init_sha256": "e2279963e16d84c91d68e7dbb1f7efed840533387dfeb844b7398bff45fbde38",
|
||||||
|
"camera_width": 800,
|
||||||
|
"camera_height": 600,
|
||||||
|
"calibration_slot": "camera_1",
|
||||||
|
"calibration_identity_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9",
|
||||||
|
"exactly_one_media_epoch": true,
|
||||||
|
"seekable": true
|
||||||
|
},
|
||||||
|
"source_adapter": {
|
||||||
|
"adapter_id": "xgrids-k1-recorded-observatory-v2",
|
||||||
|
"version": 2,
|
||||||
|
"contract_sha256": "4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de"
|
||||||
|
},
|
||||||
|
"components": [
|
||||||
|
{
|
||||||
|
"component_id": "k1-camera-1-calibration-v1",
|
||||||
|
"kind": "calibration",
|
||||||
|
"sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"component_id": "m49-tgs-portable-profile-v2",
|
||||||
|
"kind": "configuration",
|
||||||
|
"sha256": "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"models": [],
|
||||||
|
"resource_profile": {
|
||||||
|
"schema_version": "missioncore.observatory-portable-resource-profile/v2",
|
||||||
|
"profile_id": "worker006-cpu-single-run-portable-v2",
|
||||||
|
"contour_id": "worker-006",
|
||||||
|
"accelerator_id": "cpu-only",
|
||||||
|
"concurrency": 1,
|
||||||
|
"checkpoint_policy": "non-checkpointable",
|
||||||
|
"allowed_checkpoints": [],
|
||||||
|
"profile_sha256": "49e373f1cbf314e7fab2d176db295f3d3d9ddbbcbd724bbde988ef60ad767dee"
|
||||||
|
},
|
||||||
|
"result_contract": {
|
||||||
|
"schema_version": "missioncore.observatory-portable-result-contract/v2",
|
||||||
|
"contract_id": "m49-tgs-portable-review-v2",
|
||||||
|
"version": 2,
|
||||||
|
"result_schema": "missioncore.recorded-tgs-costmap-review/v2",
|
||||||
|
"result_kind": "recorded-perception-qualification",
|
||||||
|
"publication": "observatory",
|
||||||
|
"contract_sha256": "9dd80c8e2504559d2156fca933de6eb27901e35305e6853aeb84707e1cb13892"
|
||||||
|
},
|
||||||
|
"executor": {
|
||||||
|
"contour_id": "worker-006",
|
||||||
|
"state": "not-installed",
|
||||||
|
"release_id": null,
|
||||||
|
"release_sha256": null,
|
||||||
|
"image_sha256": null,
|
||||||
|
"reason_code": "m49-portable-executor-not-installed",
|
||||||
|
"reason": "A source-independent M4.9 TGS executor release and image are not sealed or installed on Worker 006."
|
||||||
},
|
},
|
||||||
"authority": {
|
"authority": {
|
||||||
"commands_enabled": false,
|
"commands_enabled": false,
|
||||||
|
|||||||
@@ -0,0 +1,292 @@
|
|||||||
|
{
|
||||||
|
"schema_version": "missioncore.observatory-portable-worker-runtime-registry/v1",
|
||||||
|
"candidates": [
|
||||||
|
{
|
||||||
|
"schema_version": "missioncore.observatory-portable-worker-runtime-candidate/v1",
|
||||||
|
"adapter_id": "lab-v1-eomt-ddrnet-worker006-v2",
|
||||||
|
"setup_id": "lab-v1-eomt-ddrnet-portable-v1",
|
||||||
|
"definition_id": "lab-v1-eomt-ddrnet-portable",
|
||||||
|
"definition_version": 2,
|
||||||
|
"definition_sha256": "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2",
|
||||||
|
"source_adapter_sha256": "4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de",
|
||||||
|
"model_manifest_sha256": "3fd2d43af73bd73f89d9ffae95d8770cfdeb46033ec967509124fac6ae4afe56",
|
||||||
|
"resource_profile_sha256": "7468138cad115210eda18e7a3350e3423d3d39ac0aae650b3266cdc2bc63fd5d",
|
||||||
|
"result_contract_sha256": "b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a",
|
||||||
|
"state": "blocked",
|
||||||
|
"executor": null,
|
||||||
|
"reusable_assets": [
|
||||||
|
{
|
||||||
|
"asset_id": "ddrnet-checkpoint",
|
||||||
|
"kind": "model-artifact",
|
||||||
|
"sha256": "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6",
|
||||||
|
"byte_length": 259419077,
|
||||||
|
"component_id": null,
|
||||||
|
"model_release_id": "lab-v1-ddrnet-39-goose-fine-64-v1",
|
||||||
|
"model_artifact_role": "checkpoint"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "ddrnet-goose-image",
|
||||||
|
"kind": "container-image",
|
||||||
|
"sha256": "591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd",
|
||||||
|
"byte_length": null,
|
||||||
|
"component_id": null,
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "ddrnet-goose-runner",
|
||||||
|
"kind": "local-file",
|
||||||
|
"sha256": "b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1",
|
||||||
|
"byte_length": 32877,
|
||||||
|
"component_id": null,
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "ddrnet-portable-config",
|
||||||
|
"kind": "definition-component",
|
||||||
|
"sha256": "c0ff657dadc86607d77d232e84d041fbf2d8b63e86d02319e9cd607220d00f21",
|
||||||
|
"byte_length": 4324,
|
||||||
|
"component_id": "ddrnet-portable-runtime-config-v2",
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-config-json",
|
||||||
|
"kind": "model-artifact",
|
||||||
|
"sha256": "7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650",
|
||||||
|
"byte_length": 1575,
|
||||||
|
"component_id": null,
|
||||||
|
"model_release_id": "eomt-cityscapes-large-1024-v1",
|
||||||
|
"model_artifact_role": "config-json"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-image",
|
||||||
|
"kind": "container-image",
|
||||||
|
"sha256": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
|
||||||
|
"byte_length": null,
|
||||||
|
"component_id": null,
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-model-weights",
|
||||||
|
"kind": "model-artifact",
|
||||||
|
"sha256": "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782",
|
||||||
|
"byte_length": 1276175488,
|
||||||
|
"component_id": null,
|
||||||
|
"model_release_id": "eomt-cityscapes-large-1024-v1",
|
||||||
|
"model_artifact_role": "model-weights"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-orchestrator",
|
||||||
|
"kind": "definition-component",
|
||||||
|
"sha256": "d3e9435939444ab35b27a744ac314e289ebd66a13fa56e3d59f121e088d22774",
|
||||||
|
"byte_length": 21489,
|
||||||
|
"component_id": "eomt-recorded-orchestrator-v1",
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-preprocessor-config",
|
||||||
|
"kind": "model-artifact",
|
||||||
|
"sha256": "97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7",
|
||||||
|
"byte_length": 666,
|
||||||
|
"component_id": null,
|
||||||
|
"model_release_id": "eomt-cityscapes-large-1024-v1",
|
||||||
|
"model_artifact_role": "preprocessor-config"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-profile",
|
||||||
|
"kind": "definition-component",
|
||||||
|
"sha256": "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875",
|
||||||
|
"byte_length": 3805,
|
||||||
|
"component_id": "eomt-recorded-profile-v1",
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-runner",
|
||||||
|
"kind": "definition-component",
|
||||||
|
"sha256": "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4",
|
||||||
|
"byte_length": 30720,
|
||||||
|
"component_id": "eomt-recorded-runner-v1",
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "vegetation-policy",
|
||||||
|
"kind": "definition-component",
|
||||||
|
"sha256": "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35",
|
||||||
|
"byte_length": 3022,
|
||||||
|
"component_id": "vegetation-mission-policy-v1",
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "vegetation-provider-map",
|
||||||
|
"kind": "definition-component",
|
||||||
|
"sha256": "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352",
|
||||||
|
"byte_length": 2756,
|
||||||
|
"component_id": "vegetation-provider-label-map-v1",
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"phases": [
|
||||||
|
{
|
||||||
|
"phase_id": "source-delivery",
|
||||||
|
"state": "implemented"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase_id": "eomt-runtime",
|
||||||
|
"state": "implemented"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase_id": "ddrnet-portable-runtime",
|
||||||
|
"state": "missing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase_id": "portable-lab-orchestrator",
|
||||||
|
"state": "implemented"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase_id": "result-v2-assembler",
|
||||||
|
"state": "implemented"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase_id": "observatory-result-publisher",
|
||||||
|
"state": "implemented"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"blockers": [
|
||||||
|
"combined-executor-image-unsealed",
|
||||||
|
"commit-bound-source-unavailable",
|
||||||
|
"ddrnet-component-port-uninstalled",
|
||||||
|
"eomt-component-port-uninstalled",
|
||||||
|
"executor-release-unsealed",
|
||||||
|
"fixture-smoke-unaccepted",
|
||||||
|
"worker-installation-receipt-unavailable"
|
||||||
|
],
|
||||||
|
"authority": {
|
||||||
|
"commands_enabled": false,
|
||||||
|
"actuation_allowed": false,
|
||||||
|
"navigation_or_safety_accepted": false,
|
||||||
|
"production_accepted": false
|
||||||
|
},
|
||||||
|
"candidate_sha256": "b22b8fa16cca6dd68cf1ee68abeea84b33f4c420bb3844931b4dc7fd19434a81"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"schema_version": "missioncore.observatory-portable-worker-runtime-candidate/v1",
|
||||||
|
"adapter_id": "m49-tgs-worker006-portable-v2",
|
||||||
|
"setup_id": "m49-tgs-portable-v2",
|
||||||
|
"definition_id": "m49-tgs-portable",
|
||||||
|
"definition_version": 2,
|
||||||
|
"definition_sha256": "73611f24d70319ea1edca428726d6538a3cbad012a415cc0c1a7ecb7d9b4d910",
|
||||||
|
"source_adapter_sha256": "4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de",
|
||||||
|
"model_manifest_sha256": "489a43448f720a9b5c7993dc8279d167b77191a586f0d87b6d38b81cf728e2f1",
|
||||||
|
"resource_profile_sha256": "49e373f1cbf314e7fab2d176db295f3d3d9ddbbcbd724bbde988ef60ad767dee",
|
||||||
|
"result_contract_sha256": "9dd80c8e2504559d2156fca933de6eb27901e35305e6853aeb84707e1cb13892",
|
||||||
|
"state": "blocked",
|
||||||
|
"executor": null,
|
||||||
|
"reusable_assets": [
|
||||||
|
{
|
||||||
|
"asset_id": "m49-portable-profile",
|
||||||
|
"kind": "definition-component",
|
||||||
|
"sha256": "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9",
|
||||||
|
"byte_length": 1683,
|
||||||
|
"component_id": "m49-tgs-portable-profile-v2",
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "m49-portable-runner-manifest",
|
||||||
|
"kind": "local-file",
|
||||||
|
"sha256": "264c79258195c91448cadad2fff68fdee067c539cccdf1227657a4417ed4cc85",
|
||||||
|
"byte_length": 1825,
|
||||||
|
"component_id": null,
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "m49-portable-runner-source",
|
||||||
|
"kind": "local-file",
|
||||||
|
"sha256": "52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9",
|
||||||
|
"byte_length": 11052,
|
||||||
|
"component_id": null,
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "m49-portable-runner-wrapper",
|
||||||
|
"kind": "local-file",
|
||||||
|
"sha256": "2d6c32560682647f868e4ce4c2605749f17c60482a609f8c03ff951411f48ffb",
|
||||||
|
"byte_length": 858,
|
||||||
|
"component_id": null,
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "m49-portable-smoke",
|
||||||
|
"kind": "local-file",
|
||||||
|
"sha256": "e5da73c2cf89ed0671de8ac52e13014a7faf79ce1ff1f97a090154063a61c18d",
|
||||||
|
"byte_length": 1117,
|
||||||
|
"component_id": null,
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "travel-tgs-image",
|
||||||
|
"kind": "container-image",
|
||||||
|
"sha256": "7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f",
|
||||||
|
"byte_length": null,
|
||||||
|
"component_id": null,
|
||||||
|
"model_release_id": null,
|
||||||
|
"model_artifact_role": null
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"phases": [
|
||||||
|
{
|
||||||
|
"phase_id": "source-delivery",
|
||||||
|
"state": "missing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase_id": "camera-lidar-timeline-materializer",
|
||||||
|
"state": "missing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase_id": "portable-tgs-input-materializer",
|
||||||
|
"state": "missing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase_id": "portable-tgs-runner",
|
||||||
|
"state": "implemented"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase_id": "result-v2-assembler",
|
||||||
|
"state": "missing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase_id": "observatory-result-publisher",
|
||||||
|
"state": "missing"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"blockers": [
|
||||||
|
"executor-release-unsealed",
|
||||||
|
"portable-camera-lidar-timeline-unimplemented",
|
||||||
|
"portable-result-assembler-unimplemented",
|
||||||
|
"portable-source-delivery-unimplemented",
|
||||||
|
"portable-tgs-input-unimplemented",
|
||||||
|
"portable-tgs-runner-unsealed",
|
||||||
|
"result-publisher-integration-unaccepted"
|
||||||
|
],
|
||||||
|
"authority": {
|
||||||
|
"commands_enabled": false,
|
||||||
|
"actuation_allowed": false,
|
||||||
|
"navigation_or_safety_accepted": false,
|
||||||
|
"production_accepted": false
|
||||||
|
},
|
||||||
|
"candidate_sha256": "e2085c25e45a46de58f693d1128914dc25febbe18a28341cdf57c79f3472f180"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
{
|
||||||
|
"schema_version": "missioncore.lab-v1-eomt-ddrnet-portable-profile/v2",
|
||||||
|
"profile_id": "lab-v1-eomt-ddrnet-portable-v2",
|
||||||
|
"lab_id": "LAB-V1",
|
||||||
|
"worker_id": "worker-006",
|
||||||
|
"source_binding": {
|
||||||
|
"mode": "admitted-k1-recording",
|
||||||
|
"camera_source_id": "sensor.camera.right",
|
||||||
|
"camera_timeline": "dynamic",
|
||||||
|
"filesystem_paths": "executor-resolved",
|
||||||
|
"expected_width": 800,
|
||||||
|
"expected_height": 600,
|
||||||
|
"expected_frame_count": "source-derived",
|
||||||
|
"frame_indices": "source-derived-representative",
|
||||||
|
"crop_contract": "center-600-square-to-512; outside-crop-is-undefined"
|
||||||
|
},
|
||||||
|
"effective_config_contract": {
|
||||||
|
"schema_version": "missioncore.lab-v1-goose-vegetation-benchmark/v1",
|
||||||
|
"dynamic_field": "ravnoves",
|
||||||
|
"source_id": "sealed-source-derived",
|
||||||
|
"source_sha256": "sealed-camera-input-derived",
|
||||||
|
"base_m4_result_id": null
|
||||||
|
},
|
||||||
|
"runtime": {
|
||||||
|
"super_gradients_version": "3.2.0",
|
||||||
|
"super_gradients_revision": "54d062ecb1081944a672ce447cf3e96a36708ff9",
|
||||||
|
"python_version": "3.9",
|
||||||
|
"numpy_version": "1.23.0",
|
||||||
|
"cmake_version": "3.31.6",
|
||||||
|
"onnxsim_version": "0.4.36",
|
||||||
|
"opencv_python_version": "4.8.1.78",
|
||||||
|
"pytorch_version": "1.13.1",
|
||||||
|
"torchvision_version": "0.14.1",
|
||||||
|
"pytorch_cuda_version": "11.7",
|
||||||
|
"container_base": "nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04@sha256:ad6d59a3bbf3e82c1c849c9ac09cfc2a3e0bbb8655042fd899be6681b3fe2a85",
|
||||||
|
"miniconda_installer": "Miniconda3-py39_24.11.1-0-Linux-x86_64.sh",
|
||||||
|
"miniconda_installer_sha256": "3ea8373098d72140e08aac9217822b047ec094eb457e7f73945af7c6f68bf6f5"
|
||||||
|
},
|
||||||
|
"dataset": {
|
||||||
|
"dataset_id": "goose-2d-validation-visible-rgb",
|
||||||
|
"relative_root": "goose-2d/validation",
|
||||||
|
"mapping_relative_path": "goose_label_mapping.csv",
|
||||||
|
"mapping_sha256": "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f",
|
||||||
|
"image_glob": "images/val/**/*_windshield_vis.png",
|
||||||
|
"expected_pair_count": 962,
|
||||||
|
"input_size": [
|
||||||
|
512,
|
||||||
|
512
|
||||||
|
],
|
||||||
|
"preprocessing": [
|
||||||
|
"center-square-crop",
|
||||||
|
"nearest-neighbor-resize",
|
||||||
|
"rgb-to-tensor-0-1"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"candidates": {
|
||||||
|
"ddrnet": {
|
||||||
|
"candidate_id": "goose-ddrnet-class-512",
|
||||||
|
"model_names": [
|
||||||
|
"ddrnet_39"
|
||||||
|
],
|
||||||
|
"checkpoint_relative_path": "models/goose/ddrnet_class_512.pth",
|
||||||
|
"checkpoint_size_bytes": 259419077,
|
||||||
|
"checkpoint_sha256": "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6",
|
||||||
|
"published_validation_miou_percent": 46.53
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"vegetation_class_names": [
|
||||||
|
"leaves",
|
||||||
|
"forest",
|
||||||
|
"bush",
|
||||||
|
"moss",
|
||||||
|
"tree_crown",
|
||||||
|
"tree_trunk",
|
||||||
|
"crops",
|
||||||
|
"low_grass",
|
||||||
|
"high_grass",
|
||||||
|
"scenery_vegetation",
|
||||||
|
"hedge",
|
||||||
|
"tree_root"
|
||||||
|
],
|
||||||
|
"visual_case_contract": {
|
||||||
|
"selection_basis": "ground-truth-class-support-only",
|
||||||
|
"case_count": 12,
|
||||||
|
"minimum_focus_pixels": 2048,
|
||||||
|
"strata": [
|
||||||
|
{
|
||||||
|
"class_name": "high_grass",
|
||||||
|
"count": 2
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"class_name": "low_grass",
|
||||||
|
"count": 2
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"class_name": "bush",
|
||||||
|
"count": 2
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"class_name": "tree_trunk",
|
||||||
|
"count": 2
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"class_name": "tree_crown",
|
||||||
|
"count": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"class_name": "hedge",
|
||||||
|
"count": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"class_name": "forest",
|
||||||
|
"count": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"class_name": "crops",
|
||||||
|
"count": 1
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"error_overlay": {
|
||||||
|
"correct_material_rgba": [
|
||||||
|
34,
|
||||||
|
197,
|
||||||
|
94,
|
||||||
|
72
|
||||||
|
],
|
||||||
|
"missed_vegetation_rgba": [
|
||||||
|
239,
|
||||||
|
68,
|
||||||
|
68,
|
||||||
|
220
|
||||||
|
],
|
||||||
|
"false_vegetation_rgba": [
|
||||||
|
245,
|
||||||
|
158,
|
||||||
|
11,
|
||||||
|
220
|
||||||
|
],
|
||||||
|
"wrong_vegetation_material_rgba": [
|
||||||
|
168,
|
||||||
|
85,
|
||||||
|
247,
|
||||||
|
220
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"policy_action_colors": {
|
||||||
|
"ALLOW": "#22c55e",
|
||||||
|
"HIGH_COST": "#f59e0b",
|
||||||
|
"NO_GO": "#ef4444"
|
||||||
|
},
|
||||||
|
"invariants": {
|
||||||
|
"one_heavy_candidate_at_a_time": true,
|
||||||
|
"raw_fisheye_is_immutable": true,
|
||||||
|
"outside_center_crop_is_free": false,
|
||||||
|
"missing_or_unknown_is_free": false,
|
||||||
|
"camera_semantics_can_clear_rigid_geometry": false,
|
||||||
|
"navigation_authority": false,
|
||||||
|
"actuation_authority": false,
|
||||||
|
"canonical_triton_mutation_allowed": false
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
{
|
||||||
|
"schema_version": "missioncore.m49-tgs-portable-profile/v2",
|
||||||
|
"profile_id": "m49-tgs-portable-v2",
|
||||||
|
"source_binding": {
|
||||||
|
"mode": "admitted-k1-recording",
|
||||||
|
"camera_timeline": "dynamic",
|
||||||
|
"lidar_replay": "dynamic",
|
||||||
|
"trajectory": "dynamic",
|
||||||
|
"frame_counts": "source-derived",
|
||||||
|
"filesystem_paths": "executor-resolved"
|
||||||
|
},
|
||||||
|
"alignment": {
|
||||||
|
"timeline": "recorded-camera-host-arrival",
|
||||||
|
"lidar_selection": "latest-not-newer-than-camera-frame",
|
||||||
|
"maximum_lidar_age_seconds": 1.0,
|
||||||
|
"future_frames_allowed": false
|
||||||
|
},
|
||||||
|
"tgs": {
|
||||||
|
"max_range_m": 80.0,
|
||||||
|
"min_range_m": 1.0,
|
||||||
|
"resolution_m": 8.0,
|
||||||
|
"num_iterations": 3,
|
||||||
|
"num_lowest_representative_points": 5,
|
||||||
|
"minimum_points": 10,
|
||||||
|
"seed_threshold_m": 0.5,
|
||||||
|
"distance_threshold_m": 0.125,
|
||||||
|
"outlier_threshold_m": 0.3,
|
||||||
|
"normal_threshold": 0.94,
|
||||||
|
"weight_threshold": 200.0,
|
||||||
|
"lcc_normal_similarity": 0.03,
|
||||||
|
"lcc_planar_distance_m": 0.1,
|
||||||
|
"obstacle_height_m": 1.0,
|
||||||
|
"refine_mode": true
|
||||||
|
},
|
||||||
|
"rolling_profile": {
|
||||||
|
"history_seconds": 1.0,
|
||||||
|
"local_radius_m": 12.0,
|
||||||
|
"missing_lidar_policy": "all-cells-unobserved"
|
||||||
|
},
|
||||||
|
"costmap": {
|
||||||
|
"coordinate_frame": "map-gravity-local",
|
||||||
|
"cell_size_m": 0.45,
|
||||||
|
"radius_m": 12.0,
|
||||||
|
"state_priority": [
|
||||||
|
"NONGROUND_OCCUPIED",
|
||||||
|
"UNKNOWN_REJECTED",
|
||||||
|
"GROUND_SUPPORT",
|
||||||
|
"UNOBSERVED"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"invariants": {
|
||||||
|
"aos_allowed": false,
|
||||||
|
"gpu_allowed": false,
|
||||||
|
"missing_support_means_free": false,
|
||||||
|
"missing_lidar_means_unobserved": true,
|
||||||
|
"unobserved_cells_are_emitted": true,
|
||||||
|
"camera_projection_is_authoritative": false,
|
||||||
|
"navigation_or_actuation_allowed": false
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -278,11 +278,13 @@ an integrity or product need justifies a targeted migration.
|
|||||||
## Observatory durable recorded queue and Worker dispatch
|
## Observatory durable recorded queue and Worker dispatch
|
||||||
|
|
||||||
Selecting a source and a laboratory setup in Observatory is not itself a run.
|
Selecting a source and a laboratory setup in Observatory is not itself a run.
|
||||||
The public submission contains only `source_session_id`, `setup_id` and an
|
The exact legacy submission contains only `source_session_id`, `setup_id` and an
|
||||||
idempotency key. It cannot supply commands, executable text, filesystem paths,
|
idempotency key. A portable submission additionally returns the server-owned
|
||||||
container images, model identities, resource limits or priority. The server
|
`definition_sha256` and one content-bound `check_sha256` obtained from preflight;
|
||||||
resolves the allowlisted pair and seals all executable identity into one durable
|
both must be echoed unchanged during submit. Neither request can supply commands,
|
||||||
record:
|
executable text, filesystem paths, container images, model identities, resource
|
||||||
|
limits or priority. The server resolves the allowlisted identities and seals all
|
||||||
|
executable identity into one durable record:
|
||||||
|
|
||||||
- the current source-catalog snapshot captured at admission, plus immutable source
|
- the current source-catalog snapshot captured at admission, plus immutable source
|
||||||
bundle and source-capability-manifest SHA-256 identities;
|
bundle and source-capability-manifest SHA-256 identities;
|
||||||
@@ -307,22 +309,33 @@ not an unknown model dependency. The binding pins the exact source pack rather
|
|||||||
than a volatile whole-catalog digest; the server captures and seals the current
|
than a volatile whole-catalog digest; the server captures and seals the current
|
||||||
catalog snapshot into each admitted job.
|
catalog snapshot into each admitted job.
|
||||||
|
|
||||||
`LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39` now has a source-independent
|
Two source-independent definitions are projected by the portable catalog:
|
||||||
portable RunDefinition and lightweight recorded-source capability probe. The
|
|
||||||
definition seals the exact model/component/resource identities, observation-only
|
|
||||||
authority and generic `missioncore.recorded-eomt-ddrnet-review/v2` result
|
|
||||||
contract; it contains no source Session id or label. Compatibility is derived
|
|
||||||
from the selected Session's real K1 capabilities rather than its name. A
|
|
||||||
compatible source may therefore report capability `pass` independently from
|
|
||||||
executor readiness.
|
|
||||||
|
|
||||||
That portable foundation is not an executable product path yet. Its executor is
|
- `LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39` seals the exact
|
||||||
`not-installed`, and there is no accepted server-side definition-SHA/check-SHA
|
model/component/resource identities and the
|
||||||
fenced check/submit API, generic v2 result assembler/publisher or deployed Worker
|
`missioncore.recorded-eomt-ddrnet-review/v2` result contract;
|
||||||
executor. Preflight consequently remains blocked and the UI must not promise or
|
- `M4.9T5 · TRAVEL TGS · CPU-only, без ML` v2 seals an explicitly empty model
|
||||||
expose enqueue. The old `missioncore.lab-v1-vegetation-shadow/v1` result is not an
|
manifest, dynamic source-derived frame counts, causal TGS invariants and the
|
||||||
exact/existing result of the generic portable definition, even for its original
|
`missioncore.recorded-tgs-costmap-review/v2` result contract.
|
||||||
source; it remains available only in the immutable legacy LAB catalog.
|
|
||||||
|
Neither portable definition contains a source Session id, label, fixed frame
|
||||||
|
count or filesystem path. Compatibility is derived independently for every
|
||||||
|
setup from the selected Session's real K1 capabilities. A compatible source may
|
||||||
|
therefore report capability `pass` while that setup's executor remains blocked.
|
||||||
|
Blocked definitions stay projectable and do not prevent an unrelated ready
|
||||||
|
definition from entering the durable queue allowlist.
|
||||||
|
|
||||||
|
The server now owns a definition-SHA/check-SHA fenced portable check/submit
|
||||||
|
boundary. Submission still fails closed unless that exact definition has a
|
||||||
|
sealed `ready` executor release and image and is present in the durable queue.
|
||||||
|
Both repository portable definitions currently declare `not-installed`, so
|
||||||
|
their preflight remains blocked; no release or image hash is fabricated. The
|
||||||
|
generic v2 assemblers, verified publisher and Worker transport exist as a
|
||||||
|
dormant fail-closed implementation, but exact executor installation, central
|
||||||
|
storage, authentication, tunnel acceptance and end-to-end canaries remain
|
||||||
|
pending. The old `missioncore.lab-v1-vegetation-shadow/v1` result and the exact
|
||||||
|
RAVNOVES00 M4.9 result remain only in their existing immutable catalogs;
|
||||||
|
neither is reclassified as a result of a portable definition.
|
||||||
|
|
||||||
Recorded work has priority rank `100`. A future live K1 lease has rank `0` and
|
Recorded work has priority rank `100`. A future live K1 lease has rank `0` and
|
||||||
closes new recorded claims while it is pending or active. Cooperative executors
|
closes new recorded claims while it is pending or active. Cooperative executors
|
||||||
@@ -342,8 +355,10 @@ foundation. The production app hard-disables the Worker router even when a valid
|
|||||||
credential is present until an expiring claim lease and verified result publisher
|
credential is present until an expiring claim lease and verified result publisher
|
||||||
are accepted. Installation of the exact executors, Worker deployment and wiring
|
are accepted. Installation of the exact executors, Worker deployment and wiring
|
||||||
from the real K1 lifecycle to live-lease triggers remain pending. Therefore a
|
from the real K1 lifecycle to live-lease triggers remain pending. Therefore a
|
||||||
submitted M4.9T5 job may honestly wait in `queued` without implying that Worker
|
submitted exact legacy M4.9T5 job may honestly wait in `queued` without implying
|
||||||
006 can execute it yet; portable LAB V1 cannot currently be submitted at all.
|
that Worker 006 can execute it yet. Portable LAB V1 and portable M4.9T5 cannot
|
||||||
|
currently be submitted because their executor states are `not-installed` and
|
||||||
|
the authenticated Worker dispatch gate is closed.
|
||||||
|
|
||||||
Worker telemetry remains secondary observation evidence. It does not replace the
|
Worker telemetry remains secondary observation evidence. It does not replace the
|
||||||
authoritative queue ledger, result validation or common laboratory receipt. K1
|
authoritative queue ledger, result validation or common laboratory receipt. K1
|
||||||
|
|||||||
@@ -0,0 +1,121 @@
|
|||||||
|
# ADR 0047: Verified portable Observatory result publication
|
||||||
|
|
||||||
|
Date: 2026-08-31
|
||||||
|
Status: accepted as a backend foundation; production validators, transport and
|
||||||
|
executor wiring remain blocked
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
The durable Observatory queue deliberately treats a Worker `succeed` call as a
|
||||||
|
transport acknowledgement. Its `result_id` and SHA-256 do not prove that an
|
||||||
|
artifact exists, that it was computed from the admitted source, that it obeys
|
||||||
|
the selected RunDefinition, or that it has observation-only authority. Publishing
|
||||||
|
that acknowledgement directly as a LAB session would let incomplete, corrupt or
|
||||||
|
mislabelled output enter the same catalog as immutable evidence.
|
||||||
|
|
||||||
|
The current canonical LAB V1 result is a separate preserved legacy result. It
|
||||||
|
must not be reinterpreted as a portable v2 result, rewritten with new provenance,
|
||||||
|
or used as evidence that the portable executor exists.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Portable results cross a new backend-only verification boundary implemented in
|
||||||
|
`k1link.observatory.portable_result_contract` and
|
||||||
|
`k1link.observatory.portable_result_publisher`. The boundary does not change the
|
||||||
|
queue, the source-admission service, Session API, setup projection, UI, K1,
|
||||||
|
Simulation, or the legacy canonical publisher.
|
||||||
|
|
||||||
|
A Worker-side assembler must produce one directory whose basename is the
|
||||||
|
SHA-256 of its canonical `manifest.json`. The manifest schema is
|
||||||
|
`missioncore.observatory-portable-result-package/v1`; JSON bytes are canonical
|
||||||
|
UTF-8 with sorted keys and no insignificant whitespace. Its separately hashed
|
||||||
|
identity binds:
|
||||||
|
|
||||||
|
1. queue `job_id`, request identity, execution identity, submission receipt and
|
||||||
|
claim generation;
|
||||||
|
2. source Session, catalog snapshot, source bundle, capability manifest and
|
||||||
|
source-adapter identities;
|
||||||
|
3. the complete portable RunDefinition identity, including source requirements,
|
||||||
|
components, models, resource profile, result contract, executor and authority;
|
||||||
|
4. result id, result schema, result kind and result-contract SHA-256;
|
||||||
|
5. a canonical, role-sorted artifact list with confined relative paths, media
|
||||||
|
types, byte lengths and SHA-256 identities;
|
||||||
|
6. observation-only authority.
|
||||||
|
|
||||||
|
Exactly one non-empty `result-document` JSON artifact is required. Package roots,
|
||||||
|
the manifest, every path component and every artifact are checked without
|
||||||
|
following symlinks outside the package. The queue's terminal result SHA must equal
|
||||||
|
the canonical manifest SHA and the package directory name.
|
||||||
|
|
||||||
|
Before catalog publication, the server also:
|
||||||
|
|
||||||
|
- resolves the exact `(setup_id, definition_sha256)` in the portable registry and
|
||||||
|
proves every recorded queue field equals the resulting RunDefinition;
|
||||||
|
- rejects not-installed executors and the reserved legacy
|
||||||
|
`lab-v1-vegetation-shadow-<sha256>` namespace;
|
||||||
|
- rechecks the current SessionStore catalog snapshot;
|
||||||
|
- reads both source-admission documents from
|
||||||
|
`observatory-portable-source-contracts/<sha256>.json`, verifies their bytes,
|
||||||
|
schemas, cross-links, adapter, source and authority;
|
||||||
|
- invokes a validator registered by exact result-contract SHA-256;
|
||||||
|
- copies the package manifest and all output artifacts into the central immutable
|
||||||
|
content-addressed artifact store and records its manifest id;
|
||||||
|
- publishes one idempotent `LabSessionBinding` through `SessionStore`.
|
||||||
|
|
||||||
|
There is intentionally no generic “JSON looks plausible” validator. An unknown
|
||||||
|
result contract fails before artifact-store or SessionStore mutation. A validator
|
||||||
|
must understand the exact result schema and determine that the result document
|
||||||
|
and supporting artifacts are accepted evidence.
|
||||||
|
|
||||||
|
## Calculation-profile provenance
|
||||||
|
|
||||||
|
The publisher does not read a browser selection or infer a profile from a result
|
||||||
|
name. It requires a server-owned policy bound to the exact definition id, version
|
||||||
|
and SHA-256. The resulting immutable provenance contains
|
||||||
|
`missioncore.observatory-calculation-profile/v1` with:
|
||||||
|
|
||||||
|
- `setup_id`;
|
||||||
|
- full display name;
|
||||||
|
- origin `archived-definition`;
|
||||||
|
- definition id, version and SHA-256.
|
||||||
|
|
||||||
|
It also stores a SHA-256 of that calculation-profile document. A later Session API
|
||||||
|
projection can therefore read `calculation_profile` from the result's provenance
|
||||||
|
instead of reporting whichever setup happens to be selected now.
|
||||||
|
|
||||||
|
No replay capability is invented. The portable result package is preserved in
|
||||||
|
the central artifact store, while a result-schema-specific viewer/replay adapter
|
||||||
|
must be accepted separately before the catalog binding can claim visual replay.
|
||||||
|
|
||||||
|
## Current fail-closed blockers
|
||||||
|
|
||||||
|
The publisher and focused contract tests are implemented, but the end-to-end
|
||||||
|
production loop remains unavailable for concrete reasons:
|
||||||
|
|
||||||
|
1. neither `recorded-eomt-ddrnet-review-v2` nor
|
||||||
|
`m49-tgs-portable-review-v2` has an installed exact result-contract validator;
|
||||||
|
2. the current Worker protocol returns only `result_id` and manifest SHA-256; it
|
||||||
|
has no accepted package upload/CAS handoff that gives Mission Core the matching
|
||||||
|
content-addressed directory;
|
||||||
|
3. portable executor releases that emit this package contract are not installed
|
||||||
|
and physically accepted on Worker 006;
|
||||||
|
4. the application has not registered definition-bound calculation-profile
|
||||||
|
publication policies or wired the publisher into the terminal Worker flow;
|
||||||
|
5. no result-schema-specific replay-capability adapter has been accepted.
|
||||||
|
|
||||||
|
These are explicit blockers. The backend must not fabricate a package, reuse a
|
||||||
|
legacy result, trust a Worker success receipt, guess model/profile provenance, or
|
||||||
|
expose an enqueue/result promise to bypass them.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- Queue success and Observatory publication remain distinct evidence states.
|
||||||
|
- Exact retry is safe: content-addressed artifact publication and the immutable
|
||||||
|
SessionStore binding are idempotent; a conflicting result id fails closed.
|
||||||
|
- Source, definition, model, resource-profile and calculation-profile identities
|
||||||
|
remain available in one portable provenance document.
|
||||||
|
- Artifact-store writes may leave harmless immutable unreferenced objects if the
|
||||||
|
final SessionStore transaction detects a conflict; they cannot overwrite an
|
||||||
|
existing identity.
|
||||||
|
- The legacy canonical LAB V1 result and its admission logic remain byte-for-byte
|
||||||
|
outside this publisher.
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
# Portable Observatory Worker 006 operational boundary
|
||||||
|
|
||||||
|
## Scope and current state
|
||||||
|
|
||||||
|
This runbook covers only recorded, observation-only Observatory jobs. It does
|
||||||
|
not change K1 acquisition/control, Simulation/Gaussian, legacy LAB execution,
|
||||||
|
navigation or safety authority.
|
||||||
|
|
||||||
|
The server queue, renewable claim lease, verified source/result transport and
|
||||||
|
result publisher exist, but the production Worker API remains deliberately
|
||||||
|
disabled. All three application gates stay `False` until exact executors are
|
||||||
|
installed and a complete transport smoke has passed:
|
||||||
|
|
||||||
|
- `OBSERVATORY_WORKER_CLAIM_LEASE_READY`;
|
||||||
|
- `OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY`;
|
||||||
|
- `OBSERVATORY_WORKER_PRODUCTION_API_ENABLED`.
|
||||||
|
|
||||||
|
The canonical central artifact store is already declared in the installed
|
||||||
|
Mission Core LaunchAgent as:
|
||||||
|
|
||||||
|
```text
|
||||||
|
MISSIONCORE_ARTIFACT_STORE_ROOT=/Volumes/docker/nodedc-mission-core/artifact-store
|
||||||
|
```
|
||||||
|
|
||||||
|
On 2026-08-31 `/Volumes/docker` was not mounted. This is a fail-closed
|
||||||
|
preflight failure, not permission to create a checkout-local substitute. The
|
||||||
|
portable Worker server composition now requires `central_status=ready` before
|
||||||
|
it can be constructed.
|
||||||
|
|
||||||
|
The installed `com.nodedc.mission-core.local` LaunchAgent was running from the
|
||||||
|
M5 Observatory feature worktree while retaining the established Mission Core
|
||||||
|
data directory in the main checkout. It declares the central artifact-store
|
||||||
|
root and cache limits, but not the two portable storage roots below. Do not
|
||||||
|
silently edit or restart this hybrid local service; carry the environment and
|
||||||
|
working-directory change through one separately reviewed, hash-gated service
|
||||||
|
transition after the code release is sealed.
|
||||||
|
|
||||||
|
Large portable inputs and in-flight results have no Mission Core data-directory
|
||||||
|
fallback. The server requires both roots through environment-only
|
||||||
|
configuration:
|
||||||
|
|
||||||
|
```text
|
||||||
|
MISSIONCORE_OBSERVATORY_WORKER_SOURCE_CAS_ROOT=/Volumes/docker/nodedc-mission-core/observatory-worker/source-cas
|
||||||
|
MISSIONCORE_OBSERVATORY_WORKER_RESULT_STAGING_ROOT=/Volumes/docker/nodedc-mission-core/observatory-worker/result-staging
|
||||||
|
```
|
||||||
|
|
||||||
|
Both directories must already exist, be canonical non-symlink directories,
|
||||||
|
remain inside `/Volumes/docker/nodedc-mission-core`, and be disjoint from each
|
||||||
|
other and from `artifact-store`. Mission Core never creates these configured
|
||||||
|
roots. If `/Volumes/docker` is absent or is only a local directory rather than
|
||||||
|
a mounted volume, composition fails closed before queue or Worker API exposure.
|
||||||
|
Provision the directories only through the reviewed server deployment after
|
||||||
|
the SMB mount preflight succeeds.
|
||||||
|
|
||||||
|
Read-only Worker evidence on the same date:
|
||||||
|
|
||||||
|
- strict-pinned `ssh -o BatchMode=yes mission-gpu` succeeds;
|
||||||
|
- Worker host identity remains `DESKTOP-OPJ8J04`;
|
||||||
|
- Windows OpenSSH `sshd` is running;
|
||||||
|
- `AllowTcpForwarding` and `GatewayPorts` use OpenSSH defaults: forwarding is
|
||||||
|
allowed and remote listeners are not exposed beyond loopback;
|
||||||
|
- Worker loopback port `18080` had no listener.
|
||||||
|
|
||||||
|
## Network shape
|
||||||
|
|
||||||
|
Mission Core remains the only backend on Mac loopback port `8000`. The Mac
|
||||||
|
owns one reverse SSH tunnel through the existing strict-pinned `mission-gpu`
|
||||||
|
alias:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Worker 006 process
|
||||||
|
-> http://127.0.0.1:18080
|
||||||
|
-> encrypted SSH reverse forwarding
|
||||||
|
-> Mac http://127.0.0.1:8000
|
||||||
|
```
|
||||||
|
|
||||||
|
The exact forwarding declaration is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
-R 127.0.0.1:18080:127.0.0.1:8000
|
||||||
|
```
|
||||||
|
|
||||||
|
It neither opens a LAN listener nor sends a credential in process arguments.
|
||||||
|
`ObservatoryWorkerHttpGateway` independently rejects plaintext HTTP to any
|
||||||
|
non-loopback host. `worker_tunnel_launchd.py` builds a pure, hashable launchd
|
||||||
|
plan; `scripts/plan_observatory_worker_tunnel.py` prints that plan and performs
|
||||||
|
no installation.
|
||||||
|
|
||||||
|
## Bearer credential
|
||||||
|
|
||||||
|
The server reads one credential from the fixed private data path:
|
||||||
|
|
||||||
|
```text
|
||||||
|
<MISSIONCORE_DATA_DIR>/worker-auth/observatory-worker.token
|
||||||
|
```
|
||||||
|
|
||||||
|
The installed Worker release receives the same secret through its own private,
|
||||||
|
runner-managed file and passes only its path as
|
||||||
|
`MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE`. The token is ASCII, 32–512 bytes,
|
||||||
|
has no newline, is never stored in Git, an artifact, a plist, an environment
|
||||||
|
value, a command line or Ops plaintext, and is read with no-follow semantics.
|
||||||
|
The admitted service runtime is POSIX and requires mode `0600` or narrower;
|
||||||
|
native Windows ACL handling is intentionally not guessed.
|
||||||
|
|
||||||
|
The Worker service accepts only these non-executable settings:
|
||||||
|
|
||||||
|
```text
|
||||||
|
MISSIONCORE_OBSERVATORY_WORKER_BASE_URL=http://127.0.0.1:18080
|
||||||
|
MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE=<absolute-private-path>
|
||||||
|
MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT=<absolute-private-D-backed-path>
|
||||||
|
MISSIONCORE_OBSERVATORY_WORKER_IDLE_POLL_SECONDS=1
|
||||||
|
MISSIONCORE_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS=5
|
||||||
|
MISSIONCORE_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES=12
|
||||||
|
```
|
||||||
|
|
||||||
|
There is no configurable module, command, image or executable entrypoint.
|
||||||
|
|
||||||
|
## Install-time executor seam
|
||||||
|
|
||||||
|
`compose_installed_observatory_worker_service` is called only by a reviewed
|
||||||
|
Worker release. That release injects a constructed
|
||||||
|
`ObservatoryWorkerExecutorRegistry` directly in memory. Before opening the
|
||||||
|
HTTP gateway, service composition resolves the four-digest identity of every
|
||||||
|
portable RunDefinition whose executor state is `ready`:
|
||||||
|
|
||||||
|
- executor release SHA-256;
|
||||||
|
- executor image SHA-256;
|
||||||
|
- model manifest SHA-256;
|
||||||
|
- resource profile SHA-256.
|
||||||
|
|
||||||
|
No ready definitions, an empty registry, or one missing identity stops the
|
||||||
|
service before its first claim. Blocked candidates cannot be selected through
|
||||||
|
configuration and are not silently registered.
|
||||||
|
|
||||||
|
## Staged deployment and smoke sequence
|
||||||
|
|
||||||
|
No step below was applied by this implementation increment.
|
||||||
|
|
||||||
|
1. Seal each executor release and installation receipt. Change a portable
|
||||||
|
RunDefinition to `ready` only when its exact release/image identities and
|
||||||
|
local adapter admission agree.
|
||||||
|
2. Mount the existing canonical SMB artifact store. Require Mission Core
|
||||||
|
artifact status `central_status=ready`; do not initialize a local surrogate.
|
||||||
|
Through the reviewed server deployment, provision the two disjoint portable
|
||||||
|
roots above, set both environment values, and verify they resolve inside the
|
||||||
|
same mounted `/Volumes/docker/nodedc-mission-core` boundary.
|
||||||
|
3. Provision one bearer credential through the deployment-owned secret path
|
||||||
|
on both Mac and the admitted POSIX Worker service runtime. Verify file type,
|
||||||
|
no-link handling and private permissions without printing the value.
|
||||||
|
4. Generate the reverse-tunnel launchd plan, review its SHA-256 and exact
|
||||||
|
arguments, then install it through a separate hash-gated local-service
|
||||||
|
change. Accept only Worker-side `127.0.0.1:18080/api/health` reaching the
|
||||||
|
canonical Mac service; no second backend is started.
|
||||||
|
5. Install the exact Worker release. Its fixed entrypoint loads the sealed
|
||||||
|
RunDefinition registry and its in-memory executor registry, then calls
|
||||||
|
`compose_installed_observatory_worker_service`. The process must refuse a
|
||||||
|
missing token, non-loopback plaintext URL, unsafe work root, absent ready
|
||||||
|
definition or executor coverage gap.
|
||||||
|
6. Restart the canonical Mission Core process once with CAS and server token
|
||||||
|
available. Keep the Worker route disabled and verify that K1,
|
||||||
|
Simulation/Gaussian and legacy LAB surfaces are unchanged.
|
||||||
|
7. In a separate reviewed source gate, flip the claim-lease, verified-publisher
|
||||||
|
and production-API flags together. Restart only the canonical port `8000`
|
||||||
|
service. An authenticated empty claim must return `204`; missing/wrong bearer
|
||||||
|
and wrong contour identity must remain `401/403`.
|
||||||
|
8. Submit one short recorded K1 canary for each profile. Require exact source
|
||||||
|
materialization, lease heartbeat, result-package digest validation, central
|
||||||
|
artifact publication, immutable calculation-profile provenance and a
|
||||||
|
reopenable Observatory result.
|
||||||
|
9. During a bounded recorded canary, start the existing live K1 priority
|
||||||
|
transition. Require the recorded job to pause/defer and resume only after
|
||||||
|
live K1 releases the single Worker resource. This test never grants control
|
||||||
|
or navigation authority.
|
||||||
|
|
||||||
|
## Remaining blockers
|
||||||
|
|
||||||
|
- canonical SMB artifact store is currently unmounted on the Mac;
|
||||||
|
- portable source CAS and result-staging directories/environment values are not
|
||||||
|
provisioned;
|
||||||
|
- the installed Mission Core LaunchAgent has not received a sealed
|
||||||
|
working-directory/environment transition for this release;
|
||||||
|
- the shared bearer credential has not been provisioned;
|
||||||
|
- the reverse tunnel plan has not been installed or smoked;
|
||||||
|
- the Worker polling entrypoint has not been packaged into an admitted POSIX
|
||||||
|
Worker release;
|
||||||
|
- both exact executor releases/install receipts still need their own seals;
|
||||||
|
- production flags and the server route remain off by design;
|
||||||
|
- no end-to-end result has yet crossed Worker 006 -> central store ->
|
||||||
|
Observatory under this new path.
|
||||||
|
|
||||||
|
The Synology root-owned `nodedc-deploy` registry has no
|
||||||
|
`mission-core-worker` component. Older Mission Core Worker shadow artifacts
|
||||||
|
explicitly declare that they are outside that registry. Do not route a Windows
|
||||||
|
Worker install through an unrelated NAS component or weaken the deploy canon;
|
||||||
|
the durable Worker release needs its own exact reviewed installation transition
|
||||||
|
and rollback evidence.
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
FROM ndc/mission-core-m49-t3-travel:20260826 AS builder
|
||||||
|
|
||||||
|
COPY run_m49_tgs_portable.cpp /build/run_m49_tgs_portable.cpp
|
||||||
|
|
||||||
|
RUN echo "52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9 /build/run_m49_tgs_portable.cpp" \
|
||||||
|
| sha256sum --check --strict \
|
||||||
|
&& g++ -std=c++17 -O3 -DNDEBUG -pthread \
|
||||||
|
-I/opt/travel/src/TRAVEL/cpp/travel/core \
|
||||||
|
-I/usr/include/eigen3 \
|
||||||
|
/build/run_m49_tgs_portable.cpp \
|
||||||
|
-o /build/run_m49_tgs_portable
|
||||||
|
|
||||||
|
FROM ndc/mission-core-m49-t3-travel:20260826
|
||||||
|
|
||||||
|
COPY --from=builder /build/run_m49_tgs_portable /opt/nodedc/m49-tgs-portable/bin/run_m49_tgs_portable
|
||||||
|
COPY m49-tgs-portable-v2.json /opt/nodedc/m49-tgs-portable/profile/m49-tgs-portable-v2.json
|
||||||
|
COPY smoke_m49_tgs_portable.sh /opt/nodedc/m49-tgs-portable/bin/smoke_m49_tgs_portable
|
||||||
|
|
||||||
|
RUN echo "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9 /opt/nodedc/m49-tgs-portable/profile/m49-tgs-portable-v2.json" \
|
||||||
|
| sha256sum --check --strict \
|
||||||
|
&& echo "e5da73c2cf89ed0671de8ac52e13014a7faf79ce1ff1f97a090154063a61c18d /opt/nodedc/m49-tgs-portable/bin/smoke_m49_tgs_portable" \
|
||||||
|
| sha256sum --check --strict \
|
||||||
|
&& chmod 0555 /opt/nodedc/m49-tgs-portable/bin/smoke_m49_tgs_portable \
|
||||||
|
&& test -x /opt/nodedc/m49-tgs-portable/bin/run_m49_tgs_portable
|
||||||
|
|
||||||
|
LABEL com.nodedc.product="mission-core" \
|
||||||
|
com.nodedc.stack="ndc-mission-core-observatory" \
|
||||||
|
com.nodedc.role="m49-tgs-portable-executor" \
|
||||||
|
com.nodedc.managed-by="mission-core-worker-release" \
|
||||||
|
com.nodedc.worker-contour="worker-006" \
|
||||||
|
com.nodedc.authority="observation-only" \
|
||||||
|
com.nodedc.base-image.sha256="7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f" \
|
||||||
|
com.nodedc.runner-source.sha256="52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9" \
|
||||||
|
com.nodedc.fixture-smoke.sha256="e5da73c2cf89ed0671de8ac52e13014a7faf79ce1ff1f97a090154063a61c18d" \
|
||||||
|
com.nodedc.profile.sha256="6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9"
|
||||||
|
|
||||||
|
ENTRYPOINT ["/opt/nodedc/m49-tgs-portable/bin/run_m49_tgs_portable"]
|
||||||
+401
@@ -0,0 +1,401 @@
|
|||||||
|
[CmdletBinding()]
|
||||||
|
param(
|
||||||
|
[Parameter(Mandatory = $true)]
|
||||||
|
[ValidatePattern("^[a-f0-9]{64}$")]
|
||||||
|
[string]$CandidateId,
|
||||||
|
|
||||||
|
[Parameter(Mandatory = $true)]
|
||||||
|
[ValidatePattern("^[a-f0-9]{64}$")]
|
||||||
|
[string]$ArchiveSha256
|
||||||
|
)
|
||||||
|
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
$ProgressPreference = "SilentlyContinue"
|
||||||
|
|
||||||
|
$WorkerId = "worker-006"
|
||||||
|
$ExpectedComputer = "DESKTOP-OPJ8J04"
|
||||||
|
$ReleaseId = "m49-tgs-portable-executor-v1"
|
||||||
|
$ReleaseParent = "D:\NDC_MISSIONCORE\runtime\releases\observatory-portable"
|
||||||
|
$BaseImage = "ndc/mission-core-m49-t3-travel:20260826"
|
||||||
|
$BaseImageId = "sha256:7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f"
|
||||||
|
$ProfileSha256 = "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9"
|
||||||
|
$RunnerSourceSha256 = "52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9"
|
||||||
|
$RunnerWrapperSha256 = "2d6c32560682647f868e4ce4c2605749f17c60482a609f8c03ff951411f48ffb"
|
||||||
|
$FixtureSmokeSha256 = "e5da73c2cf89ed0671de8ac52e13014a7faf79ce1ff1f97a090154063a61c18d"
|
||||||
|
$ResultContractSha256 = "9dd80c8e2504559d2156fca933de6eb27901e35305e6853aeb84707e1cb13892"
|
||||||
|
$ProtectedContainers = @(
|
||||||
|
"ndc-mission-core-triton",
|
||||||
|
"ndc-mission-core-perception-worker",
|
||||||
|
"ndc-gaussian-pipeline-gaussian-gateway-1",
|
||||||
|
"ndc-gaussian-pipeline-gaussian-pipeline-1",
|
||||||
|
"ndc-gaussian-pipeline-gaussian-terrain-executor-1"
|
||||||
|
)
|
||||||
|
$Authority = [ordered]@{
|
||||||
|
commands_enabled = $false
|
||||||
|
actuation_allowed = $false
|
||||||
|
navigation_or_safety_accepted = $false
|
||||||
|
production_accepted = $false
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-LastExitCode([string]$Operation) {
|
||||||
|
if ($LASTEXITCODE -ne 0) {
|
||||||
|
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Resolve-DDirectory([string]$Path, [string]$Label) {
|
||||||
|
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||||
|
if (
|
||||||
|
-not $item.PSIsContainer -or
|
||||||
|
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
||||||
|
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
|
||||||
|
) {
|
||||||
|
throw "$Label must be a real D: directory"
|
||||||
|
}
|
||||||
|
return $item.FullName
|
||||||
|
}
|
||||||
|
|
||||||
|
function Resolve-DFile([string]$Path, [string]$Label) {
|
||||||
|
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||||
|
if (
|
||||||
|
$item.PSIsContainer -or
|
||||||
|
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
||||||
|
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
|
||||||
|
) {
|
||||||
|
throw "$Label must be a real D: file"
|
||||||
|
}
|
||||||
|
return $item.FullName
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-Sha256([string]$Path) {
|
||||||
|
return (Get-FileHash -Algorithm SHA256 -LiteralPath $Path).Hash.ToLowerInvariant()
|
||||||
|
}
|
||||||
|
|
||||||
|
function Write-Utf8NoBom([string]$Path, [string]$Value) {
|
||||||
|
$encoding = New-Object System.Text.UTF8Encoding($false)
|
||||||
|
[IO.File]::WriteAllText($Path, $Value, $encoding)
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-ProtectedRuntime() {
|
||||||
|
$rows = @(((& docker inspect $ProtectedContainers) | ConvertFrom-Json))
|
||||||
|
Assert-LastExitCode "protected runtime inspection"
|
||||||
|
if ($rows.Count -ne $ProtectedContainers.Count) {
|
||||||
|
throw "protected runtime inventory is incomplete"
|
||||||
|
}
|
||||||
|
return $rows
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-ProtectedRuntime([object[]]$Before, [object[]]$After) {
|
||||||
|
$beforeByName = @{}
|
||||||
|
foreach ($row in $Before) {
|
||||||
|
$beforeByName[[string]$row.Name] = [string]$row.Id
|
||||||
|
}
|
||||||
|
foreach ($row in $After) {
|
||||||
|
$name = [string]$row.Name
|
||||||
|
if (-not $beforeByName.ContainsKey($name) -or $beforeByName[$name] -cne [string]$row.Id) {
|
||||||
|
throw "protected runtime identity changed: $name"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$triton = @($After | Where-Object { [string]$_.Name -ceq "/ndc-mission-core-triton" })
|
||||||
|
if (
|
||||||
|
$triton.Count -ne 1 -or
|
||||||
|
-not $triton[0].State.Running -or
|
||||||
|
[string]$triton[0].State.Health.Status -cne "healthy"
|
||||||
|
) {
|
||||||
|
throw "canonical Mission Core Triton is not healthy"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($env:COMPUTERNAME -cne $ExpectedComputer) {
|
||||||
|
throw "portable M4.9 executor is pinned to Worker 006"
|
||||||
|
}
|
||||||
|
|
||||||
|
$releaseCandidate = Join-Path $ReleaseParent "m49-tgs-portable-candidate-$CandidateId"
|
||||||
|
$releaseRoot = Resolve-DDirectory $releaseCandidate "portable M4.9 candidate release"
|
||||||
|
$archive = Resolve-DFile (Join-Path $releaseRoot "candidate.tgz") "portable M4.9 candidate archive"
|
||||||
|
if ((Get-Sha256 $archive) -cne $ArchiveSha256) {
|
||||||
|
throw "portable M4.9 candidate archive digest changed"
|
||||||
|
}
|
||||||
|
|
||||||
|
$sourceRoot = Join-Path $releaseRoot "source-candidate"
|
||||||
|
if (Test-Path -LiteralPath $sourceRoot) {
|
||||||
|
throw "portable M4.9 source candidate was already extracted"
|
||||||
|
}
|
||||||
|
$null = New-Item -ItemType Directory -Path $sourceRoot
|
||||||
|
& tar -xzf $archive -C $sourceRoot
|
||||||
|
Assert-LastExitCode "portable M4.9 candidate extraction"
|
||||||
|
$sourceRoot = Resolve-DDirectory $sourceRoot "portable M4.9 source candidate"
|
||||||
|
|
||||||
|
$manifestPath = Resolve-DFile (Join-Path $sourceRoot "release-manifest.json") "portable M4.9 candidate manifest"
|
||||||
|
$manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
|
||||||
|
if (
|
||||||
|
[string]$manifest.candidate_sha256 -cne $CandidateId -or
|
||||||
|
[string]$manifest.source_state -cne "uncommitted-candidate" -or
|
||||||
|
[string]$manifest.state -cne "blocked" -or
|
||||||
|
[string]$manifest.travel_build_image_sha256 -cne $BaseImageId.Substring(7) -or
|
||||||
|
[string]$manifest.profile_sha256 -cne $ProfileSha256 -or
|
||||||
|
[string]$manifest.result_contract_sha256 -cne $ResultContractSha256
|
||||||
|
) {
|
||||||
|
throw "portable M4.9 candidate manifest identity changed"
|
||||||
|
}
|
||||||
|
|
||||||
|
$base = @(((& docker image inspect $BaseImage) | ConvertFrom-Json))
|
||||||
|
Assert-LastExitCode "TRAVEL predecessor image inspection"
|
||||||
|
if ($base.Count -ne 1 -or [string]$base[0].Id -cne $BaseImageId) {
|
||||||
|
throw "exact TRAVEL predecessor image changed"
|
||||||
|
}
|
||||||
|
|
||||||
|
$protectedBefore = @(Get-ProtectedRuntime)
|
||||||
|
Assert-ProtectedRuntime $protectedBefore $protectedBefore
|
||||||
|
$legacyTaskBefore = Get-ScheduledTask -TaskName "MissionCore-M49TgsFullShadow" -ErrorAction SilentlyContinue
|
||||||
|
|
||||||
|
$shortCandidate = $CandidateId.Substring(0, 16)
|
||||||
|
$imageTag = "ndc/mission-core-m49-tgs-portable-executor:$shortCandidate-candidate"
|
||||||
|
$existingImageTags = @(
|
||||||
|
& docker image ls --format "{{.Repository}}:{{.Tag}}" --filter "reference=$imageTag"
|
||||||
|
)
|
||||||
|
Assert-LastExitCode "portable M4.9 executor image collision check"
|
||||||
|
if ($existingImageTags -contains $imageTag) {
|
||||||
|
throw "portable M4.9 executor image tag already exists"
|
||||||
|
}
|
||||||
|
|
||||||
|
$payload = Resolve-DDirectory (Join-Path $sourceRoot "payload") "portable M4.9 candidate payload"
|
||||||
|
$dockerfile = Resolve-DFile (
|
||||||
|
Join-Path $payload "experiments\perception\worker\observatory_portable\Dockerfile.m49-portable-executor"
|
||||||
|
) "portable M4.9 Dockerfile"
|
||||||
|
$runnerSource = Resolve-DFile (
|
||||||
|
Join-Path $payload "experiments\perception\worker\observatory_portable\run_m49_tgs_portable.cpp"
|
||||||
|
) "portable M4.9 runner source"
|
||||||
|
$runnerWrapper = Resolve-DFile (
|
||||||
|
Join-Path $payload "experiments\perception\worker\observatory_portable\run_m49_tgs_portable.sh"
|
||||||
|
) "portable M4.9 runner wrapper"
|
||||||
|
$fixtureSmoke = Resolve-DFile (
|
||||||
|
Join-Path $payload "experiments\perception\worker\observatory_portable\smoke_m49_tgs_portable.sh"
|
||||||
|
) "portable M4.9 fixture smoke"
|
||||||
|
$profile = Resolve-DFile (
|
||||||
|
Join-Path $payload "config\perception\m49-tgs-portable-v2.json"
|
||||||
|
) "portable M4.9 profile"
|
||||||
|
if (
|
||||||
|
(Get-Sha256 $runnerSource) -cne $RunnerSourceSha256 -or
|
||||||
|
(Get-Sha256 $runnerWrapper) -cne $RunnerWrapperSha256 -or
|
||||||
|
(Get-Sha256 $fixtureSmoke) -cne $FixtureSmokeSha256 -or
|
||||||
|
(Get-Sha256 $profile) -cne $ProfileSha256
|
||||||
|
) {
|
||||||
|
throw "portable M4.9 exact source files changed"
|
||||||
|
}
|
||||||
|
|
||||||
|
$context = Join-Path $releaseRoot "build-context"
|
||||||
|
if (Test-Path -LiteralPath $context) {
|
||||||
|
throw "portable M4.9 immutable build context already exists"
|
||||||
|
}
|
||||||
|
$null = New-Item -ItemType Directory -Path $context
|
||||||
|
Copy-Item -LiteralPath $dockerfile -Destination (Join-Path $context "Dockerfile")
|
||||||
|
Copy-Item -LiteralPath $runnerSource -Destination (Join-Path $context "run_m49_tgs_portable.cpp")
|
||||||
|
Copy-Item -LiteralPath $profile -Destination (Join-Path $context "m49-tgs-portable-v2.json")
|
||||||
|
Copy-Item -LiteralPath $fixtureSmoke -Destination (Join-Path $context "smoke_m49_tgs_portable.sh")
|
||||||
|
$context = Resolve-DDirectory $context "portable M4.9 immutable build context"
|
||||||
|
|
||||||
|
$dockerConfig = Join-Path $releaseRoot "docker-config"
|
||||||
|
$null = New-Item -ItemType Directory -Path $dockerConfig
|
||||||
|
Write-Utf8NoBom (Join-Path $dockerConfig "config.json") '{"auths":{}}'
|
||||||
|
$env:DOCKER_CONFIG = $dockerConfig
|
||||||
|
|
||||||
|
$buildOutput = @(& docker build --quiet --pull=false --no-cache --network none --tag $imageTag $context)
|
||||||
|
Assert-LastExitCode "portable M4.9 executor image build"
|
||||||
|
$builtImageId = ([string]$buildOutput[-1]).Trim()
|
||||||
|
$image = @(((& docker image inspect $imageTag) | ConvertFrom-Json))
|
||||||
|
Assert-LastExitCode "portable M4.9 executor image inspection"
|
||||||
|
if ($image.Count -ne 1 -or [string]$image[0].Id -cne $builtImageId) {
|
||||||
|
throw "portable M4.9 executor image identity is ambiguous"
|
||||||
|
}
|
||||||
|
$expectedLabels = [ordered]@{
|
||||||
|
"com.nodedc.product" = "mission-core"
|
||||||
|
"com.nodedc.stack" = "ndc-mission-core-observatory"
|
||||||
|
"com.nodedc.role" = "m49-tgs-portable-executor"
|
||||||
|
"com.nodedc.managed-by" = "mission-core-worker-release"
|
||||||
|
"com.nodedc.worker-contour" = $WorkerId
|
||||||
|
"com.nodedc.authority" = "observation-only"
|
||||||
|
"com.nodedc.base-image.sha256" = $BaseImageId.Substring(7)
|
||||||
|
"com.nodedc.runner-source.sha256" = $RunnerSourceSha256
|
||||||
|
"com.nodedc.fixture-smoke.sha256" = $FixtureSmokeSha256
|
||||||
|
"com.nodedc.profile.sha256" = $ProfileSha256
|
||||||
|
}
|
||||||
|
foreach ($entry in $expectedLabels.GetEnumerator()) {
|
||||||
|
if ([string]$image[0].Config.Labels.($entry.Key) -cne [string]$entry.Value) {
|
||||||
|
throw "portable M4.9 executor image label changed: $($entry.Key)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$installed = Join-Path $releaseRoot "installed"
|
||||||
|
$null = New-Item -ItemType Directory -Path $installed
|
||||||
|
$extractName = "ndc-mission-core-m49-portable-extract-$shortCandidate"
|
||||||
|
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$extractName$") {
|
||||||
|
throw "portable M4.9 extraction container already exists"
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$null = & docker create --name $extractName $imageTag
|
||||||
|
Assert-LastExitCode "portable M4.9 extraction container creation"
|
||||||
|
& docker cp (
|
||||||
|
"$extractName`:/opt/nodedc/m49-tgs-portable/bin/run_m49_tgs_portable"
|
||||||
|
) (Join-Path $installed "run_m49_tgs_portable")
|
||||||
|
Assert-LastExitCode "portable M4.9 binary extraction"
|
||||||
|
& docker cp (
|
||||||
|
"$extractName`:/opt/nodedc/m49-tgs-portable/profile/m49-tgs-portable-v2.json"
|
||||||
|
) (Join-Path $installed "m49-tgs-portable-v2.json")
|
||||||
|
Assert-LastExitCode "portable M4.9 profile extraction"
|
||||||
|
} finally {
|
||||||
|
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$extractName$") {
|
||||||
|
& docker rm --force $extractName *> $null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$installedBinary = Resolve-DFile (Join-Path $installed "run_m49_tgs_portable") "installed portable M4.9 runner"
|
||||||
|
$installedProfile = Resolve-DFile (Join-Path $installed "m49-tgs-portable-v2.json") "installed portable M4.9 profile"
|
||||||
|
if ((Get-Sha256 $installedProfile) -cne $ProfileSha256) {
|
||||||
|
throw "installed portable M4.9 profile changed"
|
||||||
|
}
|
||||||
|
$binarySha256 = Get-Sha256 $installedBinary
|
||||||
|
$binaryLength = (Get-Item -LiteralPath $installedBinary).Length
|
||||||
|
|
||||||
|
$buildSeal = [ordered]@{
|
||||||
|
schema_version = "missioncore.m49-tgs-portable-compiled-runner-build-candidate/v1"
|
||||||
|
source_revision = [string]$manifest.source_revision
|
||||||
|
source_state = [string]$manifest.source_state
|
||||||
|
candidate_sha256 = $CandidateId
|
||||||
|
build_image_sha256 = $BaseImageId.Substring(7)
|
||||||
|
profile_sha256 = $ProfileSha256
|
||||||
|
runner_source_sha256 = $RunnerSourceSha256
|
||||||
|
runner_wrapper_sha256 = $RunnerWrapperSha256
|
||||||
|
compiler_contract = [ordered]@{
|
||||||
|
compiler = "g++"
|
||||||
|
language_standard = "c++17"
|
||||||
|
flags = @("-O3", "-DNDEBUG", "-pthread")
|
||||||
|
travel_include = "/opt/travel/src/TRAVEL/cpp/travel/core"
|
||||||
|
eigen_include = "/usr/include/eigen3"
|
||||||
|
}
|
||||||
|
binary = [ordered]@{
|
||||||
|
file_name = "run_m49_tgs_portable"
|
||||||
|
format = "elf"
|
||||||
|
byte_length = [long]$binaryLength
|
||||||
|
sha256 = $binarySha256
|
||||||
|
}
|
||||||
|
authority = $Authority
|
||||||
|
}
|
||||||
|
$buildSealPath = Join-Path $installed "compiled-runner-build-candidate.json"
|
||||||
|
Write-Utf8NoBom $buildSealPath (($buildSeal | ConvertTo-Json -Depth 8 -Compress) + "`n")
|
||||||
|
$buildSealSha256 = Get-Sha256 $buildSealPath
|
||||||
|
|
||||||
|
$smokeName = "ndc-mission-core-m49-portable-smoke-$shortCandidate"
|
||||||
|
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$smokeName$") {
|
||||||
|
throw "portable M4.9 smoke container already exists"
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$smokeOutput = @(& docker run --rm --name $smokeName --network none --read-only --cpus 2 --memory 2g --pids-limit 256 --tmpfs "/smoke:rw,nosuid,nodev,size=32m" --entrypoint /opt/nodedc/m49-tgs-portable/bin/smoke_m49_tgs_portable $imageTag)
|
||||||
|
Assert-LastExitCode "portable M4.9 fixture smoke"
|
||||||
|
} finally {
|
||||||
|
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$smokeName$") {
|
||||||
|
& docker rm --force $smokeName *> $null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($smokeOutput[-1] -cne "M49_PORTABLE_FIXTURE_SMOKE_OK frames=2 points_per_frame=124668") {
|
||||||
|
throw "portable M4.9 fixture smoke receipt changed"
|
||||||
|
}
|
||||||
|
|
||||||
|
$protectedAfter = @(Get-ProtectedRuntime)
|
||||||
|
Assert-ProtectedRuntime $protectedBefore $protectedAfter
|
||||||
|
$legacyTaskAfter = Get-ScheduledTask -TaskName "MissionCore-M49TgsFullShadow" -ErrorAction SilentlyContinue
|
||||||
|
if (
|
||||||
|
($null -eq $legacyTaskBefore) -ne ($null -eq $legacyTaskAfter) -or
|
||||||
|
($null -ne $legacyTaskBefore -and [string]$legacyTaskBefore.State -cne [string]$legacyTaskAfter.State)
|
||||||
|
) {
|
||||||
|
throw "legacy M49 scheduled task changed during executor installation"
|
||||||
|
}
|
||||||
|
|
||||||
|
$releaseDocument = [ordered]@{
|
||||||
|
schema_version = "missioncore.m49-tgs-portable-executor-installed-candidate/v1"
|
||||||
|
release_id = $ReleaseId
|
||||||
|
state = "installed-candidate-blocked"
|
||||||
|
worker_id = $WorkerId
|
||||||
|
candidate_sha256 = $CandidateId
|
||||||
|
source_revision = [string]$manifest.source_revision
|
||||||
|
source_state = [string]$manifest.source_state
|
||||||
|
source_archive_sha256 = $ArchiveSha256
|
||||||
|
base_image_sha256 = $BaseImageId.Substring(7)
|
||||||
|
executor_image = [ordered]@{
|
||||||
|
tag = $imageTag
|
||||||
|
sha256 = ([string]$image[0].Id).Substring(7)
|
||||||
|
size_bytes = [long]$image[0].Size
|
||||||
|
}
|
||||||
|
compiled_runner = [ordered]@{
|
||||||
|
relative_path = "installed/run_m49_tgs_portable"
|
||||||
|
byte_length = [long]$binaryLength
|
||||||
|
sha256 = $binarySha256
|
||||||
|
}
|
||||||
|
compiled_runner_build_seal = [ordered]@{
|
||||||
|
relative_path = "installed/compiled-runner-build-candidate.json"
|
||||||
|
sha256 = $buildSealSha256
|
||||||
|
}
|
||||||
|
profile = [ordered]@{
|
||||||
|
relative_path = "installed/m49-tgs-portable-v2.json"
|
||||||
|
sha256 = $ProfileSha256
|
||||||
|
}
|
||||||
|
result_contract_sha256 = $ResultContractSha256
|
||||||
|
fixture_smoke = [ordered]@{
|
||||||
|
source = "pinned TRAVEL KITTI fixture"
|
||||||
|
frame_count = 2
|
||||||
|
points_per_frame = 124668
|
||||||
|
network = "none"
|
||||||
|
result = "passed"
|
||||||
|
}
|
||||||
|
blockers = @("committed-source-snapshot-missing")
|
||||||
|
authority = $Authority
|
||||||
|
}
|
||||||
|
$releasePath = Join-Path $installed "executor-release-candidate.json"
|
||||||
|
Write-Utf8NoBom $releasePath (($releaseDocument | ConvertTo-Json -Depth 10 -Compress) + "`n")
|
||||||
|
$releaseSha256 = Get-Sha256 $releasePath
|
||||||
|
|
||||||
|
$protectedIdentities = @()
|
||||||
|
foreach ($row in $protectedAfter) {
|
||||||
|
$protectedIdentities += [ordered]@{
|
||||||
|
name = ([string]$row.Name).TrimStart("/")
|
||||||
|
container_id = [string]$row.Id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$receipt = [ordered]@{
|
||||||
|
schema_version = "missioncore.m49-tgs-portable-worker-installation-receipt/v1"
|
||||||
|
receipt_state = "installed-candidate-blocked"
|
||||||
|
worker_id = $WorkerId
|
||||||
|
computer_name = $env:COMPUTERNAME
|
||||||
|
installed_at_utc = [DateTimeOffset]::UtcNow.ToString("o")
|
||||||
|
release_id = $ReleaseId
|
||||||
|
release_sha256 = $releaseSha256
|
||||||
|
release_root = $releaseRoot
|
||||||
|
executor_image_sha256 = ([string]$image[0].Id).Substring(7)
|
||||||
|
compiled_runner_sha256 = $binarySha256
|
||||||
|
compiled_runner_build_seal_sha256 = $buildSealSha256
|
||||||
|
fixture_smoke = "passed"
|
||||||
|
protected_runtime = $protectedIdentities
|
||||||
|
legacy_m49_task_state = if ($null -eq $legacyTaskAfter) { $null } else { [string]$legacyTaskAfter.State }
|
||||||
|
blockers = @("committed-source-snapshot-missing")
|
||||||
|
authority = $Authority
|
||||||
|
}
|
||||||
|
$receiptPath = Join-Path $installed "worker-installation-receipt.json"
|
||||||
|
Write-Utf8NoBom $receiptPath (($receipt | ConvertTo-Json -Depth 10 -Compress) + "`n")
|
||||||
|
$receiptSha256 = Get-Sha256 $receiptPath
|
||||||
|
|
||||||
|
[pscustomobject]@{
|
||||||
|
ok = $true
|
||||||
|
receipt_state = "installed-candidate-blocked"
|
||||||
|
worker_id = $WorkerId
|
||||||
|
candidate_sha256 = $CandidateId
|
||||||
|
release_id = $ReleaseId
|
||||||
|
release_sha256 = $releaseSha256
|
||||||
|
executor_image_tag = $imageTag
|
||||||
|
executor_image_sha256 = ([string]$image[0].Id).Substring(7)
|
||||||
|
compiled_runner_sha256 = $binarySha256
|
||||||
|
compiled_runner_build_seal_sha256 = $buildSealSha256
|
||||||
|
worker_installation_receipt_sha256 = $receiptSha256
|
||||||
|
fixture_smoke = "passed"
|
||||||
|
blocker = "committed-source-snapshot-missing"
|
||||||
|
release_root = $releaseRoot
|
||||||
|
} | ConvertTo-Json -Compress
|
||||||
+218
@@ -0,0 +1,218 @@
|
|||||||
|
{
|
||||||
|
"assets": [
|
||||||
|
{
|
||||||
|
"asset_id": "ddrnet-checkpoint",
|
||||||
|
"byte_length": 259419077,
|
||||||
|
"kind": "model-artifact",
|
||||||
|
"repository_path": null,
|
||||||
|
"sha256": "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "ddrnet-goose-image",
|
||||||
|
"byte_length": null,
|
||||||
|
"kind": "container-image",
|
||||||
|
"repository_path": null,
|
||||||
|
"sha256": "591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "ddrnet-goose-mapping",
|
||||||
|
"byte_length": null,
|
||||||
|
"kind": "runtime-artifact",
|
||||||
|
"repository_path": null,
|
||||||
|
"sha256": "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "ddrnet-goose-runner",
|
||||||
|
"byte_length": 32877,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "experiments/perception/worker/lab_v1_vegetation_goose/run_goose_vegetation_benchmark.py",
|
||||||
|
"sha256": "b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "ddrnet-image-dockerfile",
|
||||||
|
"byte_length": 1793,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "experiments/perception/worker/lab_v1_vegetation_goose/Dockerfile",
|
||||||
|
"sha256": "8203fd01e05d8f5bcce11c328dd39dbb6b97df54ca5d9690706536bafe3d3bad"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "ddrnet-portable-config",
|
||||||
|
"byte_length": 4324,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "config/perception/lab-v1-eomt-ddrnet-portable-v2.json",
|
||||||
|
"sha256": "c0ff657dadc86607d77d232e84d041fbf2d8b63e86d02319e9cd607220d00f21"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-config-json",
|
||||||
|
"byte_length": 1575,
|
||||||
|
"kind": "model-artifact",
|
||||||
|
"repository_path": null,
|
||||||
|
"sha256": "7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-dependency-set",
|
||||||
|
"byte_length": null,
|
||||||
|
"kind": "definition-component",
|
||||||
|
"repository_path": null,
|
||||||
|
"sha256": "4eb1f8d33236806e74f9e5bb96b7dce2ac37623dc39b2184be2aa8d7d00e983e"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-evaluation-helper",
|
||||||
|
"byte_length": 28915,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "experiments/perception/worker/run_evaluation_prelabels.py",
|
||||||
|
"sha256": "25baf30c0df564734e08f38ace88cc4bc147cacf240c761622279511e361daa4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-image",
|
||||||
|
"byte_length": null,
|
||||||
|
"kind": "container-image",
|
||||||
|
"repository_path": null,
|
||||||
|
"sha256": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-model-weights",
|
||||||
|
"byte_length": 1276175488,
|
||||||
|
"kind": "model-artifact",
|
||||||
|
"repository_path": null,
|
||||||
|
"sha256": "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-orchestrator",
|
||||||
|
"byte_length": 21489,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "experiments/perception/worker/Invoke-E4FullSessionSegmentation.ps1",
|
||||||
|
"sha256": "d3e9435939444ab35b27a744ac314e289ebd66a13fa56e3d59f121e088d22774"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-preprocessor-config",
|
||||||
|
"byte_length": 666,
|
||||||
|
"kind": "model-artifact",
|
||||||
|
"repository_path": null,
|
||||||
|
"sha256": "97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-profile",
|
||||||
|
"byte_length": 3805,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "experiments/perception/worker/e3_k1_camera1_profile.json",
|
||||||
|
"sha256": "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-profile-runtime",
|
||||||
|
"byte_length": 45789,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "experiments/perception/worker/run_e3_rectified_segmentation.py",
|
||||||
|
"sha256": "01881862d4eaa218955f776a948124bf19c34be2b5ec282115daeacb15c53ae6"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-recorded-runtime",
|
||||||
|
"byte_length": 34899,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "experiments/perception/worker/run_recorded_perception_epoch.py",
|
||||||
|
"sha256": "4dcc4fc8bdf33702651a199be69d0dd4fadb243d2e65aee1c3d1ae7a58fdf675"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "eomt-runner",
|
||||||
|
"byte_length": 30720,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "experiments/perception/worker/run_e4_full_session_segmentation.py",
|
||||||
|
"sha256": "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "k1-calibration",
|
||||||
|
"byte_length": null,
|
||||||
|
"kind": "definition-component",
|
||||||
|
"repository_path": null,
|
||||||
|
"sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "k1-valid-fov-identity",
|
||||||
|
"byte_length": null,
|
||||||
|
"kind": "definition-component",
|
||||||
|
"repository_path": null,
|
||||||
|
"sha256": "b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "k1-valid-fov-mask",
|
||||||
|
"byte_length": null,
|
||||||
|
"kind": "definition-component",
|
||||||
|
"repository_path": null,
|
||||||
|
"sha256": "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "lab-v1-portable-contracts",
|
||||||
|
"byte_length": 114905,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "src/k1link/observatory/portable_lab_v1_executor.py",
|
||||||
|
"sha256": "c8ba210336138617a1cdb02ed2795935b25a265ae7b905f75cd0338774338c55"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "lab-v1-portable-worker",
|
||||||
|
"byte_length": 46859,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "src/k1link/observatory/portable_lab_v1_worker.py",
|
||||||
|
"sha256": "ad00418d59b793328a281fa432e450919690fe4957f024f296e2d125c1b3c03a"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "portable-result-contracts",
|
||||||
|
"byte_length": 22008,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "src/k1link/observatory/portable_result_contract.py",
|
||||||
|
"sha256": "936d6f20789c26e9bed1c9e34ee51259368eb95bc7e3aa549821d1f7de79030c"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "portable-worker-runtime",
|
||||||
|
"byte_length": 39120,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "src/k1link/observatory/portable_worker_runtime.py",
|
||||||
|
"sha256": "205d32116f1ba75d5257d15cc79b5146574d93acd52028a5950f6fca4d8129cb"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "vegetation-policy",
|
||||||
|
"byte_length": 3022,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "config/perception/lab-v1-vegetation-mission-policy-v1.json",
|
||||||
|
"sha256": "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"asset_id": "vegetation-provider-map",
|
||||||
|
"byte_length": 2756,
|
||||||
|
"kind": "repository-file",
|
||||||
|
"repository_path": "config/perception/lab-v1-vegetation-provider-label-map-v1.json",
|
||||||
|
"sha256": "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"authority": {
|
||||||
|
"actuation_allowed": false,
|
||||||
|
"commands_enabled": false,
|
||||||
|
"navigation_or_safety_accepted": false,
|
||||||
|
"production_accepted": false
|
||||||
|
},
|
||||||
|
"candidate_sha256": "6e7adbf25acbc3a9daedf8671ee57a7c00043ea73897bc9abf1e9947eaa5d0bd",
|
||||||
|
"declared_blockers": [
|
||||||
|
"combined-executor-entrypoint-uninstalled",
|
||||||
|
"combined-executor-image-unsealed",
|
||||||
|
"commit-bound-source-unavailable",
|
||||||
|
"ddrnet-component-port-uninstalled",
|
||||||
|
"eomt-component-port-uninstalled",
|
||||||
|
"fixture-smoke-unaccepted",
|
||||||
|
"worker-installation-receipt-unavailable"
|
||||||
|
],
|
||||||
|
"definition_id": "lab-v1-eomt-ddrnet-portable",
|
||||||
|
"definition_sha256": "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2",
|
||||||
|
"definition_version": 2,
|
||||||
|
"executor_image_sha256": null,
|
||||||
|
"phases": [
|
||||||
|
"source-materialization",
|
||||||
|
"eomt-full-session",
|
||||||
|
"ddrnet-full-session",
|
||||||
|
"result-v2-assembly",
|
||||||
|
"result-v2-validation",
|
||||||
|
"portable-result-packaging"
|
||||||
|
],
|
||||||
|
"release_id": "lab-v1-eomt-ddrnet-worker006-candidate-v2",
|
||||||
|
"result_contract_sha256": "b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a",
|
||||||
|
"schema_version": "missioncore.observatory-portable-lab-v1-executor-candidate/v1",
|
||||||
|
"setup_id": "lab-v1-eomt-ddrnet-portable-v1"
|
||||||
|
}
|
||||||
+51
@@ -0,0 +1,51 @@
|
|||||||
|
{
|
||||||
|
"schema_version": "missioncore.m49-tgs-portable-runner-source/v1",
|
||||||
|
"release_id": "m49-tgs-portable-runner-source-v1",
|
||||||
|
"worker_contour_id": "worker-006",
|
||||||
|
"container_image_sha256": "7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f",
|
||||||
|
"profile_sha256": "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9",
|
||||||
|
"input_contract": {
|
||||||
|
"schema_version": "missioncore.m49-tgs-portable-source-stage/v1",
|
||||||
|
"sequence": "ordered-kitti-xyzi-float32-files/v1",
|
||||||
|
"schedule_columns": [
|
||||||
|
"timeline_frame_index",
|
||||||
|
"source_frame_index",
|
||||||
|
"session_seconds",
|
||||||
|
"available_slot",
|
||||||
|
"point_count"
|
||||||
|
],
|
||||||
|
"timeline_frame_count": "source-derived",
|
||||||
|
"available_lidar_frame_count": "source-derived",
|
||||||
|
"missing_lidar_policy": "all-cells-unobserved"
|
||||||
|
},
|
||||||
|
"intermediate_result_contract": {
|
||||||
|
"schema_version": "missioncore.m49-tgs-portable-intermediate/v1",
|
||||||
|
"outputs": "ground-and-nonground-xyzi-by-timeline-frame",
|
||||||
|
"timing": "one-row-per-timeline-frame",
|
||||||
|
"publication_ready": false
|
||||||
|
},
|
||||||
|
"files": [
|
||||||
|
{
|
||||||
|
"name": "run_m49_tgs_portable.cpp",
|
||||||
|
"byte_length": 11052,
|
||||||
|
"sha256": "52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "run_m49_tgs_portable.sh",
|
||||||
|
"byte_length": 858,
|
||||||
|
"sha256": "2d6c32560682647f868e4ce4c2605749f17c60482a609f8c03ff951411f48ffb"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "smoke_m49_tgs_portable.sh",
|
||||||
|
"byte_length": 1117,
|
||||||
|
"sha256": "e5da73c2cf89ed0671de8ac52e13014a7faf79ce1ff1f97a090154063a61c18d"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"authority": {
|
||||||
|
"commands_enabled": false,
|
||||||
|
"actuation_allowed": false,
|
||||||
|
"navigation_or_safety_accepted": false,
|
||||||
|
"production_accepted": false
|
||||||
|
},
|
||||||
|
"source_release_sha256": "50818f50892ded83285a0fe2875066e32f2c5a13aa8af6c6ce55f63eef863569"
|
||||||
|
}
|
||||||
@@ -0,0 +1,266 @@
|
|||||||
|
#include <chrono>
|
||||||
|
#include <cmath>
|
||||||
|
#include <filesystem>
|
||||||
|
#include <fstream>
|
||||||
|
#include <iomanip>
|
||||||
|
#include <iostream>
|
||||||
|
#include <limits>
|
||||||
|
#include <memory>
|
||||||
|
#include <sstream>
|
||||||
|
#include <stdexcept>
|
||||||
|
#include <string>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "travel/point_types.hpp"
|
||||||
|
#include "travel/tgs.hpp"
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
using Clock = std::chrono::steady_clock;
|
||||||
|
|
||||||
|
struct ScheduleRow {
|
||||||
|
std::size_t timeline_frame_index;
|
||||||
|
long long source_frame_index;
|
||||||
|
double session_seconds;
|
||||||
|
long long available_slot;
|
||||||
|
std::size_t point_count;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct Schedule {
|
||||||
|
std::vector<ScheduleRow> rows;
|
||||||
|
std::size_t available_count;
|
||||||
|
};
|
||||||
|
|
||||||
|
Schedule readSchedule(const std::string& path) {
|
||||||
|
std::ifstream input(path);
|
||||||
|
if (!input) {
|
||||||
|
throw std::runtime_error("cannot open portable TGS schedule");
|
||||||
|
}
|
||||||
|
std::string line;
|
||||||
|
std::getline(input, line);
|
||||||
|
if (line !=
|
||||||
|
"timeline_frame_index\tsource_frame_index\tsession_seconds\tavailable_slot\tpoint_count") {
|
||||||
|
throw std::runtime_error("portable TGS schedule header changed");
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<ScheduleRow> rows;
|
||||||
|
std::size_t available_count = 0;
|
||||||
|
long long previous_source_frame = -1;
|
||||||
|
double previous_session_seconds = -1.0;
|
||||||
|
while (std::getline(input, line)) {
|
||||||
|
if (line.empty()) {
|
||||||
|
throw std::runtime_error("portable TGS schedule contains an empty row");
|
||||||
|
}
|
||||||
|
std::istringstream stream(line);
|
||||||
|
ScheduleRow row{};
|
||||||
|
std::string trailing;
|
||||||
|
if (!(stream >> row.timeline_frame_index >> row.source_frame_index >> row.session_seconds >>
|
||||||
|
row.available_slot >> row.point_count) ||
|
||||||
|
(stream >> trailing)) {
|
||||||
|
throw std::runtime_error("portable TGS schedule row is invalid");
|
||||||
|
}
|
||||||
|
if (row.timeline_frame_index != rows.size() || row.source_frame_index < 0 ||
|
||||||
|
row.source_frame_index <= previous_source_frame || !std::isfinite(row.session_seconds) ||
|
||||||
|
row.session_seconds <= previous_session_seconds) {
|
||||||
|
throw std::runtime_error("portable TGS schedule order changed");
|
||||||
|
}
|
||||||
|
if (row.available_slot == -1) {
|
||||||
|
if (row.point_count != 0) {
|
||||||
|
throw std::runtime_error("missing portable TGS frame contains points");
|
||||||
|
}
|
||||||
|
} else if (row.available_slot < 0 ||
|
||||||
|
static_cast<std::size_t>(row.available_slot) != available_count ||
|
||||||
|
row.point_count == 0) {
|
||||||
|
throw std::runtime_error("portable TGS available slot order changed");
|
||||||
|
} else {
|
||||||
|
++available_count;
|
||||||
|
}
|
||||||
|
rows.push_back(row);
|
||||||
|
previous_source_frame = row.source_frame_index;
|
||||||
|
previous_session_seconds = row.session_seconds;
|
||||||
|
}
|
||||||
|
if (rows.size() < 2 || available_count == 0) {
|
||||||
|
throw std::runtime_error("portable TGS schedule is incomplete");
|
||||||
|
}
|
||||||
|
return Schedule{std::move(rows), available_count};
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string framePath(const std::string& directory, std::size_t slot) {
|
||||||
|
std::ostringstream name;
|
||||||
|
name << directory << "/" << std::setw(6) << std::setfill('0') << slot << ".bin";
|
||||||
|
return name.str();
|
||||||
|
}
|
||||||
|
|
||||||
|
void verifySequence(const std::string& directory, std::size_t expected_count) {
|
||||||
|
if (!std::filesystem::is_directory(directory)) {
|
||||||
|
throw std::runtime_error("portable TGS sequence directory is unavailable");
|
||||||
|
}
|
||||||
|
std::size_t actual_count = 0;
|
||||||
|
for (const auto& entry : std::filesystem::directory_iterator(directory)) {
|
||||||
|
if (!entry.is_regular_file() || entry.path().extension() != ".bin") {
|
||||||
|
throw std::runtime_error("portable TGS sequence contains an unexpected member");
|
||||||
|
}
|
||||||
|
++actual_count;
|
||||||
|
}
|
||||||
|
if (actual_count != expected_count) {
|
||||||
|
throw std::runtime_error("portable TGS sequence and schedule differ");
|
||||||
|
}
|
||||||
|
for (std::size_t slot = 0; slot < expected_count; ++slot) {
|
||||||
|
if (!std::filesystem::is_regular_file(framePath(directory, slot))) {
|
||||||
|
throw std::runtime_error("portable TGS sequence order changed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
travel::PointCloud<travel::PointXYZI>::Ptr readXYZI(
|
||||||
|
const std::string& directory,
|
||||||
|
std::size_t slot,
|
||||||
|
std::size_t point_count) {
|
||||||
|
if (point_count > std::numeric_limits<std::size_t>::max() / (4 * sizeof(float))) {
|
||||||
|
throw std::runtime_error("portable TGS point count exceeds the addressable bound");
|
||||||
|
}
|
||||||
|
const auto path = framePath(directory, slot);
|
||||||
|
const auto expected_bytes = point_count * 4 * sizeof(float);
|
||||||
|
if (std::filesystem::file_size(path) != expected_bytes) {
|
||||||
|
throw std::runtime_error("portable TGS input byte count changed");
|
||||||
|
}
|
||||||
|
std::ifstream input(path, std::ios::binary);
|
||||||
|
if (!input) {
|
||||||
|
throw std::runtime_error("cannot open portable TGS input");
|
||||||
|
}
|
||||||
|
std::vector<float> values(point_count * 4);
|
||||||
|
input.read(
|
||||||
|
reinterpret_cast<char*>(values.data()),
|
||||||
|
static_cast<std::streamsize>(expected_bytes));
|
||||||
|
if (!input || input.peek() != std::ifstream::traits_type::eof()) {
|
||||||
|
throw std::runtime_error("cannot read exact portable TGS input");
|
||||||
|
}
|
||||||
|
auto cloud = std::make_shared<travel::PointCloud<travel::PointXYZI>>();
|
||||||
|
cloud->resize(point_count);
|
||||||
|
for (std::size_t index = 0; index < point_count; ++index) {
|
||||||
|
auto& point = cloud->at(index);
|
||||||
|
point.x = values[index * 4];
|
||||||
|
point.y = values[index * 4 + 1];
|
||||||
|
point.z = values[index * 4 + 2];
|
||||||
|
point.intensity = values[index * 4 + 3];
|
||||||
|
if (!std::isfinite(point.x) || !std::isfinite(point.y) ||
|
||||||
|
!std::isfinite(point.z) || !std::isfinite(point.intensity)) {
|
||||||
|
throw std::runtime_error("portable TGS input contains a non-finite value");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cloud;
|
||||||
|
}
|
||||||
|
|
||||||
|
void writeXYZI(const std::string& path, const travel::PointCloud<PointXYZILID>& cloud) {
|
||||||
|
std::ofstream output(path, std::ios::binary);
|
||||||
|
if (!output) {
|
||||||
|
throw std::runtime_error("cannot open portable TGS output");
|
||||||
|
}
|
||||||
|
for (const auto& point : cloud.points) {
|
||||||
|
const float row[4] = {point.x, point.y, point.z, point.intensity};
|
||||||
|
output.write(reinterpret_cast<const char*>(row), sizeof(row));
|
||||||
|
}
|
||||||
|
if (!output) {
|
||||||
|
throw std::runtime_error("cannot write portable TGS output");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
double milliseconds(Clock::duration duration) {
|
||||||
|
return std::chrono::duration<double, std::milli>(duration).count();
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
int main(int argc, char** argv) {
|
||||||
|
if (argc != 5) {
|
||||||
|
std::cerr << "Usage: run_m49_tgs_portable <sequence_dir> <schedule.tsv> <output_dir>"
|
||||||
|
" <timing.tsv>\n";
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const std::string sequence_dir = argv[1];
|
||||||
|
const std::string schedule_path = argv[2];
|
||||||
|
const std::string output_dir = argv[3];
|
||||||
|
const std::string timing_path = argv[4];
|
||||||
|
const auto schedule = readSchedule(schedule_path);
|
||||||
|
verifySequence(sequence_dir, schedule.available_count);
|
||||||
|
if (std::filesystem::exists(output_dir) || std::filesystem::exists(timing_path)) {
|
||||||
|
throw std::runtime_error("portable TGS output already exists");
|
||||||
|
}
|
||||||
|
std::filesystem::create_directories(output_dir);
|
||||||
|
std::ofstream timing(timing_path);
|
||||||
|
if (!timing) {
|
||||||
|
throw std::runtime_error("cannot open portable TGS timing output");
|
||||||
|
}
|
||||||
|
timing << "timeline_frame_index\tsource_frame_index\tsession_seconds\tsample_available"
|
||||||
|
<< "\tavailable_slot\tinput_points\tground_points\tnonground_points\ttgs_ms"
|
||||||
|
<< "\tstage_wall_ms\n";
|
||||||
|
timing << std::fixed << std::setprecision(6);
|
||||||
|
|
||||||
|
std::size_t completed_slots = 0;
|
||||||
|
for (const auto& row : schedule.rows) {
|
||||||
|
const auto stage_started = Clock::now();
|
||||||
|
std::size_t input_points = 0;
|
||||||
|
std::size_t ground_points = 0;
|
||||||
|
std::size_t nonground_points = 0;
|
||||||
|
double tgs_seconds = 0.0;
|
||||||
|
if (row.available_slot >= 0) {
|
||||||
|
const auto slot = static_cast<std::size_t>(row.available_slot);
|
||||||
|
auto input_xyzi = readXYZI(sequence_dir, slot, row.point_count);
|
||||||
|
if (input_xyzi->size() != row.point_count) {
|
||||||
|
throw std::runtime_error("portable TGS input point count changed");
|
||||||
|
}
|
||||||
|
auto input = std::make_shared<travel::PointCloud<PointXYZILID>>();
|
||||||
|
input->reserve(input_xyzi->size());
|
||||||
|
for (const auto& point : input_xyzi->points) {
|
||||||
|
PointXYZILID value{};
|
||||||
|
value.x = point.x;
|
||||||
|
value.y = point.y;
|
||||||
|
value.z = point.z;
|
||||||
|
value.intensity = point.intensity;
|
||||||
|
value.label = 0;
|
||||||
|
value.id = 0;
|
||||||
|
input->emplace_back(value);
|
||||||
|
}
|
||||||
|
travel::TravelGroundSeg<PointXYZILID> tgs;
|
||||||
|
tgs.setParams(
|
||||||
|
80.0, 1.0, 8.0, 3, 5, 10, 0.5, 0.125, 0.3, 0.940, 200.0, 0.03,
|
||||||
|
0.1, 1.0, true, false);
|
||||||
|
travel::PointCloud<PointXYZILID> ground;
|
||||||
|
travel::PointCloud<PointXYZILID> nonground;
|
||||||
|
tgs.estimateGround(*input, ground, nonground, tgs_seconds);
|
||||||
|
input_points = input->size();
|
||||||
|
ground_points = ground.size();
|
||||||
|
nonground_points = nonground.size();
|
||||||
|
const std::string base = output_dir + "/" +
|
||||||
|
std::to_string(row.timeline_frame_index);
|
||||||
|
writeXYZI(base + "_ground.bin", ground);
|
||||||
|
writeXYZI(base + "_nonground.bin", nonground);
|
||||||
|
++completed_slots;
|
||||||
|
}
|
||||||
|
const auto completed = Clock::now();
|
||||||
|
timing << row.timeline_frame_index << '\t' << row.source_frame_index << '\t'
|
||||||
|
<< row.session_seconds << '\t' << (row.available_slot >= 0 ? 1 : 0) << '\t'
|
||||||
|
<< row.available_slot << '\t' << input_points << '\t' << ground_points << '\t'
|
||||||
|
<< nonground_points << '\t' << (tgs_seconds * 1000.0) << '\t'
|
||||||
|
<< milliseconds(completed - stage_started) << '\n';
|
||||||
|
if ((row.timeline_frame_index + 1) % 100 == 0) {
|
||||||
|
timing.flush();
|
||||||
|
std::cout << "[TGS-PORTABLE] frame=" << (row.timeline_frame_index + 1) << "/"
|
||||||
|
<< schedule.rows.size() << " available=" << completed_slots << "/"
|
||||||
|
<< schedule.available_count << "\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
timing.flush();
|
||||||
|
if (completed_slots != schedule.available_count) {
|
||||||
|
throw std::runtime_error("portable TGS available frame accounting changed");
|
||||||
|
}
|
||||||
|
std::cout << "[TGS-PORTABLE] complete timeline=" << schedule.rows.size()
|
||||||
|
<< " available=" << completed_slots << "\n";
|
||||||
|
return 0;
|
||||||
|
} catch (const std::exception& error) {
|
||||||
|
std::cerr << "[TGS-PORTABLE] " << error.what() << '\n';
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
readonly PROFILE=/release/m49-tgs-portable-v2.json
|
||||||
|
readonly PROFILE_SHA256=6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9
|
||||||
|
readonly SOURCE=/release/run_m49_tgs_portable.cpp
|
||||||
|
readonly INPUT_ROOT=/work/source/tgs
|
||||||
|
readonly OUTPUT_ROOT=/work/result
|
||||||
|
readonly BINARY=/tmp/run_m49_tgs_portable
|
||||||
|
|
||||||
|
test -f "${PROFILE}"
|
||||||
|
test -f "${SOURCE}"
|
||||||
|
test -f "${INPUT_ROOT}/schedule.tsv"
|
||||||
|
test -d "${INPUT_ROOT}/sequence"
|
||||||
|
test ! -e "${OUTPUT_ROOT}"
|
||||||
|
echo "${PROFILE_SHA256} ${PROFILE}" | sha256sum --check --strict
|
||||||
|
|
||||||
|
g++ -std=c++17 -O3 -DNDEBUG -pthread \
|
||||||
|
-I/opt/travel/src/TRAVEL/cpp/travel/core \
|
||||||
|
-I/usr/include/eigen3 \
|
||||||
|
"${SOURCE}" \
|
||||||
|
-o "${BINARY}"
|
||||||
|
|
||||||
|
exec /usr/bin/time -v "${BINARY}" \
|
||||||
|
"${INPUT_ROOT}/sequence/velodyne" \
|
||||||
|
"${INPUT_ROOT}/schedule.tsv" \
|
||||||
|
"${OUTPUT_ROOT}/outputs" \
|
||||||
|
"${OUTPUT_ROOT}/timing.tsv"
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
readonly SMOKE_ROOT=/smoke
|
||||||
|
readonly FIXTURE=/opt/travel/fixture/00/velodyne/000000.bin
|
||||||
|
readonly RUNNER=/opt/nodedc/m49-tgs-portable/bin/run_m49_tgs_portable
|
||||||
|
readonly EXPECTED_FRAME_BYTES=1994688
|
||||||
|
readonly EXPECTED_POINTS=124668
|
||||||
|
|
||||||
|
mkdir -p "${SMOKE_ROOT}/sequence"
|
||||||
|
cp "${FIXTURE}" "${SMOKE_ROOT}/sequence/000000.bin"
|
||||||
|
cp "${FIXTURE}" "${SMOKE_ROOT}/sequence/000001.bin"
|
||||||
|
printf 'timeline_frame_index\tsource_frame_index\tsession_seconds\tavailable_slot\tpoint_count\n0\t0\t0.0\t0\t124668\n1\t1\t0.1\t1\t124668\n' \
|
||||||
|
> "${SMOKE_ROOT}/schedule.tsv"
|
||||||
|
|
||||||
|
"${RUNNER}" \
|
||||||
|
"${SMOKE_ROOT}/sequence" \
|
||||||
|
"${SMOKE_ROOT}/schedule.tsv" \
|
||||||
|
"${SMOKE_ROOT}/outputs" \
|
||||||
|
"${SMOKE_ROOT}/timing.tsv"
|
||||||
|
|
||||||
|
test "$(wc -l < "${SMOKE_ROOT}/timing.tsv")" -eq 3
|
||||||
|
for index in 0 1; do
|
||||||
|
ground_bytes="$(stat -c %s "${SMOKE_ROOT}/outputs/${index}_ground.bin")"
|
||||||
|
nonground_bytes="$(stat -c %s "${SMOKE_ROOT}/outputs/${index}_nonground.bin")"
|
||||||
|
test "$((ground_bytes + nonground_bytes))" -eq "${EXPECTED_FRAME_BYTES}"
|
||||||
|
done
|
||||||
|
|
||||||
|
printf 'M49_PORTABLE_FIXTURE_SMOKE_OK frames=2 points_per_frame=%s\n' "${EXPECTED_POINTS}"
|
||||||
@@ -0,0 +1,588 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Build and verify a deterministic blocked M4.9 executor release candidate.
|
||||||
|
|
||||||
|
This builder deliberately cannot mark the executor ready. It seals the exact
|
||||||
|
portable source materializer, generic TRAVEL/TGS runner, result-v2 assembler
|
||||||
|
and validator into a reproducible candidate archive. A later installation
|
||||||
|
step must additionally provide an exact compiled runner, an executor image and
|
||||||
|
an installation receipt before the runtime registry can become ready.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
from typing import Final, cast
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
M49_EXECUTOR_RELEASE_CANDIDATE_SCHEMA: Final = (
|
||||||
|
"missioncore.m49-tgs-portable-executor-release-candidate/v1"
|
||||||
|
)
|
||||||
|
M49_COMPILED_RUNNER_BUILD_SCHEMA: Final = "missioncore.m49-tgs-portable-compiled-runner-build/v1"
|
||||||
|
M49_EXECUTOR_RELEASE_ID: Final = "m49-tgs-portable-executor-v1"
|
||||||
|
M49_TRAVEL_IMAGE_SHA256: Final = "7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f"
|
||||||
|
M49_PROFILE_SHA256: Final = "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9"
|
||||||
|
M49_RESULT_CONTRACT_SHA256: Final = (
|
||||||
|
"9dd80c8e2504559d2156fca933de6eb27901e35305e6853aeb84707e1cb13892"
|
||||||
|
)
|
||||||
|
M49_RUNNER_SOURCE_SHA256: Final = "52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9"
|
||||||
|
M49_RUNNER_WRAPPER_SHA256: Final = (
|
||||||
|
"2d6c32560682647f868e4ce4c2605749f17c60482a609f8c03ff951411f48ffb"
|
||||||
|
)
|
||||||
|
M49_COMPILER_CONTRACT: Final = {
|
||||||
|
"compiler": "g++",
|
||||||
|
"language_standard": "c++17",
|
||||||
|
"flags": ["-O3", "-DNDEBUG", "-pthread"],
|
||||||
|
"travel_include": "/opt/travel/src/TRAVEL/cpp/travel/core",
|
||||||
|
"eigen_include": "/usr/include/eigen3",
|
||||||
|
}
|
||||||
|
M49_RELEASE_BLOCKERS: Final = (
|
||||||
|
"compiled-runner-artifact-missing",
|
||||||
|
"exact-executor-image-missing",
|
||||||
|
"worker-installation-receipt-missing",
|
||||||
|
)
|
||||||
|
M49_RELEASE_SOURCES: Final = (
|
||||||
|
Path("config/perception/m49-tgs-portable-v2.json"),
|
||||||
|
Path("experiments/perception/worker/observatory_portable/Dockerfile.m49-portable-executor"),
|
||||||
|
Path(
|
||||||
|
"experiments/perception/worker/observatory_portable/"
|
||||||
|
"Invoke-M49PortableExecutorCandidateInstall.ps1"
|
||||||
|
),
|
||||||
|
Path("experiments/perception/worker/observatory_portable/m49-tgs-portable-runner-source.json"),
|
||||||
|
Path("experiments/perception/worker/observatory_portable/run_m49_tgs_portable.cpp"),
|
||||||
|
Path("experiments/perception/worker/observatory_portable/run_m49_tgs_portable.sh"),
|
||||||
|
Path("experiments/perception/worker/observatory_portable/smoke_m49_tgs_portable.sh"),
|
||||||
|
Path("experiments/perception/worker/m49_t3_travel/build_tgs_fail_closed_evidence.py"),
|
||||||
|
Path("experiments/perception/worker/m49_t3_travel/build_tgs_full_shadow_binary.sh"),
|
||||||
|
Path("experiments/perception/worker/m49_t3_travel/build_tgs_full_shadow_evidence.py"),
|
||||||
|
Path("src/k1link/compute/lidar_replay.py"),
|
||||||
|
Path("src/k1link/observatory/m49_portable_executor.py"),
|
||||||
|
Path("src/k1link/observatory/m49_portable_result.py"),
|
||||||
|
Path("src/k1link/observatory/m49_portable_source.py"),
|
||||||
|
Path("src/k1link/observatory/portable_result_contract.py"),
|
||||||
|
)
|
||||||
|
_AUTHORITY: Final = {
|
||||||
|
"commands_enabled": False,
|
||||||
|
"actuation_allowed": False,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
"production_accepted": False,
|
||||||
|
}
|
||||||
|
_REVISION: Final = re.compile(r"^[a-f0-9]{40}$")
|
||||||
|
_SHA256: Final = re.compile(r"^[a-f0-9]{64}$")
|
||||||
|
_HASH_CHUNK_BYTES: Final = 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
class M49ExecutorReleaseBuildError(RuntimeError):
|
||||||
|
"""The exact portable executor candidate cannot be built or verified."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class BuiltM49ExecutorReleaseCandidate:
|
||||||
|
archive: Path
|
||||||
|
archive_sha256: str
|
||||||
|
candidate_sha256: str
|
||||||
|
manifest: dict[str, object]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class BuiltM49CompiledRunnerSeal:
|
||||||
|
binary: Path
|
||||||
|
manifest_path: Path
|
||||||
|
manifest_sha256: str
|
||||||
|
manifest: dict[str, object]
|
||||||
|
|
||||||
|
|
||||||
|
def seal_m49_compiled_runner_build(
|
||||||
|
*,
|
||||||
|
source_root: Path,
|
||||||
|
source_revision: str,
|
||||||
|
binary_path: Path,
|
||||||
|
manifest_path: Path,
|
||||||
|
) -> BuiltM49CompiledRunnerSeal:
|
||||||
|
"""Seal, but never execute, a binary built in the exact TRAVEL image."""
|
||||||
|
|
||||||
|
if _REVISION.fullmatch(source_revision) is None:
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 compiled-runner revision is invalid")
|
||||||
|
root = source_root.expanduser().resolve(strict=True)
|
||||||
|
if root.is_symlink() or not root.is_dir():
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 compiled-runner source root is unsafe")
|
||||||
|
source = root / "experiments/perception/worker/observatory_portable/run_m49_tgs_portable.cpp"
|
||||||
|
wrapper = root / "experiments/perception/worker/observatory_portable/run_m49_tgs_portable.sh"
|
||||||
|
profile = root / "config/perception/m49-tgs-portable-v2.json"
|
||||||
|
if (
|
||||||
|
_sha256_file(source) != M49_RUNNER_SOURCE_SHA256
|
||||||
|
or _sha256_file(wrapper) != M49_RUNNER_WRAPPER_SHA256
|
||||||
|
or _sha256_file(profile) != M49_PROFILE_SHA256
|
||||||
|
):
|
||||||
|
raise M49ExecutorReleaseBuildError(
|
||||||
|
"M4.9 compiled-runner sources differ from the exact release"
|
||||||
|
)
|
||||||
|
binary = binary_path.expanduser().absolute()
|
||||||
|
try:
|
||||||
|
metadata = binary.lstat()
|
||||||
|
resolved_binary = binary.resolve(strict=True)
|
||||||
|
except OSError as exc:
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 compiled runner is unavailable") from exc
|
||||||
|
if (
|
||||||
|
stat.S_ISLNK(metadata.st_mode)
|
||||||
|
or not stat.S_ISREG(metadata.st_mode)
|
||||||
|
or not os.path.samefile(binary, resolved_binary)
|
||||||
|
or not os.access(resolved_binary, os.X_OK)
|
||||||
|
or not _is_elf(resolved_binary)
|
||||||
|
):
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 compiled runner is not an executable ELF artifact")
|
||||||
|
manifest: dict[str, object] = {
|
||||||
|
"schema_version": M49_COMPILED_RUNNER_BUILD_SCHEMA,
|
||||||
|
"source_revision": source_revision,
|
||||||
|
"source_state": "committed-snapshot",
|
||||||
|
"build_image_sha256": M49_TRAVEL_IMAGE_SHA256,
|
||||||
|
"profile_sha256": M49_PROFILE_SHA256,
|
||||||
|
"runner_source_sha256": M49_RUNNER_SOURCE_SHA256,
|
||||||
|
"runner_wrapper_sha256": M49_RUNNER_WRAPPER_SHA256,
|
||||||
|
"compiler_contract": dict(M49_COMPILER_CONTRACT),
|
||||||
|
"binary": {
|
||||||
|
"file_name": "run_m49_tgs_portable",
|
||||||
|
"format": "elf",
|
||||||
|
"byte_length": resolved_binary.stat().st_size,
|
||||||
|
"sha256": _sha256_file(resolved_binary),
|
||||||
|
},
|
||||||
|
"authority": dict(_AUTHORITY),
|
||||||
|
}
|
||||||
|
payload = _canonical_json(manifest)
|
||||||
|
target = manifest_path.expanduser().absolute()
|
||||||
|
if target.exists():
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 compiled-runner seal already exists")
|
||||||
|
target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
if target.parent.is_symlink() or not target.parent.is_dir():
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 compiled-runner seal root is unsafe")
|
||||||
|
target.write_bytes(payload)
|
||||||
|
return BuiltM49CompiledRunnerSeal(
|
||||||
|
binary=resolved_binary,
|
||||||
|
manifest_path=target,
|
||||||
|
manifest_sha256=hashlib.sha256(payload).hexdigest(),
|
||||||
|
manifest=manifest,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def git_revision(repository_root: Path = REPOSITORY_ROOT) -> str:
|
||||||
|
completed = subprocess.run(
|
||||||
|
["git", "rev-parse", "HEAD"],
|
||||||
|
cwd=repository_root,
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
revision = completed.stdout.strip()
|
||||||
|
if completed.returncode != 0 or _REVISION.fullmatch(revision) is None:
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release revision is unavailable")
|
||||||
|
return revision
|
||||||
|
|
||||||
|
|
||||||
|
def materialize_revision(
|
||||||
|
*, revision: str, destination: Path, repository_root: Path = REPOSITORY_ROOT
|
||||||
|
) -> None:
|
||||||
|
"""Extract only the explicit release source set from one exact commit."""
|
||||||
|
|
||||||
|
if _REVISION.fullmatch(revision) is None or destination.exists():
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release revision request is invalid")
|
||||||
|
verified = subprocess.run(
|
||||||
|
["git", "rev-parse", "--verify", f"{revision}^{{commit}}"],
|
||||||
|
cwd=repository_root,
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if verified.returncode != 0 or verified.stdout.strip() != revision:
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release revision is not a commit")
|
||||||
|
archive_path = destination.parent / "source.tar"
|
||||||
|
archived = subprocess.run(
|
||||||
|
[
|
||||||
|
"git",
|
||||||
|
"archive",
|
||||||
|
"--format=tar",
|
||||||
|
"--output",
|
||||||
|
str(archive_path),
|
||||||
|
revision,
|
||||||
|
"--",
|
||||||
|
*(path.as_posix() for path in M49_RELEASE_SOURCES),
|
||||||
|
],
|
||||||
|
cwd=repository_root,
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if archived.returncode != 0:
|
||||||
|
raise M49ExecutorReleaseBuildError(
|
||||||
|
"M4.9 release sources are not all present in the selected commit"
|
||||||
|
)
|
||||||
|
destination.mkdir()
|
||||||
|
resolved_root = destination.resolve()
|
||||||
|
with tarfile.open(archive_path, "r:") as archive:
|
||||||
|
members = archive.getmembers()
|
||||||
|
for member in members:
|
||||||
|
target = (destination / member.name).resolve()
|
||||||
|
if (
|
||||||
|
target != resolved_root
|
||||||
|
and resolved_root not in target.parents
|
||||||
|
or not (member.isdir() or member.isreg())
|
||||||
|
):
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 Git archive contains an unsafe member")
|
||||||
|
for member in members:
|
||||||
|
target = destination / member.name
|
||||||
|
if member.isdir():
|
||||||
|
target.mkdir(parents=True, exist_ok=True)
|
||||||
|
continue
|
||||||
|
source = archive.extractfile(member)
|
||||||
|
if source is None:
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 Git archive member is unreadable")
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with source, target.open("wb") as output:
|
||||||
|
while chunk := source.read(_HASH_CHUNK_BYTES):
|
||||||
|
output.write(chunk)
|
||||||
|
|
||||||
|
|
||||||
|
def build_m49_executor_release_candidate(
|
||||||
|
*,
|
||||||
|
source_root: Path,
|
||||||
|
output_directory: Path,
|
||||||
|
source_revision: str,
|
||||||
|
source_state: str,
|
||||||
|
) -> BuiltM49ExecutorReleaseCandidate:
|
||||||
|
"""Build a deterministic candidate; never a ready installation artifact."""
|
||||||
|
|
||||||
|
if _REVISION.fullmatch(source_revision) is None:
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 source revision is invalid")
|
||||||
|
if source_state not in {"committed-snapshot", "uncommitted-candidate"}:
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 source state is invalid")
|
||||||
|
root = source_root.expanduser().resolve(strict=True)
|
||||||
|
if root.is_symlink() or not root.is_dir():
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 source root is unsafe")
|
||||||
|
files = _source_inventory(root)
|
||||||
|
identity: dict[str, object] = {
|
||||||
|
"schema_version": M49_EXECUTOR_RELEASE_CANDIDATE_SCHEMA,
|
||||||
|
"release_id": M49_EXECUTOR_RELEASE_ID,
|
||||||
|
"state": "blocked",
|
||||||
|
"source_revision": source_revision,
|
||||||
|
"source_state": source_state,
|
||||||
|
"worker_contour_id": "worker-006",
|
||||||
|
"travel_build_image_sha256": M49_TRAVEL_IMAGE_SHA256,
|
||||||
|
"executor_image_sha256": None,
|
||||||
|
"compiled_runner": None,
|
||||||
|
"profile_sha256": M49_PROFILE_SHA256,
|
||||||
|
"result_contract_sha256": M49_RESULT_CONTRACT_SHA256,
|
||||||
|
"source_contract": {
|
||||||
|
"camera": "exact-admitted-fmp4-members",
|
||||||
|
"spatial_replay": "exact-admitted-k1mqtt-member",
|
||||||
|
"host_time_metadata": "separate-exact-admitted-member-required",
|
||||||
|
"server_paths_or_commands_allowed": False,
|
||||||
|
},
|
||||||
|
"phases": [
|
||||||
|
{"phase_id": "source-materializer", "state": "implemented"},
|
||||||
|
{"phase_id": "travel-tgs-runner", "state": "implemented"},
|
||||||
|
{"phase_id": "result-v2-assembler", "state": "implemented"},
|
||||||
|
{"phase_id": "exact-result-validator", "state": "implemented"},
|
||||||
|
{"phase_id": "package-sealer", "state": "implemented"},
|
||||||
|
],
|
||||||
|
"files": files,
|
||||||
|
"blockers": list(M49_RELEASE_BLOCKERS),
|
||||||
|
"authority": dict(_AUTHORITY),
|
||||||
|
}
|
||||||
|
candidate_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||||
|
manifest: dict[str, object] = {
|
||||||
|
**identity,
|
||||||
|
"candidate_sha256": candidate_sha256,
|
||||||
|
}
|
||||||
|
output = output_directory.expanduser().absolute()
|
||||||
|
output.mkdir(parents=True, exist_ok=True)
|
||||||
|
if output.is_symlink() or not output.is_dir():
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release output is unsafe")
|
||||||
|
target = output / f"m49-tgs-portable-executor-{candidate_sha256}.tgz"
|
||||||
|
if target.exists():
|
||||||
|
verified = verify_m49_executor_release_candidate(target)
|
||||||
|
if verified.candidate_sha256 != candidate_sha256:
|
||||||
|
raise M49ExecutorReleaseBuildError(
|
||||||
|
"existing M4.9 release candidate has another identity"
|
||||||
|
)
|
||||||
|
return verified
|
||||||
|
with tempfile.TemporaryDirectory(prefix="m49-executor-candidate-") as temporary:
|
||||||
|
stage = Path(temporary)
|
||||||
|
payload = stage / "payload"
|
||||||
|
for source in M49_RELEASE_SOURCES:
|
||||||
|
destination = payload / source
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
destination.write_bytes((root / source).read_bytes())
|
||||||
|
(stage / "release-manifest.json").write_bytes(_canonical_json(manifest))
|
||||||
|
_write_archive(stage, target)
|
||||||
|
verified = verify_m49_executor_release_candidate(target)
|
||||||
|
if verified.candidate_sha256 != candidate_sha256:
|
||||||
|
raise M49ExecutorReleaseBuildError("built M4.9 release candidate changed identity")
|
||||||
|
return verified
|
||||||
|
|
||||||
|
|
||||||
|
def verify_m49_executor_release_candidate(
|
||||||
|
archive_path: Path,
|
||||||
|
) -> BuiltM49ExecutorReleaseCandidate:
|
||||||
|
candidate = archive_path.expanduser().resolve(strict=True)
|
||||||
|
if candidate.is_symlink() or not candidate.is_file():
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release archive is unsafe")
|
||||||
|
with tarfile.open(candidate, "r:gz") as archive:
|
||||||
|
members = archive.getmembers()
|
||||||
|
names = [member.name for member in members]
|
||||||
|
payload_names: set[str] = set()
|
||||||
|
for source in M49_RELEASE_SOURCES:
|
||||||
|
parts = PurePosixPath("payload", *source.parts)
|
||||||
|
for parent in reversed(parts.parents):
|
||||||
|
name = parent.as_posix()
|
||||||
|
if name not in {".", "payload"}:
|
||||||
|
payload_names.add(name)
|
||||||
|
payload_names.add(parts.as_posix())
|
||||||
|
expected_names = [
|
||||||
|
"release-manifest.json",
|
||||||
|
"payload",
|
||||||
|
*sorted(payload_names),
|
||||||
|
]
|
||||||
|
if names != expected_names:
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release archive member set changed")
|
||||||
|
by_name = {member.name: member for member in members}
|
||||||
|
for member in members:
|
||||||
|
path = PurePosixPath(member.name)
|
||||||
|
if (
|
||||||
|
path.is_absolute()
|
||||||
|
or any(part in {"", ".", ".."} for part in path.parts)
|
||||||
|
or not (member.isdir() or member.isreg())
|
||||||
|
or member.uid != 0
|
||||||
|
or member.gid != 0
|
||||||
|
or member.mtime != 0
|
||||||
|
):
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release archive metadata is unsafe")
|
||||||
|
manifest_stream = archive.extractfile(by_name["release-manifest.json"])
|
||||||
|
if manifest_stream is None:
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release manifest is unavailable")
|
||||||
|
manifest_payload = manifest_stream.read()
|
||||||
|
try:
|
||||||
|
manifest = _object(json.loads(manifest_payload), "M4.9 release manifest")
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release manifest is invalid JSON") from exc
|
||||||
|
if manifest_payload != _canonical_json(manifest):
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release manifest is not canonical JSON")
|
||||||
|
candidate_sha256 = cast(str, manifest.pop("candidate_sha256", None))
|
||||||
|
if (
|
||||||
|
_SHA256.fullmatch(candidate_sha256 or "") is None
|
||||||
|
or manifest.get("schema_version") != M49_EXECUTOR_RELEASE_CANDIDATE_SCHEMA
|
||||||
|
or manifest.get("state") != "blocked"
|
||||||
|
or tuple(cast(list[object], manifest.get("blockers"))) != M49_RELEASE_BLOCKERS
|
||||||
|
or manifest.get("authority") != _AUTHORITY
|
||||||
|
or hashlib.sha256(_canonical_json(manifest)).hexdigest() != candidate_sha256
|
||||||
|
):
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release candidate identity changed")
|
||||||
|
files = cast(list[object], manifest.get("files"))
|
||||||
|
expected_files = {path.as_posix() for path in M49_RELEASE_SOURCES}
|
||||||
|
if {cast(dict[str, object], row).get("relative_path") for row in files} != expected_files:
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release file set changed")
|
||||||
|
for value in files:
|
||||||
|
row = _object(value, "M4.9 release file")
|
||||||
|
relative = cast(str, row["relative_path"])
|
||||||
|
member = by_name[f"payload/{relative}"]
|
||||||
|
stream = archive.extractfile(member)
|
||||||
|
if stream is None:
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release file is unavailable")
|
||||||
|
payload = stream.read()
|
||||||
|
if (
|
||||||
|
row.get("byte_length") != len(payload)
|
||||||
|
or row.get("sha256") != hashlib.sha256(payload).hexdigest()
|
||||||
|
):
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release file changed")
|
||||||
|
restored_manifest = {**manifest, "candidate_sha256": candidate_sha256}
|
||||||
|
return BuiltM49ExecutorReleaseCandidate(
|
||||||
|
archive=candidate,
|
||||||
|
archive_sha256=_sha256_file(candidate),
|
||||||
|
candidate_sha256=candidate_sha256,
|
||||||
|
manifest=restored_manifest,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def write_worktree_candidate_manifest(
|
||||||
|
*, source_root: Path, target: Path, source_revision: str
|
||||||
|
) -> dict[str, object]:
|
||||||
|
"""Write a reviewable manifest without pretending it is a Git release."""
|
||||||
|
|
||||||
|
files = _source_inventory(source_root.expanduser().resolve(strict=True))
|
||||||
|
identity: dict[str, object] = {
|
||||||
|
"schema_version": M49_EXECUTOR_RELEASE_CANDIDATE_SCHEMA,
|
||||||
|
"release_id": M49_EXECUTOR_RELEASE_ID,
|
||||||
|
"state": "blocked",
|
||||||
|
"source_revision": source_revision,
|
||||||
|
"source_state": "uncommitted-candidate",
|
||||||
|
"worker_contour_id": "worker-006",
|
||||||
|
"travel_build_image_sha256": M49_TRAVEL_IMAGE_SHA256,
|
||||||
|
"executor_image_sha256": None,
|
||||||
|
"compiled_runner": None,
|
||||||
|
"profile_sha256": M49_PROFILE_SHA256,
|
||||||
|
"result_contract_sha256": M49_RESULT_CONTRACT_SHA256,
|
||||||
|
"source_contract": {
|
||||||
|
"camera": "exact-admitted-fmp4-members",
|
||||||
|
"spatial_replay": "exact-admitted-k1mqtt-member",
|
||||||
|
"host_time_metadata": "separate-exact-admitted-member-required",
|
||||||
|
"server_paths_or_commands_allowed": False,
|
||||||
|
},
|
||||||
|
"phases": [
|
||||||
|
{"phase_id": "source-materializer", "state": "implemented"},
|
||||||
|
{"phase_id": "travel-tgs-runner", "state": "implemented"},
|
||||||
|
{"phase_id": "result-v2-assembler", "state": "implemented"},
|
||||||
|
{"phase_id": "exact-result-validator", "state": "implemented"},
|
||||||
|
{"phase_id": "package-sealer", "state": "implemented"},
|
||||||
|
],
|
||||||
|
"files": files,
|
||||||
|
"blockers": [
|
||||||
|
*M49_RELEASE_BLOCKERS,
|
||||||
|
"committed-source-snapshot-missing",
|
||||||
|
],
|
||||||
|
"authority": dict(_AUTHORITY),
|
||||||
|
}
|
||||||
|
manifest: dict[str, object] = {
|
||||||
|
**identity,
|
||||||
|
"candidate_sha256": hashlib.sha256(_canonical_json(identity)).hexdigest(),
|
||||||
|
}
|
||||||
|
destination = target.expanduser().absolute()
|
||||||
|
if destination.exists():
|
||||||
|
raise M49ExecutorReleaseBuildError("worktree candidate manifest already exists")
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
destination.write_bytes(_canonical_json(manifest))
|
||||||
|
return manifest
|
||||||
|
|
||||||
|
|
||||||
|
def _source_inventory(root: Path) -> list[dict[str, object]]:
|
||||||
|
rows: list[dict[str, object]] = []
|
||||||
|
for relative in M49_RELEASE_SOURCES:
|
||||||
|
path = root / relative
|
||||||
|
try:
|
||||||
|
resolved = path.resolve(strict=True)
|
||||||
|
except OSError as exc:
|
||||||
|
raise M49ExecutorReleaseBuildError(
|
||||||
|
f"M4.9 release source is unavailable: {relative}"
|
||||||
|
) from exc
|
||||||
|
if path.is_symlink() or not resolved.is_file() or not resolved.is_relative_to(root):
|
||||||
|
raise M49ExecutorReleaseBuildError(f"M4.9 release source is unsafe: {relative}")
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
"relative_path": relative.as_posix(),
|
||||||
|
"byte_length": resolved.stat().st_size,
|
||||||
|
"sha256": _sha256_file(resolved),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _write_archive(stage: Path, target: Path) -> None:
|
||||||
|
members = [stage / "release-manifest.json", stage / "payload"]
|
||||||
|
members.extend(sorted((stage / "payload").rglob("*")))
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with (
|
||||||
|
target.open("wb") as raw,
|
||||||
|
gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed,
|
||||||
|
tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive,
|
||||||
|
):
|
||||||
|
for path in members:
|
||||||
|
relative = path.relative_to(stage).as_posix()
|
||||||
|
info = tarfile.TarInfo(relative)
|
||||||
|
info.uid = 0
|
||||||
|
info.gid = 0
|
||||||
|
info.uname = "root"
|
||||||
|
info.gname = "root"
|
||||||
|
info.mtime = 0
|
||||||
|
if path.is_dir():
|
||||||
|
info.type = tarfile.DIRTYPE
|
||||||
|
info.mode = 0o755
|
||||||
|
archive.addfile(info, io.BytesIO())
|
||||||
|
else:
|
||||||
|
info.type = tarfile.REGTYPE
|
||||||
|
info.mode = 0o755 if path.suffix in {".sh", ".py"} else 0o644
|
||||||
|
info.size = path.stat().st_size
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
archive.addfile(info, stream)
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_json(value: object) -> bytes:
|
||||||
|
try:
|
||||||
|
return json.dumps(
|
||||||
|
value,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
allow_nan=False,
|
||||||
|
).encode("utf-8")
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise M49ExecutorReleaseBuildError("M4.9 release manifest is not JSON-compatible") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256_file(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
for chunk in iter(lambda: stream.read(_HASH_CHUNK_BYTES), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _is_elf(path: Path) -> bool:
|
||||||
|
try:
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
return stream.read(4) == b"\x7fELF"
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _object(value: object, label: str) -> dict[str, object]:
|
||||||
|
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||||
|
raise M49ExecutorReleaseBuildError(f"{label} must be an object")
|
||||||
|
return cast(dict[str, object], value)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_arguments(arguments: Sequence[str] | None = None) -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--output-directory", type=Path, required=True)
|
||||||
|
parser.add_argument("--revision")
|
||||||
|
return parser.parse_args(arguments)
|
||||||
|
|
||||||
|
|
||||||
|
def main(arguments: Sequence[str] | None = None) -> int:
|
||||||
|
options = _parse_arguments(arguments)
|
||||||
|
revision = options.revision or git_revision()
|
||||||
|
with tempfile.TemporaryDirectory(prefix="m49-revision-") as temporary:
|
||||||
|
snapshot = Path(temporary) / "source"
|
||||||
|
materialize_revision(revision=revision, destination=snapshot)
|
||||||
|
built = build_m49_executor_release_candidate(
|
||||||
|
source_root=snapshot,
|
||||||
|
output_directory=options.output_directory,
|
||||||
|
source_revision=revision,
|
||||||
|
source_state="committed-snapshot",
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"ok": True,
|
||||||
|
"state": "blocked",
|
||||||
|
"artifact": str(built.archive),
|
||||||
|
"sha256": built.archive_sha256,
|
||||||
|
"candidate_sha256": built.candidate_sha256,
|
||||||
|
"blockers": list(M49_RELEASE_BLOCKERS),
|
||||||
|
},
|
||||||
|
sort_keys=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Print the immutable Mac launchd plan for the Worker 006 reverse tunnel."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from k1link.observatory.worker_tunnel_launchd import (
|
||||||
|
plan_observatory_worker_tunnel_launch_agent,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--data-directory", type=Path, required=True)
|
||||||
|
parser.add_argument(
|
||||||
|
"--agent-path",
|
||||||
|
type=Path,
|
||||||
|
default=(
|
||||||
|
Path.home()
|
||||||
|
/ "Library/LaunchAgents/com.nodedc.observatory-worker-tunnel.local.plist"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
parser.add_argument("--ssh-path", type=Path, default=Path("/usr/bin/ssh"))
|
||||||
|
arguments = parser.parse_args()
|
||||||
|
plan = plan_observatory_worker_tunnel_launch_agent(
|
||||||
|
data_directory=arguments.data_directory,
|
||||||
|
agent_path=arguments.agent_path,
|
||||||
|
ssh_path=arguments.ssh_path,
|
||||||
|
)
|
||||||
|
print(json.dumps(plan.to_dict(), indent=2, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -24,6 +24,7 @@ from k1link.observatory.run_preparations import (
|
|||||||
from k1link.observatory.setups import (
|
from k1link.observatory.setups import (
|
||||||
LABORATORY_SETUP_CATALOG_SCHEMA,
|
LABORATORY_SETUP_CATALOG_SCHEMA,
|
||||||
LABORATORY_SETUP_REGISTRY_SCHEMA,
|
LABORATORY_SETUP_REGISTRY_SCHEMA,
|
||||||
|
OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||||
LaboratorySetupRegistry,
|
LaboratorySetupRegistry,
|
||||||
LaboratorySetupRegistryError,
|
LaboratorySetupRegistryError,
|
||||||
)
|
)
|
||||||
@@ -31,6 +32,7 @@ from k1link.observatory.setups import (
|
|||||||
__all__ = [
|
__all__ = [
|
||||||
"LABORATORY_SETUP_CATALOG_SCHEMA",
|
"LABORATORY_SETUP_CATALOG_SCHEMA",
|
||||||
"LABORATORY_SETUP_REGISTRY_SCHEMA",
|
"LABORATORY_SETUP_REGISTRY_SCHEMA",
|
||||||
|
"OBSERVATORY_CALCULATION_PROFILE_SCHEMA",
|
||||||
"MAX_RUN_PREPARATION_RECORDS",
|
"MAX_RUN_PREPARATION_RECORDS",
|
||||||
"MAX_RUN_PREPARATION_STORAGE_BYTES",
|
"MAX_RUN_PREPARATION_STORAGE_BYTES",
|
||||||
"OBSERVATORY_RUN_PREPARATION_REQUEST_SCHEMA",
|
"OBSERVATORY_RUN_PREPARATION_REQUEST_SCHEMA",
|
||||||
|
|||||||
@@ -0,0 +1,523 @@
|
|||||||
|
"""Exact local composition for the portable M4.9 TRAVEL/TGS executor.
|
||||||
|
|
||||||
|
The server supplies only a sealed recorded-job identity. Source delivery and
|
||||||
|
result upload remain implementations of the shared portable Worker ports; all
|
||||||
|
filesystem locations, the compiled runner and its build seal are selected by
|
||||||
|
reviewed Worker-local configuration.
|
||||||
|
|
||||||
|
This module does not register an executor or make a blocked runtime ready. It
|
||||||
|
is the adapter that an installer may bind only after the release archive,
|
||||||
|
compiled runner, executor image and local admission have all been sealed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
from collections.abc import Callable
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Final, Protocol, cast
|
||||||
|
|
||||||
|
from k1link.observatory.m49_portable_result import (
|
||||||
|
M49_PORTABLE_PROFILE_SHA256,
|
||||||
|
M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||||
|
assemble_m49_portable_result,
|
||||||
|
)
|
||||||
|
from k1link.observatory.m49_portable_source import (
|
||||||
|
M49_PORTABLE_STAGE_SCHEDULE,
|
||||||
|
M49_PORTABLE_TGS_SEQUENCE,
|
||||||
|
M49PortableSourceStage,
|
||||||
|
materialize_m49_portable_source_from_worker_stage,
|
||||||
|
validate_m49_portable_source_stage,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_result_contract import (
|
||||||
|
OBSERVATION_ONLY_AUTHORITY,
|
||||||
|
canonical_json,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_run_definitions import PortableRunDefinition
|
||||||
|
from k1link.observatory.portable_worker_runtime import (
|
||||||
|
PortableWorkerExecutorAdapter,
|
||||||
|
PortableWorkerResultDraft,
|
||||||
|
PortableWorkerResultPublisher,
|
||||||
|
PortableWorkerRuntimeAdmission,
|
||||||
|
PortableWorkerRuntimeCandidate,
|
||||||
|
PortableWorkerRuntimeJobRejectedError,
|
||||||
|
PortableWorkerRuntimePlan,
|
||||||
|
PortableWorkerRuntimeUnavailableError,
|
||||||
|
PortableWorkerSourceMaterializer,
|
||||||
|
PortableWorkerSourceStage,
|
||||||
|
)
|
||||||
|
from k1link.observatory.worker_agent import SealedObservatoryRecordedJob
|
||||||
|
|
||||||
|
M49_COMPILED_RUNNER_BUILD_SCHEMA: Final = "missioncore.m49-tgs-portable-compiled-runner-build/v1"
|
||||||
|
M49_PORTABLE_RUNNER_SOURCE_SHA256: Final = (
|
||||||
|
"52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9"
|
||||||
|
)
|
||||||
|
M49_PORTABLE_RUNNER_WRAPPER_SHA256: Final = (
|
||||||
|
"2d6c32560682647f868e4ce4c2605749f17c60482a609f8c03ff951411f48ffb"
|
||||||
|
)
|
||||||
|
M49_PORTABLE_TRAVEL_BUILD_IMAGE_SHA256: Final = (
|
||||||
|
"7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f"
|
||||||
|
)
|
||||||
|
M49_PORTABLE_RUNTIME_PHASES: Final = (
|
||||||
|
"source-delivery",
|
||||||
|
"camera-lidar-timeline-materializer",
|
||||||
|
"portable-tgs-input-materializer",
|
||||||
|
"portable-tgs-runner",
|
||||||
|
"result-v2-assembler",
|
||||||
|
"observatory-result-publisher",
|
||||||
|
)
|
||||||
|
M49_PORTABLE_COMPILED_RUNNER_ASSET_ID: Final = "m49-portable-compiled-runner"
|
||||||
|
M49_PORTABLE_COMPILED_RUNNER_BUILD_SEAL_ASSET_ID: Final = "m49-portable-compiled-runner-build-seal"
|
||||||
|
M49_PORTABLE_PROFILE_ASSET_ID: Final = "m49-portable-profile"
|
||||||
|
M49_PORTABLE_TRAVEL_IMAGE_ASSET_ID: Final = "travel-tgs-image"
|
||||||
|
M49_PORTABLE_COMPILER_CONTRACT: Final = {
|
||||||
|
"compiler": "g++",
|
||||||
|
"language_standard": "c++17",
|
||||||
|
"flags": ["-O3", "-DNDEBUG", "-pthread"],
|
||||||
|
"travel_include": "/opt/travel/src/TRAVEL/cpp/travel/core",
|
||||||
|
"eigen_include": "/usr/include/eigen3",
|
||||||
|
}
|
||||||
|
|
||||||
|
_MAX_BUILD_SEAL_BYTES: Final = 128 * 1024
|
||||||
|
_MAX_RUN_SECONDS: Final = 24 * 60 * 60
|
||||||
|
|
||||||
|
|
||||||
|
class M49PortableExecutorError(PortableWorkerRuntimeUnavailableError):
|
||||||
|
"""The local M4.9 executor installation or invocation is not exact."""
|
||||||
|
|
||||||
|
|
||||||
|
class M49PortableRunnerInvoker(Protocol):
|
||||||
|
def __call__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
binary: Path,
|
||||||
|
sequence: Path,
|
||||||
|
schedule: Path,
|
||||||
|
output: Path,
|
||||||
|
timing: Path,
|
||||||
|
workspace: Path,
|
||||||
|
timeout_seconds: int,
|
||||||
|
) -> None: ...
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class M49PortableRunnerInstallation:
|
||||||
|
"""Worker-local paths bound to an exact build-only runner seal."""
|
||||||
|
|
||||||
|
profile_path: Path
|
||||||
|
runner_binary_path: Path
|
||||||
|
runner_build_seal_path: Path
|
||||||
|
runner_build_seal_sha256: str
|
||||||
|
output_parent: Path
|
||||||
|
timeout_seconds: int = _MAX_RUN_SECONDS
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
profile = _exact_file(
|
||||||
|
self.profile_path,
|
||||||
|
M49_PORTABLE_PROFILE_SHA256,
|
||||||
|
"portable M4.9 profile",
|
||||||
|
)
|
||||||
|
seal = _exact_file(
|
||||||
|
self.runner_build_seal_path,
|
||||||
|
self.runner_build_seal_sha256,
|
||||||
|
"portable M4.9 runner build seal",
|
||||||
|
)
|
||||||
|
binary = _regular_file(self.runner_binary_path, "portable M4.9 compiled runner")
|
||||||
|
document = _read_build_seal(seal)
|
||||||
|
binary_row = _object(document["binary"], "portable M4.9 build-seal binary")
|
||||||
|
if binary_row != {
|
||||||
|
"file_name": "run_m49_tgs_portable",
|
||||||
|
"format": "elf",
|
||||||
|
"byte_length": binary.stat().st_size,
|
||||||
|
"sha256": _sha256_file(binary),
|
||||||
|
} or document["profile_sha256"] != _sha256_file(profile):
|
||||||
|
raise M49PortableExecutorError(
|
||||||
|
"portable M4.9 compiled runner differs from its build seal"
|
||||||
|
)
|
||||||
|
if not os.access(binary, os.X_OK):
|
||||||
|
raise M49PortableExecutorError("portable M4.9 compiled runner is not executable")
|
||||||
|
if not _is_elf(binary):
|
||||||
|
raise M49PortableExecutorError("portable M4.9 compiled runner is not ELF")
|
||||||
|
parent = self.output_parent.expanduser().absolute()
|
||||||
|
parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
if parent.is_symlink() or not parent.is_dir():
|
||||||
|
raise M49PortableExecutorError("portable M4.9 output root is unsafe")
|
||||||
|
if (
|
||||||
|
isinstance(self.timeout_seconds, bool)
|
||||||
|
or not isinstance(self.timeout_seconds, int)
|
||||||
|
or not 1 <= self.timeout_seconds <= _MAX_RUN_SECONDS
|
||||||
|
):
|
||||||
|
raise ValueError("portable M4.9 runner timeout is invalid")
|
||||||
|
object.__setattr__(self, "profile_path", profile)
|
||||||
|
object.__setattr__(self, "runner_binary_path", binary)
|
||||||
|
object.__setattr__(self, "runner_build_seal_path", seal)
|
||||||
|
object.__setattr__(self, "output_parent", parent)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def runner_binary_sha256(self) -> str:
|
||||||
|
binary = _object(
|
||||||
|
_read_build_seal(self.runner_build_seal_path)["binary"],
|
||||||
|
"portable M4.9 build-seal binary",
|
||||||
|
)
|
||||||
|
return cast(str, binary["sha256"])
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class M49PortableBoundSourceStage(PortableWorkerSourceStage):
|
||||||
|
"""Claim-bound in-memory extension of the shared source-stage port."""
|
||||||
|
|
||||||
|
job: SealedObservatoryRecordedJob
|
||||||
|
m49_stage: M49PortableSourceStage
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
PortableWorkerSourceStage.__post_init__(self)
|
||||||
|
if (
|
||||||
|
self.root != self.m49_stage.root
|
||||||
|
or self.source_bundle_sha256 != self.job.source_bundle_sha256
|
||||||
|
or self.source_capability_manifest_sha256 != self.job.source_capability_manifest_sha256
|
||||||
|
or self.source_adapter_sha256 != self.job.source_adapter_sha256
|
||||||
|
):
|
||||||
|
raise M49PortableExecutorError("portable M4.9 bound source differs from its sealed job")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class M49PortableSourceMaterializerAdapter:
|
||||||
|
"""Adapt exact Worker transport delivery to the M4.9 TGS source port."""
|
||||||
|
|
||||||
|
upstream: PortableWorkerSourceMaterializer
|
||||||
|
profile_path: Path
|
||||||
|
output_parent: Path
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
profile = _exact_file(
|
||||||
|
self.profile_path,
|
||||||
|
M49_PORTABLE_PROFILE_SHA256,
|
||||||
|
"portable M4.9 profile",
|
||||||
|
)
|
||||||
|
parent = self.output_parent.expanduser().absolute()
|
||||||
|
parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
if parent.is_symlink() or not parent.is_dir():
|
||||||
|
raise M49PortableExecutorError("portable M4.9 source output root is unsafe")
|
||||||
|
object.__setattr__(self, "profile_path", profile)
|
||||||
|
object.__setattr__(self, "output_parent", parent)
|
||||||
|
|
||||||
|
def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage:
|
||||||
|
delivered = self.upstream.materialize(job)
|
||||||
|
materialized = materialize_m49_portable_source_from_worker_stage(
|
||||||
|
worker_stage=delivered,
|
||||||
|
job=job,
|
||||||
|
profile_path=self.profile_path,
|
||||||
|
output_parent=self.output_parent,
|
||||||
|
)
|
||||||
|
return M49PortableBoundSourceStage(
|
||||||
|
root=materialized.root,
|
||||||
|
source_bundle_sha256=job.source_bundle_sha256,
|
||||||
|
source_capability_manifest_sha256=job.source_capability_manifest_sha256,
|
||||||
|
source_adapter_sha256=job.source_adapter_sha256,
|
||||||
|
job=job,
|
||||||
|
m49_stage=materialized,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class M49PortableProfileRunnerAdapter:
|
||||||
|
"""Run the exact installed binary and assemble the deterministic result-v2."""
|
||||||
|
|
||||||
|
definition: PortableRunDefinition
|
||||||
|
installation: M49PortableRunnerInstallation
|
||||||
|
created_at_utc: Callable[[], str]
|
||||||
|
invoker: M49PortableRunnerInvoker | None = None
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_verify_definition(self.definition)
|
||||||
|
|
||||||
|
def run(
|
||||||
|
self,
|
||||||
|
plan: PortableWorkerRuntimePlan,
|
||||||
|
source: PortableWorkerSourceStage,
|
||||||
|
) -> PortableWorkerResultDraft:
|
||||||
|
if not isinstance(source, M49PortableBoundSourceStage):
|
||||||
|
raise PortableWorkerRuntimeJobRejectedError(
|
||||||
|
"portable M4.9 runner requires its claim-bound source stage"
|
||||||
|
)
|
||||||
|
job = source.job
|
||||||
|
_verify_plan(plan, job=job, definition=self.definition)
|
||||||
|
stage = validate_m49_portable_source_stage(source.root)
|
||||||
|
_verify_installation_unchanged(self.installation)
|
||||||
|
workspace = Path(
|
||||||
|
tempfile.mkdtemp(prefix=".m49-portable-run-", dir=self.installation.output_parent)
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
output = workspace / "outputs"
|
||||||
|
timing = workspace / "timing.tsv"
|
||||||
|
invoker = self.invoker or _invoke_exact_runner
|
||||||
|
invoker(
|
||||||
|
binary=self.installation.runner_binary_path,
|
||||||
|
sequence=stage.root / M49_PORTABLE_TGS_SEQUENCE,
|
||||||
|
schedule=stage.root / M49_PORTABLE_STAGE_SCHEDULE,
|
||||||
|
output=output,
|
||||||
|
timing=timing,
|
||||||
|
workspace=workspace,
|
||||||
|
timeout_seconds=self.installation.timeout_seconds,
|
||||||
|
)
|
||||||
|
package = assemble_m49_portable_result(
|
||||||
|
source_stage_root=stage.root,
|
||||||
|
runner_output_root=output,
|
||||||
|
runner_timing_path=timing,
|
||||||
|
profile_path=self.installation.profile_path,
|
||||||
|
output_parent=self.installation.output_parent / "result-packages",
|
||||||
|
job=job,
|
||||||
|
definition=self.definition,
|
||||||
|
created_at_utc=self.created_at_utc(),
|
||||||
|
)
|
||||||
|
return PortableWorkerResultDraft(
|
||||||
|
root=package.root,
|
||||||
|
result_id=package.result_id,
|
||||||
|
result_sha256=package.manifest.manifest_sha256,
|
||||||
|
result_contract_sha256=M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(workspace, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def compose_m49_portable_executor_adapter(
|
||||||
|
*,
|
||||||
|
candidate: PortableWorkerRuntimeCandidate,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
admission: PortableWorkerRuntimeAdmission,
|
||||||
|
source_transport: PortableWorkerSourceMaterializer,
|
||||||
|
result_transport: PortableWorkerResultPublisher,
|
||||||
|
installation: M49PortableRunnerInstallation,
|
||||||
|
source_output_parent: Path,
|
||||||
|
created_at_utc: Callable[[], str],
|
||||||
|
invoker: M49PortableRunnerInvoker | None = None,
|
||||||
|
) -> PortableWorkerExecutorAdapter:
|
||||||
|
"""Compose shared Worker ports without changing their API or registry state."""
|
||||||
|
|
||||||
|
_verify_candidate_assets(candidate, installation)
|
||||||
|
return PortableWorkerExecutorAdapter(
|
||||||
|
candidate=candidate,
|
||||||
|
definition=definition,
|
||||||
|
admission=admission,
|
||||||
|
source_materializer=M49PortableSourceMaterializerAdapter(
|
||||||
|
upstream=source_transport,
|
||||||
|
profile_path=installation.profile_path,
|
||||||
|
output_parent=source_output_parent,
|
||||||
|
),
|
||||||
|
runner=M49PortableProfileRunnerAdapter(
|
||||||
|
definition=definition,
|
||||||
|
installation=installation,
|
||||||
|
created_at_utc=created_at_utc,
|
||||||
|
invoker=invoker,
|
||||||
|
),
|
||||||
|
publisher=result_transport,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _invoke_exact_runner(
|
||||||
|
*,
|
||||||
|
binary: Path,
|
||||||
|
sequence: Path,
|
||||||
|
schedule: Path,
|
||||||
|
output: Path,
|
||||||
|
timing: Path,
|
||||||
|
workspace: Path,
|
||||||
|
timeout_seconds: int,
|
||||||
|
) -> None:
|
||||||
|
stdout = workspace / "runner.stdout.log"
|
||||||
|
stderr = workspace / "runner.stderr.log"
|
||||||
|
try:
|
||||||
|
with stdout.open("xb") as stdout_stream, stderr.open("xb") as stderr_stream:
|
||||||
|
completed = subprocess.run(
|
||||||
|
[str(binary), str(sequence), str(schedule), str(output), str(timing)],
|
||||||
|
cwd=workspace,
|
||||||
|
env={"LANG": "C", "LC_ALL": "C", "TZ": "UTC"},
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
stdout=stdout_stream,
|
||||||
|
stderr=stderr_stream,
|
||||||
|
check=False,
|
||||||
|
timeout=timeout_seconds,
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.SubprocessError) as exc:
|
||||||
|
raise M49PortableExecutorError("portable M4.9 runner invocation failed") from exc
|
||||||
|
if completed.returncode != 0:
|
||||||
|
raise M49PortableExecutorError("portable M4.9 runner rejected its exact source stage")
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_definition(definition: PortableRunDefinition) -> None:
|
||||||
|
components = {component.component_id: component for component in definition.components}
|
||||||
|
profile = components.get("m49-tgs-portable-profile-v2")
|
||||||
|
if (
|
||||||
|
definition.setup_id != "m49-tgs-portable-v2"
|
||||||
|
or definition.definition_id != "m49-tgs-portable"
|
||||||
|
or definition.result_contract.contract_sha256 != M49_PORTABLE_RESULT_CONTRACT_SHA256
|
||||||
|
or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY
|
||||||
|
or profile is None
|
||||||
|
or profile.sha256 != M49_PORTABLE_PROFILE_SHA256
|
||||||
|
):
|
||||||
|
raise M49PortableExecutorError("portable M4.9 RunDefinition identity changed")
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_plan(
|
||||||
|
plan: PortableWorkerRuntimePlan,
|
||||||
|
*,
|
||||||
|
job: SealedObservatoryRecordedJob,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
) -> None:
|
||||||
|
if (
|
||||||
|
plan.job_id != job.job_id
|
||||||
|
or plan.setup_id != job.setup_id
|
||||||
|
or plan.definition_sha256 != job.definition_sha256
|
||||||
|
or plan.source_bundle_sha256 != job.source_bundle_sha256
|
||||||
|
or plan.source_capability_manifest_sha256 != job.source_capability_manifest_sha256
|
||||||
|
or plan.result_contract_sha256 != M49_PORTABLE_RESULT_CONTRACT_SHA256
|
||||||
|
or plan.phases != M49_PORTABLE_RUNTIME_PHASES
|
||||||
|
or job.definition_sha256 != definition.definition_sha256
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeJobRejectedError(
|
||||||
|
"portable M4.9 runtime plan differs from its sealed job"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_candidate_assets(
|
||||||
|
candidate: PortableWorkerRuntimeCandidate,
|
||||||
|
installation: M49PortableRunnerInstallation,
|
||||||
|
) -> None:
|
||||||
|
assets = {asset.asset_id: asset for asset in candidate.reusable_assets}
|
||||||
|
binary = assets.get(M49_PORTABLE_COMPILED_RUNNER_ASSET_ID)
|
||||||
|
build_seal = assets.get(M49_PORTABLE_COMPILED_RUNNER_BUILD_SEAL_ASSET_ID)
|
||||||
|
profile = assets.get(M49_PORTABLE_PROFILE_ASSET_ID)
|
||||||
|
image = assets.get(M49_PORTABLE_TRAVEL_IMAGE_ASSET_ID)
|
||||||
|
if (
|
||||||
|
binary is None
|
||||||
|
or binary.kind != "local-file"
|
||||||
|
or binary.sha256 != installation.runner_binary_sha256
|
||||||
|
or binary.byte_length != installation.runner_binary_path.stat().st_size
|
||||||
|
or build_seal is None
|
||||||
|
or build_seal.kind != "local-file"
|
||||||
|
or build_seal.sha256 != installation.runner_build_seal_sha256
|
||||||
|
or build_seal.byte_length != installation.runner_build_seal_path.stat().st_size
|
||||||
|
or profile is None
|
||||||
|
or profile.sha256 != M49_PORTABLE_PROFILE_SHA256
|
||||||
|
or image is None
|
||||||
|
or image.kind != "container-image"
|
||||||
|
or image.sha256 != M49_PORTABLE_TRAVEL_BUILD_IMAGE_SHA256
|
||||||
|
):
|
||||||
|
raise M49PortableExecutorError(
|
||||||
|
"portable M4.9 runtime candidate lacks exact installed runner assets"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_installation_unchanged(installation: M49PortableRunnerInstallation) -> None:
|
||||||
|
_exact_file(
|
||||||
|
installation.profile_path,
|
||||||
|
M49_PORTABLE_PROFILE_SHA256,
|
||||||
|
"portable M4.9 profile",
|
||||||
|
)
|
||||||
|
_exact_file(
|
||||||
|
installation.runner_build_seal_path,
|
||||||
|
installation.runner_build_seal_sha256,
|
||||||
|
"portable M4.9 runner build seal",
|
||||||
|
)
|
||||||
|
binary = _regular_file(
|
||||||
|
installation.runner_binary_path,
|
||||||
|
"portable M4.9 compiled runner",
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
_sha256_file(binary) != installation.runner_binary_sha256
|
||||||
|
or not os.access(binary, os.X_OK)
|
||||||
|
or not _is_elf(binary)
|
||||||
|
):
|
||||||
|
raise M49PortableExecutorError("portable M4.9 installed runner changed")
|
||||||
|
|
||||||
|
|
||||||
|
def _read_build_seal(path: Path) -> dict[str, object]:
|
||||||
|
try:
|
||||||
|
payload = path.read_bytes()
|
||||||
|
decoded: object = json.loads(payload.decode("utf-8"))
|
||||||
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise M49PortableExecutorError("portable M4.9 build seal is unreadable") from exc
|
||||||
|
if not 0 < len(payload) <= _MAX_BUILD_SEAL_BYTES:
|
||||||
|
raise M49PortableExecutorError("portable M4.9 build seal size is invalid")
|
||||||
|
document = _object(decoded, "portable M4.9 build seal")
|
||||||
|
if (
|
||||||
|
set(document)
|
||||||
|
!= {
|
||||||
|
"schema_version",
|
||||||
|
"source_revision",
|
||||||
|
"source_state",
|
||||||
|
"build_image_sha256",
|
||||||
|
"profile_sha256",
|
||||||
|
"runner_source_sha256",
|
||||||
|
"runner_wrapper_sha256",
|
||||||
|
"compiler_contract",
|
||||||
|
"binary",
|
||||||
|
"authority",
|
||||||
|
}
|
||||||
|
or payload != canonical_json(document)
|
||||||
|
or document["schema_version"] != M49_COMPILED_RUNNER_BUILD_SCHEMA
|
||||||
|
or not isinstance(document["source_revision"], str)
|
||||||
|
or len(document["source_revision"]) != 40
|
||||||
|
or any(character not in "0123456789abcdef" for character in document["source_revision"])
|
||||||
|
or document["source_state"] != "committed-snapshot"
|
||||||
|
or document["build_image_sha256"] != M49_PORTABLE_TRAVEL_BUILD_IMAGE_SHA256
|
||||||
|
or document["profile_sha256"] != M49_PORTABLE_PROFILE_SHA256
|
||||||
|
or document["runner_source_sha256"] != M49_PORTABLE_RUNNER_SOURCE_SHA256
|
||||||
|
or document["runner_wrapper_sha256"] != M49_PORTABLE_RUNNER_WRAPPER_SHA256
|
||||||
|
or document["compiler_contract"] != M49_PORTABLE_COMPILER_CONTRACT
|
||||||
|
or document["authority"] != OBSERVATION_ONLY_AUTHORITY
|
||||||
|
):
|
||||||
|
raise M49PortableExecutorError("portable M4.9 build seal identity changed")
|
||||||
|
return document
|
||||||
|
|
||||||
|
|
||||||
|
def _exact_file(path: Path, expected_sha256: str, label: str) -> Path:
|
||||||
|
candidate = _regular_file(path, label)
|
||||||
|
if _sha256_file(candidate) != expected_sha256:
|
||||||
|
raise M49PortableExecutorError(f"{label} digest changed")
|
||||||
|
return candidate
|
||||||
|
|
||||||
|
|
||||||
|
def _regular_file(path: Path, label: str) -> Path:
|
||||||
|
candidate = path.expanduser().absolute()
|
||||||
|
try:
|
||||||
|
metadata = candidate.lstat()
|
||||||
|
resolved = candidate.resolve(strict=True)
|
||||||
|
except OSError as exc:
|
||||||
|
raise M49PortableExecutorError(f"{label} is unavailable") from exc
|
||||||
|
if (
|
||||||
|
stat.S_ISLNK(metadata.st_mode)
|
||||||
|
or not stat.S_ISREG(metadata.st_mode)
|
||||||
|
or not os.path.samefile(candidate, resolved)
|
||||||
|
):
|
||||||
|
raise M49PortableExecutorError(f"{label} is unsafe")
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256_file(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _is_elf(path: Path) -> bool:
|
||||||
|
try:
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
return stream.read(4) == b"\x7fELF"
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _object(value: object, label: str) -> dict[str, object]:
|
||||||
|
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||||
|
raise M49PortableExecutorError(f"{label} must be an object")
|
||||||
|
return cast(dict[str, object], value)
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -133,9 +133,12 @@ class PortableRecordedQueueBindingService:
|
|||||||
) -> PortableRecordedSourceCapability:
|
) -> PortableRecordedSourceCapability:
|
||||||
"""Return cheap authoritative compatibility without replay preparation."""
|
"""Return cheap authoritative compatibility without replay preparation."""
|
||||||
|
|
||||||
portable, recorded = self._resolve_definition(setup_id, definition_sha256)
|
# Catalog projection must remain available for not-yet-installed
|
||||||
|
# executors. Probe only resolves the immutable portable definition;
|
||||||
|
# check/admit/submit still require a ready executor before source reads.
|
||||||
|
portable = self._definitions.resolve(setup_id, definition_sha256)
|
||||||
capability = self._source_service(portable).probe(source_session_id)
|
capability = self._source_service(portable).probe(source_session_id)
|
||||||
if recorded.source_adapter_sha256 != capability.source_adapter_sha256:
|
if portable.source_adapter.contract_sha256 != capability.source_adapter_sha256:
|
||||||
raise PortableQueueBindingIntegrityError(
|
raise PortableQueueBindingIntegrityError(
|
||||||
"portable registry and source capability adapter identities disagree"
|
"portable registry and source capability adapter identities disagree"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,601 @@
|
|||||||
|
"""Strict, path-free contracts for portable Observatory result packages."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from collections.abc import Callable, Mapping, Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
from typing import Final, cast
|
||||||
|
|
||||||
|
from k1link.observatory.portable_run_definitions import (
|
||||||
|
PortableRunDefinition,
|
||||||
|
canonical_sha256,
|
||||||
|
)
|
||||||
|
from k1link.observatory.recorded_jobs import ObservatoryRecordedJob
|
||||||
|
|
||||||
|
PORTABLE_RESULT_PACKAGE_SCHEMA: Final = (
|
||||||
|
"missioncore.observatory-portable-result-package/v1"
|
||||||
|
)
|
||||||
|
PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA: Final = (
|
||||||
|
"missioncore.observatory-portable-result-package-identity/v1"
|
||||||
|
)
|
||||||
|
PORTABLE_RESULT_PUBLICATION_SCHEMA: Final = (
|
||||||
|
"missioncore.observatory-portable-result-publication/v1"
|
||||||
|
)
|
||||||
|
OBSERVATORY_CALCULATION_PROFILE_SCHEMA: Final = (
|
||||||
|
"missioncore.observatory-calculation-profile/v1"
|
||||||
|
)
|
||||||
|
RESULT_PACKAGE_MANIFEST_NAME: Final = "manifest.json"
|
||||||
|
RESULT_DOCUMENT_ROLE: Final = "result-document"
|
||||||
|
RESULT_PACKAGE_MANIFEST_ROLE: Final = "result-package-manifest"
|
||||||
|
|
||||||
|
_MAX_MANIFEST_BYTES: Final = 1024 * 1024
|
||||||
|
_MAX_RESULT_DOCUMENT_BYTES: Final = 8 * 1024 * 1024
|
||||||
|
_MAX_ARTIFACTS: Final = 128
|
||||||
|
_MAX_ARTIFACT_BYTES: Final = (1 << 63) - 1
|
||||||
|
_SHA256: Final = re.compile(r"^[a-f0-9]{64}$")
|
||||||
|
_IDENTIFIER: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||||
|
_SESSION_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||||
|
_ROLE: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||||
|
_PATH_COMPONENT: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||||
|
_LAB_ID: Final = re.compile(r"^LAB [A-Z][A-Z0-9._-]{0,31}$")
|
||||||
|
OBSERVATION_ONLY_AUTHORITY: Final = {
|
||||||
|
"commands_enabled": False,
|
||||||
|
"actuation_allowed": False,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
"production_accepted": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class PortableResultPublisherError(RuntimeError):
|
||||||
|
"""Base class for verified portable result publication failures."""
|
||||||
|
|
||||||
|
|
||||||
|
class PortableResultPackageIntegrityError(PortableResultPublisherError):
|
||||||
|
"""The supplied package or one of its content identities is invalid."""
|
||||||
|
|
||||||
|
|
||||||
|
class PortableResultPublicationBlockedError(PortableResultPublisherError):
|
||||||
|
"""A required server-owned definition, policy, or validator is absent."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableResultArtifact:
|
||||||
|
"""One confined, content-addressed package member."""
|
||||||
|
|
||||||
|
role: str
|
||||||
|
relative_path: str
|
||||||
|
media_type: str
|
||||||
|
byte_length: int
|
||||||
|
sha256: str
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_pattern(self.role, _ROLE, "portable result artifact role")
|
||||||
|
if self.role == RESULT_PACKAGE_MANIFEST_ROLE:
|
||||||
|
raise ValueError("portable result artifact role is reserved")
|
||||||
|
relative_artifact_path(self.relative_path)
|
||||||
|
_media_type(self.media_type)
|
||||||
|
if (
|
||||||
|
not isinstance(self.byte_length, int)
|
||||||
|
or isinstance(self.byte_length, bool)
|
||||||
|
or not 0 <= self.byte_length <= _MAX_ARTIFACT_BYTES
|
||||||
|
):
|
||||||
|
raise ValueError("portable result artifact byte length is invalid")
|
||||||
|
digest(self.sha256, "portable result artifact sha256")
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"role": self.role,
|
||||||
|
"relative_path": self.relative_path,
|
||||||
|
"media_type": self.media_type,
|
||||||
|
"byte_length": self.byte_length,
|
||||||
|
"sha256": self.sha256,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableResultPackageManifest:
|
||||||
|
"""Strict Worker-produced manifest with a separately hashed identity."""
|
||||||
|
|
||||||
|
identity_sha256: str
|
||||||
|
created_at_utc: str
|
||||||
|
job: dict[str, object]
|
||||||
|
source: dict[str, object]
|
||||||
|
run_definition: dict[str, object]
|
||||||
|
result: dict[str, object]
|
||||||
|
authority: dict[str, object]
|
||||||
|
artifacts: tuple[PortableResultArtifact, ...]
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
digest(self.identity_sha256, "portable result package identity sha256")
|
||||||
|
_timestamp(self.created_at_utc, "portable result package creation time")
|
||||||
|
if self.authority != OBSERVATION_ONLY_AUTHORITY:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package is not observation-only"
|
||||||
|
)
|
||||||
|
if not 1 <= len(self.artifacts) <= _MAX_ARTIFACTS:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package artifact count is invalid"
|
||||||
|
)
|
||||||
|
roles = tuple(artifact.role for artifact in self.artifacts)
|
||||||
|
paths = tuple(artifact.relative_path for artifact in self.artifacts)
|
||||||
|
if roles != tuple(sorted(roles)) or len(set(roles)) != len(roles):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package artifacts are not canonically ordered"
|
||||||
|
)
|
||||||
|
if len(set(paths)) != len(paths):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package artifact paths are not unique"
|
||||||
|
)
|
||||||
|
result_documents = tuple(
|
||||||
|
artifact for artifact in self.artifacts if artifact.role == RESULT_DOCUMENT_ROLE
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
len(result_documents) != 1
|
||||||
|
or result_documents[0].media_type != "application/json"
|
||||||
|
or not 0 < result_documents[0].byte_length <= _MAX_RESULT_DOCUMENT_BYTES
|
||||||
|
):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package requires one JSON result document"
|
||||||
|
)
|
||||||
|
if self.identity_sha256 != canonical_sha256(self.identity_document()):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package identity digest changed"
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def create(
|
||||||
|
cls,
|
||||||
|
*,
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
result_id: str,
|
||||||
|
created_at_utc: str,
|
||||||
|
artifacts: Sequence[PortableResultArtifact],
|
||||||
|
) -> PortableResultPackageManifest:
|
||||||
|
"""Build the canonical package envelope used by a future Worker assembler."""
|
||||||
|
|
||||||
|
_pattern(result_id, _SESSION_ID, "portable result id")
|
||||||
|
job_document = job_identity_document(job)
|
||||||
|
source_document = source_identity_document(job)
|
||||||
|
definition_document = run_definition_document(definition)
|
||||||
|
result_document = result_identity_document(definition, result_id)
|
||||||
|
authority: dict[str, object] = dict(OBSERVATION_ONLY_AUTHORITY)
|
||||||
|
normalized_artifacts = tuple(
|
||||||
|
sorted(artifacts, key=lambda artifact: artifact.role)
|
||||||
|
)
|
||||||
|
identity_document = _package_identity_document(
|
||||||
|
created_at_utc=created_at_utc,
|
||||||
|
job=job_document,
|
||||||
|
source=source_document,
|
||||||
|
run_definition=definition_document,
|
||||||
|
result=result_document,
|
||||||
|
authority=authority,
|
||||||
|
artifacts=normalized_artifacts,
|
||||||
|
)
|
||||||
|
return cls(
|
||||||
|
identity_sha256=canonical_sha256(identity_document),
|
||||||
|
created_at_utc=created_at_utc,
|
||||||
|
job=job_document,
|
||||||
|
source=source_document,
|
||||||
|
run_definition=definition_document,
|
||||||
|
result=result_document,
|
||||||
|
authority=authority,
|
||||||
|
artifacts=normalized_artifacts,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_bytes(cls, payload: bytes) -> PortableResultPackageManifest:
|
||||||
|
if not 0 < len(payload) <= _MAX_MANIFEST_BYTES:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package manifest size is invalid"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
decoded: object = json.loads(payload.decode("utf-8"))
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package manifest is not valid JSON"
|
||||||
|
) from exc
|
||||||
|
document = object_document(decoded, "portable result package manifest")
|
||||||
|
exact_keys(
|
||||||
|
document,
|
||||||
|
{
|
||||||
|
"schema_version",
|
||||||
|
"identity_sha256",
|
||||||
|
"created_at_utc",
|
||||||
|
"job",
|
||||||
|
"source",
|
||||||
|
"run_definition",
|
||||||
|
"result",
|
||||||
|
"authority",
|
||||||
|
"artifacts",
|
||||||
|
},
|
||||||
|
"portable result package manifest",
|
||||||
|
)
|
||||||
|
if document["schema_version"] != PORTABLE_RESULT_PACKAGE_SCHEMA:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package manifest schema is invalid"
|
||||||
|
)
|
||||||
|
if payload != canonical_json(document):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package manifest is not canonical JSON"
|
||||||
|
)
|
||||||
|
artifacts_value = document["artifacts"]
|
||||||
|
if not isinstance(artifacts_value, list):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package artifacts are not an array"
|
||||||
|
)
|
||||||
|
artifacts = tuple(_artifact(value) for value in artifacts_value)
|
||||||
|
try:
|
||||||
|
return cls(
|
||||||
|
identity_sha256=string(
|
||||||
|
document["identity_sha256"],
|
||||||
|
"portable result package identity sha256",
|
||||||
|
),
|
||||||
|
created_at_utc=string(
|
||||||
|
document["created_at_utc"],
|
||||||
|
"portable result package creation time",
|
||||||
|
),
|
||||||
|
job=object_document(document["job"], "portable result job identity"),
|
||||||
|
source=object_document(
|
||||||
|
document["source"], "portable result source identity"
|
||||||
|
),
|
||||||
|
run_definition=object_document(
|
||||||
|
document["run_definition"],
|
||||||
|
"portable result RunDefinition",
|
||||||
|
),
|
||||||
|
result=object_document(document["result"], "portable result identity"),
|
||||||
|
authority=object_document(
|
||||||
|
document["authority"], "portable result authority"
|
||||||
|
),
|
||||||
|
artifacts=artifacts,
|
||||||
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package manifest field is invalid"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def identity_document(self) -> dict[str, object]:
|
||||||
|
return _package_identity_document(
|
||||||
|
created_at_utc=self.created_at_utc,
|
||||||
|
job=self.job,
|
||||||
|
source=self.source,
|
||||||
|
run_definition=self.run_definition,
|
||||||
|
result=self.result,
|
||||||
|
authority=self.authority,
|
||||||
|
artifacts=self.artifacts,
|
||||||
|
)
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
identity = self.identity_document()
|
||||||
|
identity.pop("schema_version")
|
||||||
|
return {
|
||||||
|
"schema_version": PORTABLE_RESULT_PACKAGE_SCHEMA,
|
||||||
|
"identity_sha256": self.identity_sha256,
|
||||||
|
**identity,
|
||||||
|
}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def canonical_bytes(self) -> bytes:
|
||||||
|
return canonical_json(self.as_dict())
|
||||||
|
|
||||||
|
@property
|
||||||
|
def manifest_sha256(self) -> str:
|
||||||
|
return hashlib.sha256(self.canonical_bytes).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableCalculationProfilePolicy:
|
||||||
|
"""Server-owned presentation identity bound to one exact RunDefinition."""
|
||||||
|
|
||||||
|
setup_id: str
|
||||||
|
definition_id: str
|
||||||
|
definition_version: int
|
||||||
|
definition_sha256: str
|
||||||
|
lab_id: str
|
||||||
|
display_name: str
|
||||||
|
include_recorded_media: bool = False
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_pattern(self.setup_id, _IDENTIFIER, "portable calculation profile setup id")
|
||||||
|
_pattern(
|
||||||
|
self.definition_id,
|
||||||
|
_IDENTIFIER,
|
||||||
|
"portable calculation profile definition id",
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
not isinstance(self.definition_version, int)
|
||||||
|
or isinstance(self.definition_version, bool)
|
||||||
|
or self.definition_version < 1
|
||||||
|
):
|
||||||
|
raise ValueError("portable calculation profile version is invalid")
|
||||||
|
digest(
|
||||||
|
self.definition_sha256,
|
||||||
|
"portable calculation profile definition sha256",
|
||||||
|
)
|
||||||
|
if _LAB_ID.fullmatch(self.lab_id) is None:
|
||||||
|
raise ValueError("portable calculation profile LAB id is invalid")
|
||||||
|
_text(self.display_name, "portable calculation profile display name", maximum=160)
|
||||||
|
if not isinstance(self.include_recorded_media, bool):
|
||||||
|
raise ValueError("portable calculation profile media policy is invalid")
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||||
|
"setup_id": self.setup_id,
|
||||||
|
"display_name": self.display_name,
|
||||||
|
"origin": "archived-definition",
|
||||||
|
"definition_id": self.definition_id,
|
||||||
|
"definition_version": self.definition_version,
|
||||||
|
"definition_sha256": self.definition_sha256,
|
||||||
|
}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def identity_sha256(self) -> str:
|
||||||
|
return canonical_sha256(self.as_dict())
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableCalculationProfileRegistry:
|
||||||
|
policies: tuple[PortableCalculationProfilePolicy, ...]
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
keys = tuple(
|
||||||
|
(policy.setup_id, policy.definition_sha256) for policy in self.policies
|
||||||
|
)
|
||||||
|
if len(keys) != len(set(keys)):
|
||||||
|
raise ValueError("portable calculation profile policies are not unique")
|
||||||
|
|
||||||
|
def resolve(
|
||||||
|
self,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
) -> PortableCalculationProfilePolicy:
|
||||||
|
for policy in self.policies:
|
||||||
|
if (
|
||||||
|
policy.setup_id == definition.setup_id
|
||||||
|
and policy.definition_sha256 == definition.definition_sha256
|
||||||
|
):
|
||||||
|
if (
|
||||||
|
policy.definition_id != definition.definition_id
|
||||||
|
or policy.definition_version != definition.version
|
||||||
|
):
|
||||||
|
raise PortableResultPublicationBlockedError(
|
||||||
|
"calculation profile policy disagrees with the RunDefinition"
|
||||||
|
)
|
||||||
|
return policy
|
||||||
|
raise PortableResultPublicationBlockedError(
|
||||||
|
"exact calculation profile policy is not registered"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableResultValidationContext:
|
||||||
|
"""Read-only inputs supplied to one exact result-contract validator."""
|
||||||
|
|
||||||
|
manifest: PortableResultPackageManifest
|
||||||
|
job: ObservatoryRecordedJob
|
||||||
|
definition: PortableRunDefinition
|
||||||
|
result_document: Mapping[str, object]
|
||||||
|
artifact_paths: Mapping[str, Path]
|
||||||
|
|
||||||
|
|
||||||
|
type PortableResultContractValidator = Callable[[PortableResultValidationContext], None]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableResultContractValidatorRegistration:
|
||||||
|
contract_sha256: str
|
||||||
|
validator: PortableResultContractValidator
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
digest(self.contract_sha256, "portable result validator contract sha256")
|
||||||
|
if not callable(self.validator):
|
||||||
|
raise ValueError("portable result contract validator is not callable")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableResultContractValidatorRegistry:
|
||||||
|
registrations: tuple[PortableResultContractValidatorRegistration, ...]
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
digests = tuple(registration.contract_sha256 for registration in self.registrations)
|
||||||
|
if len(digests) != len(set(digests)):
|
||||||
|
raise ValueError("portable result contract validators are not unique")
|
||||||
|
|
||||||
|
def resolve(self, contract_sha256: str) -> PortableResultContractValidator:
|
||||||
|
digest(contract_sha256, "portable result contract sha256")
|
||||||
|
for registration in self.registrations:
|
||||||
|
if registration.contract_sha256 == contract_sha256:
|
||||||
|
return registration.validator
|
||||||
|
raise PortableResultPublicationBlockedError(
|
||||||
|
"exact portable result-contract validator is not installed"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def job_identity_document(job: ObservatoryRecordedJob) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"job_id": job.job_id,
|
||||||
|
"request_sha256": job.request_sha256,
|
||||||
|
"identity_sha256": job.identity_sha256,
|
||||||
|
"submission_receipt_sha256": job.submission_receipt_sha256,
|
||||||
|
"claim_generation": job.claim_generation,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def source_identity_document(job: ObservatoryRecordedJob) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"session_id": job.source_session_id,
|
||||||
|
"catalog_sha256": job.source_catalog_sha256,
|
||||||
|
"bundle_sha256": job.source_bundle_sha256,
|
||||||
|
"capability_manifest_sha256": job.source_capability_manifest_sha256,
|
||||||
|
"adapter": {
|
||||||
|
"adapter_id": job.source_adapter_id,
|
||||||
|
"version": job.source_adapter_version,
|
||||||
|
"adapter_sha256": job.source_adapter_sha256,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def run_definition_document(definition: PortableRunDefinition) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
**definition.identity_document(),
|
||||||
|
"definition_sha256": definition.definition_sha256,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def result_identity_document(
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
result_id: str,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
contract = definition.result_contract
|
||||||
|
return {
|
||||||
|
"result_id": result_id,
|
||||||
|
"result_schema": contract.result_schema,
|
||||||
|
"result_kind": contract.result_kind,
|
||||||
|
"result_contract_sha256": contract.contract_sha256,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _package_identity_document(
|
||||||
|
*,
|
||||||
|
created_at_utc: str,
|
||||||
|
job: Mapping[str, object],
|
||||||
|
source: Mapping[str, object],
|
||||||
|
run_definition: Mapping[str, object],
|
||||||
|
result: Mapping[str, object],
|
||||||
|
authority: Mapping[str, object],
|
||||||
|
artifacts: Sequence[PortableResultArtifact],
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"schema_version": PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
|
||||||
|
"created_at_utc": created_at_utc,
|
||||||
|
"job": dict(job),
|
||||||
|
"source": dict(source),
|
||||||
|
"run_definition": dict(run_definition),
|
||||||
|
"result": dict(result),
|
||||||
|
"authority": dict(authority),
|
||||||
|
"artifacts": [artifact.as_dict() for artifact in artifacts],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _artifact(value: object) -> PortableResultArtifact:
|
||||||
|
row = object_document(value, "portable result artifact")
|
||||||
|
exact_keys(
|
||||||
|
row,
|
||||||
|
{"role", "relative_path", "media_type", "byte_length", "sha256"},
|
||||||
|
"portable result artifact",
|
||||||
|
)
|
||||||
|
byte_length = row["byte_length"]
|
||||||
|
if not isinstance(byte_length, int) or isinstance(byte_length, bool):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result artifact byte length is invalid"
|
||||||
|
)
|
||||||
|
return PortableResultArtifact(
|
||||||
|
role=string(row["role"], "portable result artifact role"),
|
||||||
|
relative_path=string(
|
||||||
|
row["relative_path"],
|
||||||
|
"portable result artifact relative path",
|
||||||
|
),
|
||||||
|
media_type=string(row["media_type"], "portable result artifact media type"),
|
||||||
|
byte_length=byte_length,
|
||||||
|
sha256=string(row["sha256"], "portable result artifact sha256"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def relative_artifact_path(value: object) -> PurePosixPath:
|
||||||
|
if not isinstance(value, str) or not 1 <= len(value) <= 512:
|
||||||
|
raise ValueError("portable result artifact path is invalid")
|
||||||
|
path = PurePosixPath(value)
|
||||||
|
if (
|
||||||
|
path.is_absolute()
|
||||||
|
or path.as_posix() != value
|
||||||
|
or len(path.parts) < 2
|
||||||
|
or path.parts[0] != "artifacts"
|
||||||
|
or any(
|
||||||
|
part in {"", ".", ".."} or _PATH_COMPONENT.fullmatch(part) is None
|
||||||
|
for part in path.parts
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise ValueError("portable result artifact path is invalid")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _media_type(value: object) -> str:
|
||||||
|
if (
|
||||||
|
not isinstance(value, str)
|
||||||
|
or not 3 <= len(value) <= 255
|
||||||
|
or "/" not in value
|
||||||
|
or value != value.strip()
|
||||||
|
or any(ord(character) < 32 or ord(character) > 126 for character in value)
|
||||||
|
):
|
||||||
|
raise ValueError("portable result artifact media type is invalid")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def canonical_json(value: object) -> bytes:
|
||||||
|
try:
|
||||||
|
return json.dumps(
|
||||||
|
value,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
allow_nan=False,
|
||||||
|
).encode("utf-8")
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package is not JSON-compatible"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def object_document(value: object, label: str) -> dict[str, object]:
|
||||||
|
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||||
|
raise PortableResultPackageIntegrityError(f"{label} must be an object")
|
||||||
|
return cast(dict[str, object], value)
|
||||||
|
|
||||||
|
|
||||||
|
def exact_keys(value: Mapping[str, object], expected: set[str], label: str) -> None:
|
||||||
|
if set(value) != expected:
|
||||||
|
raise PortableResultPackageIntegrityError(f"{label} fields are invalid")
|
||||||
|
|
||||||
|
|
||||||
|
def string(value: object, label: str) -> str:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise PortableResultPackageIntegrityError(f"{label} must be text")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _pattern(value: object, pattern: re.Pattern[str], label: str) -> str:
|
||||||
|
if not isinstance(value, str) or pattern.fullmatch(value) is None:
|
||||||
|
raise ValueError(f"{label} is invalid")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def digest(value: object, label: str) -> str:
|
||||||
|
return _pattern(value, _SHA256, label)
|
||||||
|
|
||||||
|
|
||||||
|
def _text(value: object, label: str, *, maximum: int) -> str:
|
||||||
|
if (
|
||||||
|
not isinstance(value, str)
|
||||||
|
or not 1 <= len(value) <= maximum
|
||||||
|
or value != value.strip()
|
||||||
|
):
|
||||||
|
raise ValueError(f"{label} is invalid")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _timestamp(value: object, label: str) -> str:
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise ValueError(f"{label} is invalid")
|
||||||
|
try:
|
||||||
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise ValueError(f"{label} is invalid") from exc
|
||||||
|
if parsed.tzinfo is None:
|
||||||
|
raise ValueError(f"{label} must include a timezone")
|
||||||
|
return value
|
||||||
@@ -0,0 +1,809 @@
|
|||||||
|
"""Verified publication boundary for portable recorded Observatory results.
|
||||||
|
|
||||||
|
A Worker success acknowledgement is only a transport receipt. This module
|
||||||
|
admits a result into the Session catalog only after a canonical result package
|
||||||
|
is bound to the exact durable job, persisted source contracts, portable
|
||||||
|
RunDefinition, result-contract validator, and observation-only authority.
|
||||||
|
|
||||||
|
The publisher deliberately has no production default validators or presentation
|
||||||
|
policy. An unknown result contract or definition-bound calculation profile is
|
||||||
|
therefore a hard publication blocker rather than an invitation to infer one from
|
||||||
|
the current UI selection or a historical LAB label.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import stat
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import cast
|
||||||
|
|
||||||
|
from k1link.artifact_gateway import (
|
||||||
|
ArtifactGatewayError,
|
||||||
|
ArtifactManifest,
|
||||||
|
ArtifactMember,
|
||||||
|
CentralArtifactStore,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_result_contract import (
|
||||||
|
OBSERVATION_ONLY_AUTHORITY,
|
||||||
|
PORTABLE_RESULT_PACKAGE_SCHEMA,
|
||||||
|
PORTABLE_RESULT_PUBLICATION_SCHEMA,
|
||||||
|
RESULT_DOCUMENT_ROLE,
|
||||||
|
RESULT_PACKAGE_MANIFEST_NAME,
|
||||||
|
RESULT_PACKAGE_MANIFEST_ROLE,
|
||||||
|
PortableCalculationProfilePolicy,
|
||||||
|
PortableCalculationProfileRegistry,
|
||||||
|
PortableResultArtifact,
|
||||||
|
PortableResultContractValidatorRegistry,
|
||||||
|
PortableResultPackageIntegrityError,
|
||||||
|
PortableResultPackageManifest,
|
||||||
|
PortableResultPublicationBlockedError,
|
||||||
|
PortableResultPublisherError,
|
||||||
|
PortableResultValidationContext,
|
||||||
|
canonical_json,
|
||||||
|
exact_keys,
|
||||||
|
job_identity_document,
|
||||||
|
object_document,
|
||||||
|
relative_artifact_path,
|
||||||
|
result_identity_document,
|
||||||
|
run_definition_document,
|
||||||
|
source_identity_document,
|
||||||
|
string,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_result_contract import (
|
||||||
|
digest as validate_digest,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_run_definitions import (
|
||||||
|
PortableRunDefinition,
|
||||||
|
PortableRunDefinitionRegistry,
|
||||||
|
PortableRunDefinitionRegistryError,
|
||||||
|
canonical_sha256,
|
||||||
|
)
|
||||||
|
from k1link.observatory.recorded_jobs import ObservatoryRecordedJob
|
||||||
|
from k1link.observatory.source_admission import (
|
||||||
|
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||||
|
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||||
|
PORTABLE_SOURCE_DOCUMENT_DIRECTORY,
|
||||||
|
)
|
||||||
|
from k1link.sessions.models import LabSessionBinding, SessionIntegrityError, SessionSummary
|
||||||
|
from k1link.sessions.store import SessionStore
|
||||||
|
|
||||||
|
_COPY_CHUNK_BYTES = 1024 * 1024
|
||||||
|
_FULL_ROUTE_LABEL = "полный маршрут и воспроизведение"
|
||||||
|
_LEGACY_CANONICAL_RESULT = re.compile(
|
||||||
|
r"^lab-v1-vegetation-shadow-[a-f0-9]{64}$"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PublishedPortableObservatoryResult:
|
||||||
|
binding: LabSessionBinding
|
||||||
|
package: PortableResultPackageManifest
|
||||||
|
artifact_manifest: ArtifactManifest
|
||||||
|
|
||||||
|
|
||||||
|
class PortableObservatoryResultPublisher:
|
||||||
|
"""Validate, archive, and project one exact portable Worker result."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
session_store: SessionStore,
|
||||||
|
artifact_store: CentralArtifactStore,
|
||||||
|
definitions: PortableRunDefinitionRegistry,
|
||||||
|
calculation_profiles: PortableCalculationProfileRegistry,
|
||||||
|
validators: PortableResultContractValidatorRegistry,
|
||||||
|
) -> None:
|
||||||
|
self._session_store = session_store
|
||||||
|
self._artifact_store = artifact_store
|
||||||
|
self._definitions = definitions
|
||||||
|
self._calculation_profiles = calculation_profiles
|
||||||
|
self._validators = validators
|
||||||
|
|
||||||
|
def publish(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
package_root: Path,
|
||||||
|
) -> PublishedPortableObservatoryResult:
|
||||||
|
"""Publish only a verified terminal package; exact retries are idempotent."""
|
||||||
|
|
||||||
|
_verify_terminal_job(job)
|
||||||
|
try:
|
||||||
|
definition = self._definitions.resolve(job.setup_id, job.definition_sha256)
|
||||||
|
except (PortableRunDefinitionRegistryError, ValueError) as exc:
|
||||||
|
raise PortableResultPublicationBlockedError(
|
||||||
|
"recorded job RunDefinition is not in the portable registry"
|
||||||
|
) from exc
|
||||||
|
_verify_definition_job_identity(definition, job)
|
||||||
|
profile = self._calculation_profiles.resolve(definition)
|
||||||
|
validator = self._validators.resolve(definition.result_contract.contract_sha256)
|
||||||
|
|
||||||
|
root, manifest_path, package = _read_package(package_root)
|
||||||
|
_verify_package_identity(package, job=job, definition=definition)
|
||||||
|
if package.manifest_sha256 != job.result_sha256 or root.name != job.result_sha256:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package content address disagrees with queue success"
|
||||||
|
)
|
||||||
|
source_summary = self._verify_source(job, definition)
|
||||||
|
artifact_paths = _verify_package_artifacts(root, package.artifacts)
|
||||||
|
result_document = _read_canonical_result_document(
|
||||||
|
artifact_paths[RESULT_DOCUMENT_ROLE]
|
||||||
|
)
|
||||||
|
context = PortableResultValidationContext(
|
||||||
|
manifest=package,
|
||||||
|
job=job,
|
||||||
|
definition=definition,
|
||||||
|
result_document=result_document,
|
||||||
|
artifact_paths=artifact_paths,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
validator(context)
|
||||||
|
except PortableResultPublisherError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result document failed its exact contract validator"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
artifact_manifest = self._archive_package(
|
||||||
|
job=job,
|
||||||
|
package=package,
|
||||||
|
manifest_path=manifest_path,
|
||||||
|
artifact_paths=artifact_paths,
|
||||||
|
profile=profile,
|
||||||
|
)
|
||||||
|
provenance = _publication_provenance(
|
||||||
|
job=job,
|
||||||
|
definition=definition,
|
||||||
|
package=package,
|
||||||
|
artifact_manifest=artifact_manifest,
|
||||||
|
profile=profile,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
binding = self._session_store.publish_lab_instance(
|
||||||
|
session_id=cast(str, job.result_id),
|
||||||
|
source_session_id=job.source_session_id,
|
||||||
|
display_name=_portable_result_display_name(source_summary),
|
||||||
|
lab_id=profile.lab_id,
|
||||||
|
result_kind=definition.result_contract.result_kind,
|
||||||
|
result_id=cast(str, job.result_id),
|
||||||
|
config_sha256=definition.definition_sha256,
|
||||||
|
run_created_at_utc=job.updated_at_utc,
|
||||||
|
replay_capability=None,
|
||||||
|
provenance=provenance,
|
||||||
|
include_recorded_media=profile.include_recorded_media,
|
||||||
|
expected_source_catalog_sha256=job.source_catalog_sha256,
|
||||||
|
)
|
||||||
|
except (SessionIntegrityError, ValueError) as exc:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result could not be projected as immutable LAB provenance"
|
||||||
|
) from exc
|
||||||
|
return PublishedPortableObservatoryResult(
|
||||||
|
binding=binding,
|
||||||
|
package=package,
|
||||||
|
artifact_manifest=artifact_manifest,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _verify_source(
|
||||||
|
self,
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
) -> SessionSummary:
|
||||||
|
try:
|
||||||
|
detail, catalog_sha256 = (
|
||||||
|
self._session_store.get_session_with_catalog_snapshot(
|
||||||
|
job.source_session_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result source session is unavailable"
|
||||||
|
) from exc
|
||||||
|
if (
|
||||||
|
detail.summary.session_id != job.source_session_id
|
||||||
|
or detail.summary.lab is not None
|
||||||
|
or catalog_sha256 != job.source_catalog_sha256
|
||||||
|
):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result source catalog changed after admission"
|
||||||
|
)
|
||||||
|
_verify_source_documents(
|
||||||
|
self._session_store.data_dir,
|
||||||
|
job=job,
|
||||||
|
definition=definition,
|
||||||
|
)
|
||||||
|
return detail.summary
|
||||||
|
|
||||||
|
def _archive_package(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
package: PortableResultPackageManifest,
|
||||||
|
manifest_path: Path,
|
||||||
|
artifact_paths: Mapping[str, Path],
|
||||||
|
profile: PortableCalculationProfilePolicy,
|
||||||
|
) -> ArtifactManifest:
|
||||||
|
members = [
|
||||||
|
_publish_exact_member(
|
||||||
|
self._artifact_store,
|
||||||
|
role=RESULT_PACKAGE_MANIFEST_ROLE,
|
||||||
|
media_type="application/json",
|
||||||
|
source=manifest_path,
|
||||||
|
expected_sha256=cast(str, job.result_sha256),
|
||||||
|
expected_byte_length=len(package.canonical_bytes),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
for artifact in package.artifacts:
|
||||||
|
members.append(
|
||||||
|
_publish_exact_member(
|
||||||
|
self._artifact_store,
|
||||||
|
role=artifact.role,
|
||||||
|
media_type=artifact.media_type,
|
||||||
|
source=artifact_paths[artifact.role],
|
||||||
|
expected_sha256=artifact.sha256,
|
||||||
|
expected_byte_length=artifact.byte_length,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
archived = self._artifact_store.publish_manifest(
|
||||||
|
artifact_type="observatory-portable-result",
|
||||||
|
subject_id=cast(str, job.result_id),
|
||||||
|
members=members,
|
||||||
|
metadata={
|
||||||
|
"package-sha256": cast(str, job.result_sha256),
|
||||||
|
"package-identity-sha256": package.identity_sha256,
|
||||||
|
"job-id": job.job_id,
|
||||||
|
"job-identity-sha256": job.identity_sha256,
|
||||||
|
"definition-sha256": job.definition_sha256,
|
||||||
|
"source-bundle-sha256": job.source_bundle_sha256,
|
||||||
|
"result-contract-sha256": string(
|
||||||
|
package.result["result_contract_sha256"],
|
||||||
|
"portable result contract sha256",
|
||||||
|
),
|
||||||
|
"calculation-profile-sha256": profile.identity_sha256,
|
||||||
|
},
|
||||||
|
created_at_utc=job.updated_at_utc,
|
||||||
|
)
|
||||||
|
verified = self._artifact_store.read_manifest(archived.manifest_id)
|
||||||
|
except (ArtifactGatewayError, OSError, ValueError) as exc:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package could not be archived immutably"
|
||||||
|
) from exc
|
||||||
|
if verified != archived:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result artifact manifest changed after publication"
|
||||||
|
)
|
||||||
|
return archived
|
||||||
|
|
||||||
|
|
||||||
|
def _portable_result_display_name(source: SessionSummary) -> str:
|
||||||
|
"""Keep the result label source-owned; profile provenance is rendered separately."""
|
||||||
|
|
||||||
|
candidate = f"{source.display_name} · {_FULL_ROUTE_LABEL}"
|
||||||
|
return candidate if len(candidate) <= 160 else source.display_name
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_published_portable_calculation_profile(
|
||||||
|
summary: SessionSummary,
|
||||||
|
*,
|
||||||
|
definitions: PortableRunDefinitionRegistry,
|
||||||
|
calculation_profiles: PortableCalculationProfileRegistry,
|
||||||
|
) -> dict[str, object] | None:
|
||||||
|
"""Resolve only an exact, immutable portable publication profile.
|
||||||
|
|
||||||
|
Catalog projection must never infer profile identity from a selected setup,
|
||||||
|
a display label, or a current definition with the same setup id. Any drift
|
||||||
|
in the stored publication provenance therefore makes the profile absent.
|
||||||
|
"""
|
||||||
|
|
||||||
|
binding = summary.lab
|
||||||
|
if binding is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
provenance = object_document(
|
||||||
|
binding.provenance,
|
||||||
|
"portable result publication provenance",
|
||||||
|
)
|
||||||
|
exact_keys(
|
||||||
|
provenance,
|
||||||
|
{
|
||||||
|
"schema_version",
|
||||||
|
"authority",
|
||||||
|
"calculation_profile",
|
||||||
|
"calculation_profile_sha256",
|
||||||
|
"job",
|
||||||
|
"source",
|
||||||
|
"run_definition",
|
||||||
|
"result_package",
|
||||||
|
"storage",
|
||||||
|
"method",
|
||||||
|
},
|
||||||
|
"portable result publication provenance",
|
||||||
|
)
|
||||||
|
profile_document = object_document(
|
||||||
|
provenance["calculation_profile"],
|
||||||
|
"portable calculation profile",
|
||||||
|
)
|
||||||
|
run_definition = object_document(
|
||||||
|
provenance["run_definition"],
|
||||||
|
"portable result RunDefinition",
|
||||||
|
)
|
||||||
|
source = object_document(provenance["source"], "portable result source")
|
||||||
|
setup_id = string(profile_document.get("setup_id"), "portable setup id")
|
||||||
|
definition_sha256 = string(
|
||||||
|
run_definition.get("definition_sha256"),
|
||||||
|
"portable RunDefinition sha256",
|
||||||
|
)
|
||||||
|
definition = definitions.resolve(setup_id, definition_sha256)
|
||||||
|
profile = calculation_profiles.resolve(definition)
|
||||||
|
if (
|
||||||
|
provenance["schema_version"] != PORTABLE_RESULT_PUBLICATION_SCHEMA
|
||||||
|
or provenance["authority"] != OBSERVATION_ONLY_AUTHORITY
|
||||||
|
or provenance["calculation_profile_sha256"] != profile.identity_sha256
|
||||||
|
or profile_document != profile.as_dict()
|
||||||
|
or canonical_sha256(profile_document) != profile.identity_sha256
|
||||||
|
or run_definition != run_definition_document(definition)
|
||||||
|
or binding.session_id != summary.session_id
|
||||||
|
or binding.result_id != summary.session_id
|
||||||
|
or binding.source_session_id != source.get("session_id")
|
||||||
|
or binding.lab_id != profile.lab_id
|
||||||
|
or binding.config_sha256 != definition.definition_sha256
|
||||||
|
or binding.result_kind != definition.result_contract.result_kind
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
return profile.as_dict()
|
||||||
|
except (
|
||||||
|
KeyError,
|
||||||
|
PortableResultPublisherError,
|
||||||
|
PortableRunDefinitionRegistryError,
|
||||||
|
TypeError,
|
||||||
|
ValueError,
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_terminal_job(job: ObservatoryRecordedJob) -> None:
|
||||||
|
if (
|
||||||
|
job.state != "succeeded"
|
||||||
|
or job.result_id is None
|
||||||
|
or job.result_sha256 is None
|
||||||
|
or job.terminal_code != "result-sealed"
|
||||||
|
or job.terminal_claim_token_sha256 is None
|
||||||
|
or job.claim_generation < 1
|
||||||
|
or job.active_claim_token is not None
|
||||||
|
or job.active_claimant_id is not None
|
||||||
|
):
|
||||||
|
raise PortableResultPublicationBlockedError(
|
||||||
|
"recorded job has no exact terminal Worker result receipt"
|
||||||
|
)
|
||||||
|
if _LEGACY_CANONICAL_RESULT.fullmatch(job.result_id) is not None:
|
||||||
|
raise PortableResultPublicationBlockedError(
|
||||||
|
"legacy canonical result namespace is immutable and reserved"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_definition_job_identity(
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
) -> None:
|
||||||
|
if not definition.executor.ready:
|
||||||
|
raise PortableResultPublicationBlockedError(
|
||||||
|
"portable result RunDefinition executor is not installed"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
recorded = definition.to_recorded_run_definition()
|
||||||
|
except Exception as exc:
|
||||||
|
raise PortableResultPublicationBlockedError(
|
||||||
|
"portable result RunDefinition cannot produce a queue identity"
|
||||||
|
) from exc
|
||||||
|
expected = (
|
||||||
|
recorded.setup_id,
|
||||||
|
recorded.definition_id,
|
||||||
|
recorded.definition_version,
|
||||||
|
recorded.definition_sha256,
|
||||||
|
recorded.source_adapter_id,
|
||||||
|
recorded.source_adapter_version,
|
||||||
|
recorded.source_adapter_sha256,
|
||||||
|
recorded.executor_release_id,
|
||||||
|
recorded.executor_release_sha256,
|
||||||
|
recorded.executor_image_sha256,
|
||||||
|
recorded.model_release_ids,
|
||||||
|
recorded.model_manifest_sha256,
|
||||||
|
recorded.resource_profile_id,
|
||||||
|
recorded.resource_profile_sha256,
|
||||||
|
recorded.checkpoint_policy,
|
||||||
|
recorded.allowed_checkpoints,
|
||||||
|
)
|
||||||
|
actual = (
|
||||||
|
job.setup_id,
|
||||||
|
job.definition_id,
|
||||||
|
job.definition_version,
|
||||||
|
job.definition_sha256,
|
||||||
|
job.source_adapter_id,
|
||||||
|
job.source_adapter_version,
|
||||||
|
job.source_adapter_sha256,
|
||||||
|
job.executor_release_id,
|
||||||
|
job.executor_release_sha256,
|
||||||
|
job.executor_image_sha256,
|
||||||
|
job.model_release_ids,
|
||||||
|
job.model_manifest_sha256,
|
||||||
|
job.resource_profile_id,
|
||||||
|
job.resource_profile_sha256,
|
||||||
|
job.checkpoint_policy,
|
||||||
|
job.allowed_checkpoints,
|
||||||
|
)
|
||||||
|
if actual != expected or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"recorded job and portable RunDefinition identities disagree"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _read_package(
|
||||||
|
package_root: Path,
|
||||||
|
) -> tuple[Path, Path, PortableResultPackageManifest]:
|
||||||
|
candidate = package_root.expanduser().absolute()
|
||||||
|
try:
|
||||||
|
root_metadata = candidate.lstat()
|
||||||
|
root = candidate.resolve(strict=True)
|
||||||
|
except OSError as exc:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package root is unavailable"
|
||||||
|
) from exc
|
||||||
|
if stat.S_ISLNK(root_metadata.st_mode) or not stat.S_ISDIR(root_metadata.st_mode):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package root must be a regular directory"
|
||||||
|
)
|
||||||
|
manifest_path = root / RESULT_PACKAGE_MANIFEST_NAME
|
||||||
|
try:
|
||||||
|
metadata = manifest_path.lstat()
|
||||||
|
payload = manifest_path.read_bytes()
|
||||||
|
except OSError as exc:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package manifest is unavailable"
|
||||||
|
) from exc
|
||||||
|
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package manifest must be a regular file"
|
||||||
|
)
|
||||||
|
package = PortableResultPackageManifest.from_bytes(payload)
|
||||||
|
if hashlib.sha256(payload).hexdigest() != package.manifest_sha256:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package manifest digest changed"
|
||||||
|
)
|
||||||
|
return root, manifest_path, package
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_package_identity(
|
||||||
|
package: PortableResultPackageManifest,
|
||||||
|
*,
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
) -> None:
|
||||||
|
if package.job != job_identity_document(job):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package is bound to another queue job"
|
||||||
|
)
|
||||||
|
if package.source != source_identity_document(job):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package is bound to another source"
|
||||||
|
)
|
||||||
|
if package.run_definition != run_definition_document(definition):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package is bound to another RunDefinition"
|
||||||
|
)
|
||||||
|
if package.result != result_identity_document(definition, cast(str, job.result_id)):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package violates its sealed result contract"
|
||||||
|
)
|
||||||
|
if package.authority != definition.authority.as_dict():
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result package authority changed"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_source_documents(
|
||||||
|
data_dir: Path,
|
||||||
|
*,
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
) -> None:
|
||||||
|
root = data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY
|
||||||
|
try:
|
||||||
|
metadata = root.lstat()
|
||||||
|
except OSError as exc:
|
||||||
|
raise PortableResultPublicationBlockedError(
|
||||||
|
"admitted portable source documents are unavailable"
|
||||||
|
) from exc
|
||||||
|
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable source document root is unsafe"
|
||||||
|
)
|
||||||
|
bundle = _read_content_addressed_document(root, job.source_bundle_sha256)
|
||||||
|
capability = _read_content_addressed_document(
|
||||||
|
root,
|
||||||
|
job.source_capability_manifest_sha256,
|
||||||
|
)
|
||||||
|
exact_keys(
|
||||||
|
bundle,
|
||||||
|
{
|
||||||
|
"schema_version",
|
||||||
|
"source_session_id",
|
||||||
|
"source_catalog_sha256",
|
||||||
|
"plugin_id",
|
||||||
|
"archive_id",
|
||||||
|
"source_adapter",
|
||||||
|
"sources",
|
||||||
|
"spatial_replay",
|
||||||
|
"camera",
|
||||||
|
"authority",
|
||||||
|
},
|
||||||
|
"portable source bundle",
|
||||||
|
)
|
||||||
|
exact_keys(
|
||||||
|
capability,
|
||||||
|
{
|
||||||
|
"schema_version",
|
||||||
|
"source_session_id",
|
||||||
|
"source_catalog_sha256",
|
||||||
|
"source_bundle_sha256",
|
||||||
|
"source_adapter_sha256",
|
||||||
|
"modalities",
|
||||||
|
"camera_profile",
|
||||||
|
"calibration",
|
||||||
|
"authority",
|
||||||
|
},
|
||||||
|
"portable source capability",
|
||||||
|
)
|
||||||
|
expected_adapter = {
|
||||||
|
"id": job.source_adapter_id,
|
||||||
|
"version": job.source_adapter_version,
|
||||||
|
"sha256": job.source_adapter_sha256,
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
bundle["schema_version"] != PORTABLE_SOURCE_BUNDLE_SCHEMA
|
||||||
|
or bundle["source_session_id"] != job.source_session_id
|
||||||
|
or bundle["source_catalog_sha256"] != job.source_catalog_sha256
|
||||||
|
or bundle["plugin_id"] != definition.source_requirements.plugin_id
|
||||||
|
or bundle["archive_id"] != definition.source_requirements.archive_id
|
||||||
|
or bundle["source_adapter"] != expected_adapter
|
||||||
|
or bundle["authority"] != OBSERVATION_ONLY_AUTHORITY
|
||||||
|
or capability["schema_version"] != PORTABLE_SOURCE_CAPABILITY_SCHEMA
|
||||||
|
or capability["source_session_id"] != job.source_session_id
|
||||||
|
or capability["source_catalog_sha256"] != job.source_catalog_sha256
|
||||||
|
or capability["source_bundle_sha256"] != job.source_bundle_sha256
|
||||||
|
or capability["source_adapter_sha256"] != job.source_adapter_sha256
|
||||||
|
or capability["authority"] != OBSERVATION_ONLY_AUTHORITY
|
||||||
|
):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"persisted portable source contract disagrees with the queue job"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _read_content_addressed_document(root: Path, document_sha256: str) -> dict[str, object]:
|
||||||
|
validate_digest(document_sha256, "portable source document sha256")
|
||||||
|
path = root / f"{document_sha256}.json"
|
||||||
|
try:
|
||||||
|
metadata = path.lstat()
|
||||||
|
payload = path.read_bytes()
|
||||||
|
except OSError as exc:
|
||||||
|
raise PortableResultPublicationBlockedError(
|
||||||
|
"an admitted portable source document is unavailable"
|
||||||
|
) from exc
|
||||||
|
if (
|
||||||
|
stat.S_ISLNK(metadata.st_mode)
|
||||||
|
or not stat.S_ISREG(metadata.st_mode)
|
||||||
|
or hashlib.sha256(payload).hexdigest() != document_sha256
|
||||||
|
):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"admitted portable source document digest changed"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
decoded: object = json.loads(payload.decode("utf-8"))
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"admitted portable source document is not valid JSON"
|
||||||
|
) from exc
|
||||||
|
document = object_document(decoded, "portable source document")
|
||||||
|
if payload != canonical_json(document):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"admitted portable source document is not canonical JSON"
|
||||||
|
)
|
||||||
|
return document
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_package_artifacts(
|
||||||
|
root: Path,
|
||||||
|
artifacts: Sequence[PortableResultArtifact],
|
||||||
|
) -> dict[str, Path]:
|
||||||
|
resolved: dict[str, Path] = {}
|
||||||
|
for artifact in artifacts:
|
||||||
|
path = _resolve_package_member(root, artifact.relative_path)
|
||||||
|
digest, byte_length = _hash_file(path)
|
||||||
|
if digest != artifact.sha256 or byte_length != artifact.byte_length:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
f"portable result artifact content changed: {artifact.role}"
|
||||||
|
)
|
||||||
|
resolved[artifact.role] = path
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_package_member(root: Path, relative_path: str) -> Path:
|
||||||
|
portable = relative_artifact_path(relative_path)
|
||||||
|
candidate = root.joinpath(*portable.parts)
|
||||||
|
current = root
|
||||||
|
try:
|
||||||
|
for part in portable.parts:
|
||||||
|
current = current / part
|
||||||
|
metadata = current.lstat()
|
||||||
|
if stat.S_ISLNK(metadata.st_mode):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result artifact path contains a symlink"
|
||||||
|
)
|
||||||
|
resolved = candidate.resolve(strict=True)
|
||||||
|
metadata = candidate.lstat()
|
||||||
|
except PortableResultPublisherError:
|
||||||
|
raise
|
||||||
|
except OSError as exc:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result artifact is unavailable"
|
||||||
|
) from exc
|
||||||
|
if not resolved.is_relative_to(root) or not stat.S_ISREG(metadata.st_mode):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result artifact escapes its package"
|
||||||
|
)
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def _read_canonical_result_document(path: Path) -> dict[str, object]:
|
||||||
|
try:
|
||||||
|
payload = path.read_bytes()
|
||||||
|
decoded: object = json.loads(payload.decode("utf-8"))
|
||||||
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result document is not valid JSON"
|
||||||
|
) from exc
|
||||||
|
document = object_document(decoded, "portable result document")
|
||||||
|
if payload != canonical_json(document):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result document is not canonical JSON"
|
||||||
|
)
|
||||||
|
return document
|
||||||
|
|
||||||
|
|
||||||
|
def _publish_exact_member(
|
||||||
|
store: CentralArtifactStore,
|
||||||
|
*,
|
||||||
|
role: str,
|
||||||
|
media_type: str,
|
||||||
|
source: Path,
|
||||||
|
expected_sha256: str,
|
||||||
|
expected_byte_length: int,
|
||||||
|
) -> ArtifactMember:
|
||||||
|
try:
|
||||||
|
published = store.publish_file(source)
|
||||||
|
except (ArtifactGatewayError, OSError, ValueError) as exc:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
f"portable result artifact could not be archived: {role}"
|
||||||
|
) from exc
|
||||||
|
if (
|
||||||
|
published.sha256 != expected_sha256
|
||||||
|
or published.byte_length != expected_byte_length
|
||||||
|
):
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
f"portable result artifact changed while it was archived: {role}"
|
||||||
|
)
|
||||||
|
return ArtifactMember(
|
||||||
|
role=role,
|
||||||
|
media_type=media_type,
|
||||||
|
sha256=published.sha256,
|
||||||
|
byte_length=published.byte_length,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _publication_provenance(
|
||||||
|
*,
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
package: PortableResultPackageManifest,
|
||||||
|
artifact_manifest: ArtifactManifest,
|
||||||
|
profile: PortableCalculationProfilePolicy,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
result_document = next(
|
||||||
|
artifact for artifact in package.artifacts if artifact.role == RESULT_DOCUMENT_ROLE
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"schema_version": PORTABLE_RESULT_PUBLICATION_SCHEMA,
|
||||||
|
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||||
|
"calculation_profile": profile.as_dict(),
|
||||||
|
"calculation_profile_sha256": profile.identity_sha256,
|
||||||
|
"job": job_identity_document(job),
|
||||||
|
"source": source_identity_document(job),
|
||||||
|
"run_definition": run_definition_document(definition),
|
||||||
|
"result_package": {
|
||||||
|
"schema_version": PORTABLE_RESULT_PACKAGE_SCHEMA,
|
||||||
|
"manifest_sha256": package.manifest_sha256,
|
||||||
|
"identity_sha256": package.identity_sha256,
|
||||||
|
"artifact_manifest_id": artifact_manifest.manifest_id,
|
||||||
|
"result_document_sha256": result_document.sha256,
|
||||||
|
"artifacts": [artifact.as_dict() for artifact in package.artifacts],
|
||||||
|
},
|
||||||
|
"storage": {
|
||||||
|
"mode": "central-content-addressed-artifact-store",
|
||||||
|
"include_recorded_media": profile.include_recorded_media,
|
||||||
|
"replay_capability": None,
|
||||||
|
},
|
||||||
|
"method": _laboratory_method(job, definition),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _laboratory_method(
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
components: list[dict[str, object]] = [
|
||||||
|
{
|
||||||
|
"kind": "source",
|
||||||
|
"name": job.source_session_id,
|
||||||
|
"version": f"{job.source_adapter_id}/v{job.source_adapter_version}",
|
||||||
|
"role": "immutable admitted K1 source bundle",
|
||||||
|
"identity_sha256": job.source_bundle_sha256,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"kind": "algorithm",
|
||||||
|
"name": definition.definition_id,
|
||||||
|
"version": f"v{definition.version}",
|
||||||
|
"role": "portable laboratory RunDefinition",
|
||||||
|
"identity_sha256": definition.definition_sha256,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"kind": "runtime",
|
||||||
|
"name": job.executor_release_id,
|
||||||
|
"version": "sealed-release",
|
||||||
|
"role": "portable Worker executor",
|
||||||
|
"identity_sha256": job.executor_release_sha256,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"kind": "runtime",
|
||||||
|
"name": job.resource_profile_id,
|
||||||
|
"version": "sealed-resource-profile",
|
||||||
|
"role": "exclusive Worker resource contract",
|
||||||
|
"identity_sha256": job.resource_profile_sha256,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
components.extend(
|
||||||
|
{
|
||||||
|
"kind": "model",
|
||||||
|
"name": model.release_id,
|
||||||
|
"version": model.revision or "sealed-artifacts",
|
||||||
|
"role": "portable inference model",
|
||||||
|
"identity_sha256": model.identity_sha256,
|
||||||
|
}
|
||||||
|
for model in definition.models
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"schema_version": "missioncore.laboratory-method/v1",
|
||||||
|
"completeness": "complete",
|
||||||
|
"execution_class": "hybrid" if definition.models else "deterministic",
|
||||||
|
"pipeline_id": definition.definition_id,
|
||||||
|
"components": components,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _hash_file(path: Path) -> tuple[str, int]:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
byte_length = 0
|
||||||
|
try:
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
while chunk := stream.read(_COPY_CHUNK_BYTES):
|
||||||
|
digest.update(chunk)
|
||||||
|
byte_length += len(chunk)
|
||||||
|
except OSError as exc:
|
||||||
|
raise PortableResultPackageIntegrityError(
|
||||||
|
"portable result artifact could not be read"
|
||||||
|
) from exc
|
||||||
|
return digest.hexdigest(), byte_length
|
||||||
@@ -510,26 +510,46 @@ class PortableRunDefinition:
|
|||||||
by_kind: dict[str, list[ImmutableComponentIdentity]] = {}
|
by_kind: dict[str, list[ImmutableComponentIdentity]] = {}
|
||||||
for component in self.components:
|
for component in self.components:
|
||||||
by_kind.setdefault(component.kind, []).append(component)
|
by_kind.setdefault(component.kind, []).append(component)
|
||||||
for required_kind in (
|
# Calibration is common to every admitted K1 source. Learned-model
|
||||||
"calibration",
|
# definitions additionally keep the original profile/runner/FOV seals;
|
||||||
|
# algorithm-only definitions such as M4.9 may remain projectable while
|
||||||
|
# their portable runner is not installed yet.
|
||||||
|
required_kinds: tuple[str, ...] = ("calibration",)
|
||||||
|
if self.models:
|
||||||
|
required_kinds += (
|
||||||
|
"profile",
|
||||||
|
"runner",
|
||||||
|
"valid-fov-identity",
|
||||||
|
"valid-fov-mask",
|
||||||
|
)
|
||||||
|
for required_kind in required_kinds:
|
||||||
|
if len(by_kind.get(required_kind, [])) != 1:
|
||||||
|
raise PortableRunDefinitionRegistryError(
|
||||||
|
f"portable definition requires exactly one {required_kind} component"
|
||||||
|
)
|
||||||
|
for singleton_kind in (
|
||||||
"profile",
|
"profile",
|
||||||
"runner",
|
"runner",
|
||||||
"valid-fov-identity",
|
"valid-fov-identity",
|
||||||
"valid-fov-mask",
|
"valid-fov-mask",
|
||||||
):
|
):
|
||||||
if len(by_kind.get(required_kind, [])) != 1:
|
if len(by_kind.get(singleton_kind, [])) > 1:
|
||||||
raise PortableRunDefinitionRegistryError(
|
raise PortableRunDefinitionRegistryError(
|
||||||
f"portable definition requires exactly one {required_kind} component"
|
f"portable definition allows at most one {singleton_kind} component"
|
||||||
)
|
)
|
||||||
|
if not self.models and self.executor.ready and len(by_kind.get("runner", [])) != 1:
|
||||||
|
raise PortableRunDefinitionRegistryError(
|
||||||
|
"ready portable definition requires exactly one runner component"
|
||||||
|
)
|
||||||
calibration = by_kind["calibration"][0]
|
calibration = by_kind["calibration"][0]
|
||||||
if calibration.sha256 != self.source_requirements.calibration_identity_sha256:
|
if calibration.sha256 != self.source_requirements.calibration_identity_sha256:
|
||||||
raise PortableRunDefinitionRegistryError(
|
raise PortableRunDefinitionRegistryError(
|
||||||
"source capability and runtime calibration identities disagree"
|
"source capability and runtime calibration identities disagree"
|
||||||
)
|
)
|
||||||
model_ids = [model.release_id for model in self.models]
|
model_ids = [model.release_id for model in self.models]
|
||||||
if not model_ids or model_ids != sorted(model_ids) or len(model_ids) != len(set(model_ids)):
|
if model_ids != sorted(model_ids) or len(model_ids) != len(set(model_ids)):
|
||||||
raise PortableRunDefinitionRegistryError(
|
raise PortableRunDefinitionRegistryError(
|
||||||
"models must be non-empty, unique, and canonically ordered"
|
"models must be unique and canonically ordered"
|
||||||
)
|
)
|
||||||
if self.executor.contour_id != self.resource_profile.contour_id:
|
if self.executor.contour_id != self.resource_profile.contour_id:
|
||||||
raise PortableRunDefinitionRegistryError(
|
raise PortableRunDefinitionRegistryError(
|
||||||
@@ -720,13 +740,38 @@ class PortableRunDefinitionRegistry:
|
|||||||
"portable setup and definition identity are not allowlisted"
|
"portable setup and definition identity are not allowlisted"
|
||||||
)
|
)
|
||||||
|
|
||||||
def to_recorded_registry(self) -> RecordedRunDefinitionRegistry:
|
def resolve_setup(self, setup_id: str) -> PortableRunDefinition:
|
||||||
"""Convert the complete registry and fail if any definition is blocked."""
|
"""Resolve the single current definition projected for one setup."""
|
||||||
|
|
||||||
return RecordedRunDefinitionRegistry(
|
_pattern(setup_id, _IDENTIFIER, "setup id")
|
||||||
tuple(definition.to_recorded_run_definition() for definition in self.definitions)
|
for definition in self.definitions:
|
||||||
|
if definition.setup_id == setup_id:
|
||||||
|
return definition
|
||||||
|
raise PortableRunDefinitionRegistryError("portable setup is not allowlisted")
|
||||||
|
|
||||||
|
def ready_recorded_definitions(self) -> tuple[RecordedRunDefinition, ...]:
|
||||||
|
"""Return only definitions with fully sealed, installed executors.
|
||||||
|
|
||||||
|
Blocked definitions remain valid catalog entries and do not prevent an
|
||||||
|
unrelated ready definition from entering the durable queue allowlist.
|
||||||
|
"""
|
||||||
|
|
||||||
|
return tuple(
|
||||||
|
definition.to_recorded_run_definition()
|
||||||
|
for definition in self.definitions
|
||||||
|
if definition.executor.ready
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def to_recorded_registry(self) -> RecordedRunDefinitionRegistry:
|
||||||
|
"""Build a queue registry from ready definitions, ignoring blocked ones."""
|
||||||
|
|
||||||
|
definitions = self.ready_recorded_definitions()
|
||||||
|
if not definitions:
|
||||||
|
raise PortableRunDefinitionUnavailableError(
|
||||||
|
"portable registry has no sealed and installed executor"
|
||||||
|
)
|
||||||
|
return RecordedRunDefinitionRegistry(definitions)
|
||||||
|
|
||||||
|
|
||||||
def canonical_sha256(value: object) -> str:
|
def canonical_sha256(value: object) -> str:
|
||||||
"""Return the repository-wide canonical JSON SHA-256 identity."""
|
"""Return the repository-wide canonical JSON SHA-256 identity."""
|
||||||
|
|||||||
@@ -1,15 +1,9 @@
|
|||||||
"""UI-ready Observatory projection for the portable LAB V1 definition.
|
"""UI-ready Observatory projection for source-independent portable setups.
|
||||||
|
|
||||||
This module deliberately does not extend the legacy setup registry and does
|
The projector keeps source capability, executor availability, and dispatch
|
||||||
not submit work. It projects two independent facts for one selected source:
|
availability separate. One blocked definition therefore remains visible
|
||||||
|
without hiding another definition or weakening either definition's admission
|
||||||
* the result of a lightweight recorded-source capability probe;
|
contract. Historical LAB results remain exclusively in the legacy catalog.
|
||||||
* the executor state sealed by the portable RunDefinition;
|
|
||||||
|
|
||||||
Keeping those facts separate prevents a compatible new recording from being
|
|
||||||
described as incompatible merely because the executor is not installed yet.
|
|
||||||
Historical vegetation-shadow results belong only to the legacy catalog and
|
|
||||||
never become exact results of the generic portable v2 definition.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -17,11 +11,16 @@ from __future__ import annotations
|
|||||||
import re
|
import re
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Final, Protocol, runtime_checkable
|
from typing import Final, Protocol
|
||||||
|
|
||||||
|
from k1link.observatory.portable_result_contract import (
|
||||||
|
PortableCalculationProfilePolicy,
|
||||||
|
PortableCalculationProfileRegistry,
|
||||||
|
)
|
||||||
from k1link.observatory.portable_run_definitions import (
|
from k1link.observatory.portable_run_definitions import (
|
||||||
PortableRunDefinition,
|
PortableRunDefinition,
|
||||||
PortableRunDefinitionRegistry,
|
PortableRunDefinitionRegistry,
|
||||||
|
PortableRunDefinitionRegistryError,
|
||||||
)
|
)
|
||||||
from k1link.observatory.source_admission import (
|
from k1link.observatory.source_admission import (
|
||||||
PortableRecordedSourceCapability,
|
PortableRecordedSourceCapability,
|
||||||
@@ -34,6 +33,8 @@ PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA: Final = (
|
|||||||
)
|
)
|
||||||
PORTABLE_LAB_V1_SETUP_ID: Final = "lab-v1-eomt-ddrnet-portable-v1"
|
PORTABLE_LAB_V1_SETUP_ID: Final = "lab-v1-eomt-ddrnet-portable-v1"
|
||||||
PORTABLE_LAB_V1_DISPLAY_NAME: Final = "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39"
|
PORTABLE_LAB_V1_DISPLAY_NAME: Final = "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39"
|
||||||
|
PORTABLE_M49_SETUP_ID: Final = "m49-tgs-portable-v2"
|
||||||
|
PORTABLE_M49_DISPLAY_NAME: Final = "M4.9T5 · TRAVEL TGS · CPU-only, без ML"
|
||||||
|
|
||||||
_SOURCE_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
_SOURCE_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||||
_OBSERVATION_ONLY_AUTHORITY: Final = {
|
_OBSERVATION_ONLY_AUTHORITY: Final = {
|
||||||
@@ -43,6 +44,25 @@ _OBSERVATION_ONLY_AUTHORITY: Final = {
|
|||||||
"production_accepted": False,
|
"production_accepted": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_SETUP_PRESENTATION: Final = {
|
||||||
|
PORTABLE_LAB_V1_SETUP_ID: {
|
||||||
|
"lab_id": "LAB V1",
|
||||||
|
"display_name": PORTABLE_LAB_V1_DISPLAY_NAME,
|
||||||
|
"description": ("Проверка записанной K1-сессии моделями EoMT и DDRNet; только наблюдение."),
|
||||||
|
"compatible": "Запись соответствует требованиям EoMT + DDRNet.",
|
||||||
|
"incompatible": "Запись не соответствует требованиям EoMT + DDRNet.",
|
||||||
|
"executor_unavailable": "Вычислительный контур LAB V1 пока недоступен.",
|
||||||
|
},
|
||||||
|
PORTABLE_M49_SETUP_ID: {
|
||||||
|
"lab_id": "LAB M4.9T5",
|
||||||
|
"display_name": PORTABLE_M49_DISPLAY_NAME,
|
||||||
|
"description": ("Динамический TGS-разбор записанной K1-сессии без ML; только наблюдение."),
|
||||||
|
"compatible": "Запись соответствует требованиям TRAVEL TGS.",
|
||||||
|
"incompatible": "Запись не соответствует требованиям TRAVEL TGS.",
|
||||||
|
"executor_unavailable": "Переносимый вычислительный контур M4.9T5 пока недоступен.",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
_MODEL_PRESENTATION: Final = {
|
_MODEL_PRESENTATION: Final = {
|
||||||
"eomt-cityscapes-large-1024-v1": "EoMT Cityscapes Large 1024",
|
"eomt-cityscapes-large-1024-v1": "EoMT Cityscapes Large 1024",
|
||||||
"lab-v1-ddrnet-39-goose-fine-64-v1": "DDRNet-39",
|
"lab-v1-ddrnet-39-goose-fine-64-v1": "DDRNet-39",
|
||||||
@@ -50,82 +70,104 @@ _MODEL_PRESENTATION: Final = {
|
|||||||
|
|
||||||
|
|
||||||
class PortableSetupProjectionError(RuntimeError):
|
class PortableSetupProjectionError(RuntimeError):
|
||||||
"""The portable setup cannot be projected without weakening its contract."""
|
"""A portable setup cannot be projected without weakening its contract."""
|
||||||
|
|
||||||
|
|
||||||
@runtime_checkable
|
class PortableDefinitionCapabilityProbe(Protocol):
|
||||||
class PortableSourceCapabilityProbeService(Protocol):
|
"""Definition-bound lightweight capability probe."""
|
||||||
"""A definition-bound lightweight source-capability service."""
|
|
||||||
|
|
||||||
def probe(self, source_session_id: str) -> PortableRecordedSourceCapability:
|
def probe(
|
||||||
"""Probe one source without preparing media or persisting documents."""
|
self,
|
||||||
|
*,
|
||||||
|
source_session_id: str,
|
||||||
|
setup_id: str,
|
||||||
|
definition_sha256: str,
|
||||||
|
) -> PortableRecordedSourceCapability:
|
||||||
|
"""Probe one source against one exact portable definition."""
|
||||||
|
|
||||||
|
|
||||||
type PortableSourceCapabilityProbe = (
|
type PortableSourceCapabilityProbe = Callable[[str], PortableRecordedSourceCapability]
|
||||||
Callable[[str], PortableRecordedSourceCapability] | PortableSourceCapabilityProbeService
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class _SourceCompatibility:
|
class _SourceCompatibility:
|
||||||
compatible: bool
|
compatible: bool
|
||||||
capability: PortableRecordedSourceCapability | None
|
capability: PortableRecordedSourceCapability | None
|
||||||
|
reason: str
|
||||||
|
|
||||||
def as_dict(self) -> dict[str, object]:
|
def as_dict(self) -> dict[str, object]:
|
||||||
return {
|
return {
|
||||||
"outcome": "pass" if self.compatible else "blocked",
|
"outcome": "pass" if self.compatible else "blocked",
|
||||||
"compatible": self.compatible,
|
"compatible": self.compatible,
|
||||||
"reason": (
|
"reason": self.reason,
|
||||||
"Запись соответствует требованиям EoMT + DDRNet."
|
|
||||||
if self.compatible
|
|
||||||
else "Запись не соответствует требованиям EoMT + DDRNet."
|
|
||||||
),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class PortableLabV1SetupProjector:
|
class PortableSetupProjector:
|
||||||
"""Project the generic portable LAB V1 setup for one selected source."""
|
"""Project every allowlisted portable setup for one selected source."""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
registry: PortableRunDefinitionRegistry,
|
registry: PortableRunDefinitionRegistry,
|
||||||
capability_probe: PortableSourceCapabilityProbe,
|
capability_probe: PortableDefinitionCapabilityProbe,
|
||||||
|
dispatch_available: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._definition = _resolve_lab_v1_definition(registry)
|
if not hasattr(capability_probe, "probe"):
|
||||||
if not isinstance(
|
|
||||||
capability_probe,
|
|
||||||
PortableSourceCapabilityProbeService,
|
|
||||||
) and not callable(capability_probe):
|
|
||||||
raise PortableSetupProjectionError("portable source capability probe is unavailable")
|
raise PortableSetupProjectionError("portable source capability probe is unavailable")
|
||||||
|
self._registry = registry
|
||||||
self._capability_probe = capability_probe
|
self._capability_probe = capability_probe
|
||||||
_validate_model_presentation(self._definition)
|
self._dispatch_available = dispatch_available
|
||||||
if self._definition.authority.as_dict() != _OBSERVATION_ONLY_AUTHORITY:
|
for definition in registry.definitions:
|
||||||
raise PortableSetupProjectionError("portable LAB V1 authority is not observation-only")
|
_validate_model_presentation(definition)
|
||||||
|
if definition.authority.as_dict() != _OBSERVATION_ONLY_AUTHORITY:
|
||||||
|
raise PortableSetupProjectionError(
|
||||||
|
"portable setup authority is not observation-only"
|
||||||
|
)
|
||||||
|
|
||||||
|
def has_setup(self, setup_id: str) -> bool:
|
||||||
|
try:
|
||||||
|
self._registry.resolve_setup(setup_id)
|
||||||
|
except (PortableRunDefinitionRegistryError, ValueError):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
def catalog(self, source: SessionSummary) -> dict[str, object]:
|
def catalog(self, source: SessionSummary) -> dict[str, object]:
|
||||||
"""Return a one-setup v2 catalog projection for ``source``."""
|
"""Return all independent portable setup projections for ``source``."""
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"schema_version": PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA,
|
"schema_version": PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA,
|
||||||
"source_session_id": source.session_id,
|
"source_session_id": source.session_id,
|
||||||
"setups": [self.project(source)],
|
"setups": [
|
||||||
|
self.project(source, setup_id=definition.setup_id)
|
||||||
|
for definition in self._registry.definitions
|
||||||
|
],
|
||||||
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
||||||
}
|
}
|
||||||
|
|
||||||
def project(self, source: SessionSummary) -> dict[str, object]:
|
def project(
|
||||||
"""Return a strict, observation-only setup projection."""
|
self,
|
||||||
|
source: SessionSummary,
|
||||||
|
*,
|
||||||
|
setup_id: str,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
"""Return one strict observation-only portable setup projection."""
|
||||||
|
|
||||||
_validate_source_id(source.session_id)
|
_validate_source_id(source.session_id)
|
||||||
compatibility = self._probe_source(source.session_id)
|
try:
|
||||||
definition = self._definition
|
definition = self._registry.resolve_setup(setup_id)
|
||||||
|
except PortableRunDefinitionRegistryError as exc:
|
||||||
|
raise PortableSetupProjectionError("portable setup is unavailable") from exc
|
||||||
|
presentation = _presentation(definition)
|
||||||
|
compatibility = self._probe_source(definition, source.session_id, presentation)
|
||||||
executor = definition.executor
|
executor = definition.executor
|
||||||
|
submission_allowed = (
|
||||||
|
compatibility.compatible and executor.ready and self._dispatch_available
|
||||||
|
)
|
||||||
return {
|
return {
|
||||||
"setup_id": definition.setup_id,
|
"setup_id": definition.setup_id,
|
||||||
"display_name": PORTABLE_LAB_V1_DISPLAY_NAME,
|
"display_name": presentation["display_name"],
|
||||||
"description": (
|
"description": presentation["description"],
|
||||||
"Проверка записанной K1-сессии моделями EoMT и DDRNet; только наблюдение."
|
|
||||||
),
|
|
||||||
"origin": "portable-definition",
|
"origin": "portable-definition",
|
||||||
"source_requirements": definition.source_requirements.as_dict(),
|
"source_requirements": definition.source_requirements.as_dict(),
|
||||||
"run_definition": {
|
"run_definition": {
|
||||||
@@ -141,65 +183,169 @@ class PortableLabV1SetupProjector:
|
|||||||
"contour_id": executor.contour_id,
|
"contour_id": executor.contour_id,
|
||||||
"state": executor.state,
|
"state": executor.state,
|
||||||
"ready": executor.ready,
|
"ready": executor.ready,
|
||||||
|
"reason_code": executor.reason_code,
|
||||||
"reason": executor.reason,
|
"reason": executor.reason,
|
||||||
},
|
},
|
||||||
"existing_results": [],
|
"existing_results": [],
|
||||||
"preflight": {
|
"preflight": {
|
||||||
"outcome": "blocked",
|
"outcome": "ready" if submission_allowed else "blocked",
|
||||||
"action": "blocked",
|
"action": "check" if submission_allowed else "blocked",
|
||||||
"reason": self._preflight_reason(compatibility),
|
"reason": self._preflight_reason(
|
||||||
"submission_allowed": False,
|
definition,
|
||||||
|
compatibility,
|
||||||
|
presentation,
|
||||||
|
),
|
||||||
|
"submission_allowed": submission_allowed,
|
||||||
"existing_result_ids": [],
|
"existing_result_ids": [],
|
||||||
},
|
},
|
||||||
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
||||||
}
|
}
|
||||||
|
|
||||||
def _probe_source(self, source_session_id: str) -> _SourceCompatibility:
|
def _probe_source(
|
||||||
|
self,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
source_session_id: str,
|
||||||
|
presentation: dict[str, str],
|
||||||
|
) -> _SourceCompatibility:
|
||||||
try:
|
try:
|
||||||
if isinstance(
|
capability = self._capability_probe.probe(
|
||||||
self._capability_probe,
|
source_session_id=source_session_id,
|
||||||
PortableSourceCapabilityProbeService,
|
setup_id=definition.setup_id,
|
||||||
):
|
definition_sha256=definition.definition_sha256,
|
||||||
capability = self._capability_probe.probe(source_session_id)
|
)
|
||||||
else:
|
|
||||||
capability = self._capability_probe(source_session_id)
|
|
||||||
except PortableSourceAdmissionError:
|
except PortableSourceAdmissionError:
|
||||||
return _SourceCompatibility(compatible=False, capability=None)
|
return _SourceCompatibility(
|
||||||
|
compatible=False,
|
||||||
|
capability=None,
|
||||||
|
reason=presentation["incompatible"],
|
||||||
|
)
|
||||||
if not isinstance(capability, PortableRecordedSourceCapability):
|
if not isinstance(capability, PortableRecordedSourceCapability):
|
||||||
raise PortableSetupProjectionError("capability probe returned an invalid result")
|
raise PortableSetupProjectionError("capability probe returned an invalid result")
|
||||||
if capability.source_session_id != source_session_id:
|
if capability.source_session_id != source_session_id:
|
||||||
raise PortableSetupProjectionError(
|
raise PortableSetupProjectionError(
|
||||||
"capability probe is bound to another source session"
|
"capability probe is bound to another source session"
|
||||||
)
|
)
|
||||||
if capability.source_adapter_sha256 != self._definition.source_adapter.contract_sha256:
|
if capability.source_adapter_sha256 != definition.source_adapter.contract_sha256:
|
||||||
raise PortableSetupProjectionError("capability probe uses another source adapter")
|
raise PortableSetupProjectionError("capability probe uses another source adapter")
|
||||||
return _SourceCompatibility(compatible=True, capability=capability)
|
return _SourceCompatibility(
|
||||||
|
compatible=True,
|
||||||
def _preflight_reason(self, compatibility: _SourceCompatibility) -> str:
|
capability=capability,
|
||||||
if not compatibility.compatible:
|
reason=presentation["compatible"],
|
||||||
return "Запись не соответствует требованиям этого сетапа."
|
|
||||||
if not self._definition.executor.ready:
|
|
||||||
return "Вычислительный контур LAB V1 пока недоступен."
|
|
||||||
return (
|
|
||||||
"Server-side проверка definition/check SHA и постановка portable "
|
|
||||||
"LAB V1 в очередь пока недоступны."
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _preflight_reason(
|
||||||
|
self,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
compatibility: _SourceCompatibility,
|
||||||
|
presentation: dict[str, str],
|
||||||
|
) -> str:
|
||||||
|
if not compatibility.compatible:
|
||||||
|
return "Запись не соответствует требованиям этого сетапа."
|
||||||
|
if not definition.executor.ready:
|
||||||
|
return presentation["executor_unavailable"]
|
||||||
|
if not self._dispatch_available:
|
||||||
|
return "Server-side проверка и постановка portable-сетапа в очередь недоступны."
|
||||||
|
return "Сетап готов к server-side проверке источника перед постановкой в очередь."
|
||||||
|
|
||||||
def _resolve_lab_v1_definition(
|
|
||||||
|
class PortableLabV1SetupProjector(PortableSetupProjector):
|
||||||
|
"""Backward-compatible single-definition LAB V1 projector."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
registry: PortableRunDefinitionRegistry,
|
||||||
|
capability_probe: PortableSourceCapabilityProbe | object,
|
||||||
|
) -> None:
|
||||||
|
try:
|
||||||
|
definition = registry.resolve_setup(PORTABLE_LAB_V1_SETUP_ID)
|
||||||
|
except PortableRunDefinitionRegistryError as exc:
|
||||||
|
raise PortableSetupProjectionError(
|
||||||
|
"portable LAB V1 definition is unavailable or ambiguous"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
class _LegacyProbeAdapter:
|
||||||
|
def probe(
|
||||||
|
adapter_self,
|
||||||
|
*,
|
||||||
|
source_session_id: str,
|
||||||
|
setup_id: str,
|
||||||
|
definition_sha256: str,
|
||||||
|
) -> PortableRecordedSourceCapability:
|
||||||
|
del adapter_self
|
||||||
|
if (
|
||||||
|
setup_id != definition.setup_id
|
||||||
|
or definition_sha256 != definition.definition_sha256
|
||||||
|
):
|
||||||
|
raise PortableSetupProjectionError("portable LAB V1 definition changed")
|
||||||
|
probe_method = getattr(capability_probe, "probe", None)
|
||||||
|
if callable(probe_method):
|
||||||
|
capability = probe_method(source_session_id)
|
||||||
|
elif callable(capability_probe):
|
||||||
|
capability = capability_probe(source_session_id)
|
||||||
|
else:
|
||||||
|
raise PortableSetupProjectionError(
|
||||||
|
"portable source capability probe is unavailable"
|
||||||
|
)
|
||||||
|
if not isinstance(capability, PortableRecordedSourceCapability):
|
||||||
|
raise PortableSetupProjectionError(
|
||||||
|
"capability probe returned an invalid result"
|
||||||
|
)
|
||||||
|
return capability
|
||||||
|
|
||||||
|
super().__init__(
|
||||||
|
registry=PortableRunDefinitionRegistry((definition,)),
|
||||||
|
capability_probe=_LegacyProbeAdapter(),
|
||||||
|
dispatch_available=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def project(
|
||||||
|
self,
|
||||||
|
source: SessionSummary,
|
||||||
|
*,
|
||||||
|
setup_id: str = PORTABLE_LAB_V1_SETUP_ID,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return super().project(source, setup_id=setup_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _presentation(definition: PortableRunDefinition) -> dict[str, str]:
|
||||||
|
configured = _SETUP_PRESENTATION.get(definition.setup_id)
|
||||||
|
if configured is not None:
|
||||||
|
return dict(configured)
|
||||||
|
return {
|
||||||
|
"lab_id": "LAB PORTABLE",
|
||||||
|
"display_name": definition.setup_id,
|
||||||
|
"description": "Переносимый анализ записанной K1-сессии; только наблюдение.",
|
||||||
|
"compatible": "Запись соответствует требованиям сетапа.",
|
||||||
|
"incompatible": "Запись не соответствует требованиям сетапа.",
|
||||||
|
"executor_unavailable": "Вычислительный контур сетапа пока недоступен.",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def portable_calculation_profile_registry(
|
||||||
registry: PortableRunDefinitionRegistry,
|
registry: PortableRunDefinitionRegistry,
|
||||||
) -> PortableRunDefinition:
|
) -> PortableCalculationProfileRegistry:
|
||||||
matching = tuple(
|
"""Build exact publication policies from the server-owned presentation map."""
|
||||||
definition
|
|
||||||
for definition in registry.definitions
|
policies: list[PortableCalculationProfilePolicy] = []
|
||||||
if definition.setup_id == PORTABLE_LAB_V1_SETUP_ID
|
for definition in registry.definitions:
|
||||||
)
|
presentation = _presentation(definition)
|
||||||
if len(matching) != 1:
|
policies.append(
|
||||||
raise PortableSetupProjectionError("portable LAB V1 definition is unavailable or ambiguous")
|
PortableCalculationProfilePolicy(
|
||||||
return matching[0]
|
setup_id=definition.setup_id,
|
||||||
|
definition_id=definition.definition_id,
|
||||||
|
definition_version=definition.version,
|
||||||
|
definition_sha256=definition.definition_sha256,
|
||||||
|
lab_id=presentation["lab_id"],
|
||||||
|
display_name=presentation["display_name"],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return PortableCalculationProfileRegistry(tuple(policies))
|
||||||
|
|
||||||
|
|
||||||
def _validate_model_presentation(definition: PortableRunDefinition) -> None:
|
def _validate_model_presentation(definition: PortableRunDefinition) -> None:
|
||||||
|
if definition.setup_id != PORTABLE_LAB_V1_SETUP_ID:
|
||||||
|
return
|
||||||
releases = {model.release_id: model for model in definition.models}
|
releases = {model.release_id: model for model in definition.models}
|
||||||
if set(releases) != set(_MODEL_PRESENTATION):
|
if set(releases) != set(_MODEL_PRESENTATION):
|
||||||
raise PortableSetupProjectionError(
|
raise PortableSetupProjectionError(
|
||||||
@@ -217,15 +363,14 @@ def _validate_model_presentation(definition: PortableRunDefinition) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _project_models(definition: PortableRunDefinition) -> list[dict[str, object]]:
|
def _project_models(definition: PortableRunDefinition) -> list[dict[str, object]]:
|
||||||
by_release = {model.release_id: model for model in definition.models}
|
|
||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
"name": _MODEL_PRESENTATION[release_id],
|
"name": _MODEL_PRESENTATION.get(model.release_id, model.model_id),
|
||||||
"release_id": release_id,
|
"release_id": model.release_id,
|
||||||
"model_id": by_release[release_id].model_id,
|
"model_id": model.model_id,
|
||||||
"architecture": by_release[release_id].architecture,
|
"architecture": model.architecture,
|
||||||
}
|
}
|
||||||
for release_id in _MODEL_PRESENTATION
|
for model in definition.models
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,375 @@
|
|||||||
|
"""Fail-closed server composition for portable Observatory Worker jobs.
|
||||||
|
|
||||||
|
This module is deliberately only a composition boundary. It does not enable
|
||||||
|
the Worker router, install an executor, select commands, or grant production
|
||||||
|
authority. It binds the two admitted portable profiles to their exact result
|
||||||
|
contracts, then constructs the local artifact transport and verified result
|
||||||
|
publisher from server-owned dependencies.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import stat
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Final
|
||||||
|
|
||||||
|
from k1link.artifact_gateway import CentralArtifactStore
|
||||||
|
from k1link.observatory.m49_portable_result import (
|
||||||
|
M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||||
|
M49_PORTABLE_RESULT_SCHEMA,
|
||||||
|
validate_m49_portable_result,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_artifact_transport import (
|
||||||
|
PortableArtifactTransportError,
|
||||||
|
PortableObservatoryArtifactTransport,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_lab_v1_executor import (
|
||||||
|
PORTABLE_LAB_V1_RESULT_SCHEMA,
|
||||||
|
validate_lab_v1_result_v2,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_result_contract import (
|
||||||
|
OBSERVATION_ONLY_AUTHORITY,
|
||||||
|
PortableCalculationProfileRegistry,
|
||||||
|
PortableResultContractValidator,
|
||||||
|
PortableResultContractValidatorRegistration,
|
||||||
|
PortableResultContractValidatorRegistry,
|
||||||
|
PortableResultPublicationBlockedError,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_result_publisher import (
|
||||||
|
PortableObservatoryResultPublisher,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_run_definitions import (
|
||||||
|
PORTABLE_RESULT_CONTRACT_SCHEMA,
|
||||||
|
PortableRunDefinition,
|
||||||
|
PortableRunDefinitionRegistry,
|
||||||
|
PortableRunDefinitionRegistryError,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_setup_projection import (
|
||||||
|
PORTABLE_LAB_V1_SETUP_ID,
|
||||||
|
PORTABLE_M49_SETUP_ID,
|
||||||
|
portable_calculation_profile_registry,
|
||||||
|
)
|
||||||
|
from k1link.observatory.recorded_jobs import ObservatoryRecordedJobQueue
|
||||||
|
from k1link.sessions.media import RecordedMediaInspector
|
||||||
|
from k1link.sessions.store import SessionStore
|
||||||
|
|
||||||
|
PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256: Final = (
|
||||||
|
"b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a"
|
||||||
|
)
|
||||||
|
OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_SOURCE_CAS_ROOT"
|
||||||
|
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV: Final = (
|
||||||
|
"MISSIONCORE_OBSERVATORY_WORKER_RESULT_STAGING_ROOT"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PortableWorkerIntegrationError(RuntimeError):
|
||||||
|
"""The exact portable server composition cannot be constructed."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableWorkerStorageRoots:
|
||||||
|
"""Pre-provisioned server-owned roots for large portable artifacts."""
|
||||||
|
|
||||||
|
source_cas_root: Path
|
||||||
|
result_staging_root: Path
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_environment(
|
||||||
|
cls,
|
||||||
|
*,
|
||||||
|
artifact_store_root: Path,
|
||||||
|
environment: Mapping[str, str] | None = None,
|
||||||
|
) -> PortableWorkerStorageRoots:
|
||||||
|
"""Load both roots without creating anything on a missing mount."""
|
||||||
|
|
||||||
|
values = os.environ if environment is None else environment
|
||||||
|
return cls.from_paths(
|
||||||
|
artifact_store_root=artifact_store_root,
|
||||||
|
source_cas_root=_required_environment_path(
|
||||||
|
values,
|
||||||
|
OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV,
|
||||||
|
),
|
||||||
|
result_staging_root=_required_environment_path(
|
||||||
|
values,
|
||||||
|
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_paths(
|
||||||
|
cls,
|
||||||
|
*,
|
||||||
|
artifact_store_root: Path,
|
||||||
|
source_cas_root: Path,
|
||||||
|
result_staging_root: Path,
|
||||||
|
) -> PortableWorkerStorageRoots:
|
||||||
|
"""Validate existing, disjoint roots inside the central store boundary."""
|
||||||
|
|
||||||
|
artifact_store = _existing_canonical_directory(
|
||||||
|
artifact_store_root,
|
||||||
|
"central artifact store",
|
||||||
|
)
|
||||||
|
storage_boundary = _existing_canonical_directory(
|
||||||
|
artifact_store.parent,
|
||||||
|
"central artifact storage boundary",
|
||||||
|
)
|
||||||
|
_require_mounted_volume(storage_boundary)
|
||||||
|
source_cas = _existing_canonical_directory(
|
||||||
|
source_cas_root,
|
||||||
|
"portable source CAS",
|
||||||
|
)
|
||||||
|
result_staging = _existing_canonical_directory(
|
||||||
|
result_staging_root,
|
||||||
|
"portable result staging",
|
||||||
|
)
|
||||||
|
for label, root in (
|
||||||
|
("portable source CAS", source_cas),
|
||||||
|
("portable result staging", result_staging),
|
||||||
|
):
|
||||||
|
if root == storage_boundary or not root.is_relative_to(storage_boundary):
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
f"{label} must be inside the central artifact storage boundary"
|
||||||
|
)
|
||||||
|
if _paths_overlap(root, artifact_store):
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
f"{label} must be disjoint from the central artifact store"
|
||||||
|
)
|
||||||
|
if _paths_overlap(source_cas, result_staging):
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
"portable source CAS and result staging roots must be disjoint"
|
||||||
|
)
|
||||||
|
return cls(
|
||||||
|
source_cas_root=source_cas,
|
||||||
|
result_staging_root=result_staging,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableObservatoryWorkerIntegration:
|
||||||
|
"""Server-owned objects required by the optional Worker router."""
|
||||||
|
|
||||||
|
calculation_profiles: PortableCalculationProfileRegistry
|
||||||
|
validators: PortableResultContractValidatorRegistry
|
||||||
|
artifact_transport: PortableObservatoryArtifactTransport
|
||||||
|
result_publisher: PortableObservatoryResultPublisher
|
||||||
|
supported_setup_ids: tuple[str, ...]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _ValidatorSpec:
|
||||||
|
setup_id: str
|
||||||
|
definition_id: str
|
||||||
|
definition_version: int
|
||||||
|
contract_id: str
|
||||||
|
contract_version: int
|
||||||
|
result_schema: str
|
||||||
|
result_kind: str
|
||||||
|
contract_sha256: str
|
||||||
|
validator: PortableResultContractValidator
|
||||||
|
|
||||||
|
def expected_contract(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"schema_version": PORTABLE_RESULT_CONTRACT_SCHEMA,
|
||||||
|
"contract_id": self.contract_id,
|
||||||
|
"version": self.contract_version,
|
||||||
|
"result_schema": self.result_schema,
|
||||||
|
"result_kind": self.result_kind,
|
||||||
|
"publication": "observatory",
|
||||||
|
"contract_sha256": self.contract_sha256,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
_VALIDATOR_SPECS: Final = (
|
||||||
|
_ValidatorSpec(
|
||||||
|
setup_id=PORTABLE_LAB_V1_SETUP_ID,
|
||||||
|
definition_id="lab-v1-eomt-ddrnet-portable",
|
||||||
|
definition_version=2,
|
||||||
|
contract_id="recorded-eomt-ddrnet-review-v2",
|
||||||
|
contract_version=2,
|
||||||
|
result_schema=PORTABLE_LAB_V1_RESULT_SCHEMA,
|
||||||
|
result_kind="recorded-perception-qualification",
|
||||||
|
contract_sha256=PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256,
|
||||||
|
validator=validate_lab_v1_result_v2,
|
||||||
|
),
|
||||||
|
_ValidatorSpec(
|
||||||
|
setup_id=PORTABLE_M49_SETUP_ID,
|
||||||
|
definition_id="m49-tgs-portable",
|
||||||
|
definition_version=2,
|
||||||
|
contract_id="m49-tgs-portable-review-v2",
|
||||||
|
contract_version=2,
|
||||||
|
result_schema=M49_PORTABLE_RESULT_SCHEMA,
|
||||||
|
result_kind="recorded-perception-qualification",
|
||||||
|
contract_sha256=M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||||
|
validator=validate_m49_portable_result,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def portable_result_validator_registry(
|
||||||
|
definitions: PortableRunDefinitionRegistry,
|
||||||
|
) -> PortableResultContractValidatorRegistry:
|
||||||
|
"""Bind both product profiles to fixed result contracts and validators."""
|
||||||
|
|
||||||
|
registrations: list[PortableResultContractValidatorRegistration] = []
|
||||||
|
for spec in _VALIDATOR_SPECS:
|
||||||
|
try:
|
||||||
|
definition = definitions.resolve_setup(spec.setup_id)
|
||||||
|
except PortableRunDefinitionRegistryError as exc:
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
f"required portable setup is unavailable: {spec.setup_id}"
|
||||||
|
) from exc
|
||||||
|
_verify_validator_definition(definition, spec)
|
||||||
|
registrations.append(
|
||||||
|
PortableResultContractValidatorRegistration(
|
||||||
|
contract_sha256=spec.contract_sha256,
|
||||||
|
validator=spec.validator,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return PortableResultContractValidatorRegistry(tuple(registrations))
|
||||||
|
|
||||||
|
|
||||||
|
def build_portable_observatory_worker_integration(
|
||||||
|
*,
|
||||||
|
queue: ObservatoryRecordedJobQueue,
|
||||||
|
session_store: SessionStore,
|
||||||
|
media_inspector: RecordedMediaInspector,
|
||||||
|
definitions: PortableRunDefinitionRegistry,
|
||||||
|
artifact_store: CentralArtifactStore,
|
||||||
|
calculation_profiles: PortableCalculationProfileRegistry | None = None,
|
||||||
|
validators: PortableResultContractValidatorRegistry | None = None,
|
||||||
|
source_cas_root: Path | None = None,
|
||||||
|
result_staging_root: Path | None = None,
|
||||||
|
) -> PortableObservatoryWorkerIntegration:
|
||||||
|
"""Construct the dormant server foundation without enabling any route."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
if source_cas_root is None or result_staging_root is None:
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
"portable Worker storage roots must be explicitly configured"
|
||||||
|
)
|
||||||
|
storage_roots = PortableWorkerStorageRoots.from_paths(
|
||||||
|
artifact_store_root=artifact_store.root,
|
||||||
|
source_cas_root=source_cas_root,
|
||||||
|
result_staging_root=result_staging_root,
|
||||||
|
)
|
||||||
|
profiles = calculation_profiles or portable_calculation_profile_registry(definitions)
|
||||||
|
validator_registry = validators or portable_result_validator_registry(definitions)
|
||||||
|
_verify_composition(definitions, profiles, validator_registry)
|
||||||
|
transport = PortableObservatoryArtifactTransport(
|
||||||
|
queue=queue,
|
||||||
|
session_store=session_store,
|
||||||
|
media_inspector=media_inspector,
|
||||||
|
definitions=definitions,
|
||||||
|
source_cas_root=storage_roots.source_cas_root,
|
||||||
|
result_staging_root=storage_roots.result_staging_root,
|
||||||
|
)
|
||||||
|
except PortableWorkerIntegrationError:
|
||||||
|
raise
|
||||||
|
except (
|
||||||
|
PortableArtifactTransportError,
|
||||||
|
PortableResultPublicationBlockedError,
|
||||||
|
PortableRunDefinitionRegistryError,
|
||||||
|
OSError,
|
||||||
|
ValueError,
|
||||||
|
) as exc:
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
"portable Worker server composition is unavailable"
|
||||||
|
) from exc
|
||||||
|
publisher = PortableObservatoryResultPublisher(
|
||||||
|
session_store=session_store,
|
||||||
|
artifact_store=artifact_store,
|
||||||
|
definitions=definitions,
|
||||||
|
calculation_profiles=profiles,
|
||||||
|
validators=validator_registry,
|
||||||
|
)
|
||||||
|
return PortableObservatoryWorkerIntegration(
|
||||||
|
calculation_profiles=profiles,
|
||||||
|
validators=validator_registry,
|
||||||
|
artifact_transport=transport,
|
||||||
|
result_publisher=publisher,
|
||||||
|
supported_setup_ids=tuple(spec.setup_id for spec in _VALIDATOR_SPECS),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_validator_definition(
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
spec: _ValidatorSpec,
|
||||||
|
) -> None:
|
||||||
|
if (
|
||||||
|
definition.definition_id != spec.definition_id
|
||||||
|
or definition.version != spec.definition_version
|
||||||
|
or definition.result_contract.as_dict() != spec.expected_contract()
|
||||||
|
or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY
|
||||||
|
):
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
f"portable result contract changed for setup: {spec.setup_id}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_composition(
|
||||||
|
definitions: PortableRunDefinitionRegistry,
|
||||||
|
profiles: PortableCalculationProfileRegistry,
|
||||||
|
validators: PortableResultContractValidatorRegistry,
|
||||||
|
) -> None:
|
||||||
|
expected_validators = {spec.contract_sha256: spec.validator for spec in _VALIDATOR_SPECS}
|
||||||
|
for definition in definitions.definitions:
|
||||||
|
profiles.resolve(definition)
|
||||||
|
if definition.executor.ready:
|
||||||
|
try:
|
||||||
|
validators.resolve(definition.result_contract.contract_sha256)
|
||||||
|
except PortableResultPublicationBlockedError as exc:
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
"a ready portable definition has no exact result validator"
|
||||||
|
) from exc
|
||||||
|
for contract_sha256, expected in expected_validators.items():
|
||||||
|
if validators.resolve(contract_sha256) is not expected:
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
"portable result validator registration changed identity"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _required_environment_path(
|
||||||
|
environment: Mapping[str, str],
|
||||||
|
name: str,
|
||||||
|
) -> Path:
|
||||||
|
raw = environment.get(name)
|
||||||
|
if not isinstance(raw, str) or not raw or raw != raw.strip():
|
||||||
|
raise PortableWorkerIntegrationError(f"{name} is required")
|
||||||
|
path = Path(raw)
|
||||||
|
if not path.is_absolute():
|
||||||
|
raise PortableWorkerIntegrationError(f"{name} must be an absolute path")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _existing_canonical_directory(path: Path, label: str) -> Path:
|
||||||
|
candidate = path.expanduser().absolute()
|
||||||
|
try:
|
||||||
|
metadata = candidate.lstat()
|
||||||
|
resolved = candidate.resolve(strict=True)
|
||||||
|
except OSError as exc:
|
||||||
|
raise PortableWorkerIntegrationError(f"{label} is unavailable") from exc
|
||||||
|
if (
|
||||||
|
stat.S_ISLNK(metadata.st_mode)
|
||||||
|
or not stat.S_ISDIR(metadata.st_mode)
|
||||||
|
or resolved != candidate
|
||||||
|
):
|
||||||
|
raise PortableWorkerIntegrationError(f"{label} is not a canonical directory")
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def _paths_overlap(left: Path, right: Path) -> bool:
|
||||||
|
return left == right or left.is_relative_to(right) or right.is_relative_to(left)
|
||||||
|
|
||||||
|
|
||||||
|
def _require_mounted_volume(path: Path) -> None:
|
||||||
|
parts = path.parts
|
||||||
|
if len(parts) < 3 or parts[0] != os.sep or parts[1] != "Volumes":
|
||||||
|
return
|
||||||
|
mount_point = Path(os.sep, "Volumes", parts[2])
|
||||||
|
if not os.path.ismount(mount_point):
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
f"central artifact volume is not mounted: {mount_point}"
|
||||||
|
)
|
||||||
@@ -0,0 +1,969 @@
|
|||||||
|
"""Fail-closed local runtime contract for portable Observatory executors.
|
||||||
|
|
||||||
|
This module belongs on Worker 006, not in the browser or the K1 control path.
|
||||||
|
It binds one source-independent portable RunDefinition to locally verified
|
||||||
|
assets and to local Python adapters. A queued job can select an adapter only
|
||||||
|
through the four server-sealed executor digests; it can never supply a command,
|
||||||
|
path, environment variable, image reference, or priority.
|
||||||
|
|
||||||
|
The production candidate registry is deliberately allowed to contain blocked
|
||||||
|
candidates. Reusable historical assets are evidence, not an installed
|
||||||
|
executor. A blocked candidate cannot yield a Worker registration or execute a
|
||||||
|
job even when every reusable asset is present.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Final, Literal, Protocol
|
||||||
|
|
||||||
|
from k1link.observatory.portable_run_definitions import (
|
||||||
|
PortableRunDefinition,
|
||||||
|
PortableRunDefinitionRegistry,
|
||||||
|
canonical_sha256,
|
||||||
|
)
|
||||||
|
from k1link.observatory.worker_agent import (
|
||||||
|
ObservatoryWorkerExecutionResult,
|
||||||
|
ObservatoryWorkerExecutorIdentity,
|
||||||
|
SealedObservatoryRecordedJob,
|
||||||
|
)
|
||||||
|
|
||||||
|
PORTABLE_WORKER_RUNTIME_REGISTRY_SCHEMA: Final = (
|
||||||
|
"missioncore.observatory-portable-worker-runtime-registry/v1"
|
||||||
|
)
|
||||||
|
PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA: Final = (
|
||||||
|
"missioncore.observatory-portable-worker-runtime-candidate/v1"
|
||||||
|
)
|
||||||
|
PORTABLE_WORKER_RUNTIME_PLAN_SCHEMA: Final = (
|
||||||
|
"missioncore.observatory-portable-worker-runtime-plan/v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
_MAX_REGISTRY_BYTES: Final = 256 * 1024
|
||||||
|
_SHA256: Final = re.compile(r"^[a-f0-9]{64}$")
|
||||||
|
_IDENTIFIER: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||||
|
_ASSET_ID: Final = re.compile(r"^[a-z][a-z0-9.-]{2,127}$")
|
||||||
|
_SESSION_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||||
|
_JOB_ID: Final = re.compile(r"^observatory-run-[a-f0-9]{32}$")
|
||||||
|
|
||||||
|
_AUTHORITY: Final = {
|
||||||
|
"commands_enabled": False,
|
||||||
|
"actuation_allowed": False,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
"production_accepted": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
type CandidateState = Literal["blocked", "ready"]
|
||||||
|
type PhaseState = Literal["implemented", "missing"]
|
||||||
|
type RuntimeAssetKind = Literal[
|
||||||
|
"container-image",
|
||||||
|
"definition-component",
|
||||||
|
"local-file",
|
||||||
|
"model-artifact",
|
||||||
|
]
|
||||||
|
type AssetVerificationState = Literal["matched", "missing", "mismatched"]
|
||||||
|
|
||||||
|
|
||||||
|
class PortableWorkerRuntimeError(RuntimeError):
|
||||||
|
"""Base error for the local portable Worker runtime boundary."""
|
||||||
|
|
||||||
|
|
||||||
|
class PortableWorkerRuntimeRegistryError(PortableWorkerRuntimeError):
|
||||||
|
"""A candidate registry is malformed or drifts from a RunDefinition."""
|
||||||
|
|
||||||
|
|
||||||
|
class PortableWorkerRuntimeUnavailableError(PortableWorkerRuntimeError):
|
||||||
|
"""A candidate is not a complete, sealed, locally admitted executor."""
|
||||||
|
|
||||||
|
|
||||||
|
class PortableWorkerRuntimeJobRejectedError(PortableWorkerRuntimeError):
|
||||||
|
"""A Worker job differs from the exact local candidate identity."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableWorkerRuntimePhase:
|
||||||
|
phase_id: str
|
||||||
|
state: PhaseState
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_pattern(self.phase_id, _IDENTIFIER, "runtime phase id")
|
||||||
|
if self.state not in ("implemented", "missing"):
|
||||||
|
raise PortableWorkerRuntimeRegistryError("runtime phase state is invalid")
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, str]:
|
||||||
|
return {"phase_id": self.phase_id, "state": self.state}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableWorkerAssetRequirement:
|
||||||
|
"""One exact reusable local asset; its locator is intentionally absent."""
|
||||||
|
|
||||||
|
asset_id: str
|
||||||
|
kind: RuntimeAssetKind
|
||||||
|
sha256: str
|
||||||
|
byte_length: int | None
|
||||||
|
component_id: str | None
|
||||||
|
model_release_id: str | None
|
||||||
|
model_artifact_role: str | None
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_pattern(self.asset_id, _ASSET_ID, "runtime asset id")
|
||||||
|
if self.kind not in (
|
||||||
|
"container-image",
|
||||||
|
"definition-component",
|
||||||
|
"local-file",
|
||||||
|
"model-artifact",
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeRegistryError("runtime asset kind is invalid")
|
||||||
|
_digest(self.sha256, "runtime asset sha256")
|
||||||
|
if self.byte_length is not None and (
|
||||||
|
isinstance(self.byte_length, bool)
|
||||||
|
or not isinstance(self.byte_length, int)
|
||||||
|
or self.byte_length < 1
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeRegistryError("runtime asset byte length is invalid")
|
||||||
|
if self.kind == "container-image":
|
||||||
|
if any(
|
||||||
|
value is not None
|
||||||
|
for value in (
|
||||||
|
self.byte_length,
|
||||||
|
self.component_id,
|
||||||
|
self.model_release_id,
|
||||||
|
self.model_artifact_role,
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"container image requirement cannot impersonate a definition asset"
|
||||||
|
)
|
||||||
|
elif self.kind == "definition-component":
|
||||||
|
_optional_identifier(self.component_id, "definition component id")
|
||||||
|
if self.component_id is None or any(
|
||||||
|
value is not None
|
||||||
|
for value in (self.model_release_id, self.model_artifact_role)
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"definition component requirement is incomplete"
|
||||||
|
)
|
||||||
|
elif self.kind == "model-artifact":
|
||||||
|
_optional_identifier(self.model_release_id, "model release id")
|
||||||
|
_optional_identifier(self.model_artifact_role, "model artifact role")
|
||||||
|
if (
|
||||||
|
self.model_release_id is None
|
||||||
|
or self.model_artifact_role is None
|
||||||
|
or self.component_id is not None
|
||||||
|
or self.byte_length is None
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"model artifact requirement is incomplete"
|
||||||
|
)
|
||||||
|
elif any(
|
||||||
|
value is not None
|
||||||
|
for value in (
|
||||||
|
self.component_id,
|
||||||
|
self.model_release_id,
|
||||||
|
self.model_artifact_role,
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"local file requirement cannot impersonate a definition asset"
|
||||||
|
)
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"asset_id": self.asset_id,
|
||||||
|
"kind": self.kind,
|
||||||
|
"sha256": self.sha256,
|
||||||
|
"byte_length": self.byte_length,
|
||||||
|
"component_id": self.component_id,
|
||||||
|
"model_release_id": self.model_release_id,
|
||||||
|
"model_artifact_role": self.model_artifact_role,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableWorkerExecutorSeal:
|
||||||
|
release_id: str
|
||||||
|
release_sha256: str
|
||||||
|
image_sha256: str
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_pattern(self.release_id, _IDENTIFIER, "executor release id")
|
||||||
|
_digest(self.release_sha256, "executor release sha256")
|
||||||
|
_digest(self.image_sha256, "executor image sha256")
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, str]:
|
||||||
|
return {
|
||||||
|
"release_id": self.release_id,
|
||||||
|
"release_sha256": self.release_sha256,
|
||||||
|
"image_sha256": self.image_sha256,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableWorkerRuntimeCandidate:
|
||||||
|
adapter_id: str
|
||||||
|
setup_id: str
|
||||||
|
definition_id: str
|
||||||
|
definition_version: int
|
||||||
|
definition_sha256: str
|
||||||
|
source_adapter_sha256: str
|
||||||
|
model_manifest_sha256: str
|
||||||
|
resource_profile_sha256: str
|
||||||
|
result_contract_sha256: str
|
||||||
|
state: CandidateState
|
||||||
|
executor: PortableWorkerExecutorSeal | None
|
||||||
|
reusable_assets: tuple[PortableWorkerAssetRequirement, ...]
|
||||||
|
phases: tuple[PortableWorkerRuntimePhase, ...]
|
||||||
|
blockers: tuple[str, ...]
|
||||||
|
candidate_sha256: str
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
for value, label in (
|
||||||
|
(self.adapter_id, "runtime adapter id"),
|
||||||
|
(self.setup_id, "setup id"),
|
||||||
|
(self.definition_id, "definition id"),
|
||||||
|
):
|
||||||
|
_pattern(value, _IDENTIFIER, label)
|
||||||
|
if (
|
||||||
|
isinstance(self.definition_version, bool)
|
||||||
|
or not isinstance(self.definition_version, int)
|
||||||
|
or self.definition_version < 1
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeRegistryError("definition version is invalid")
|
||||||
|
for value, label in (
|
||||||
|
(self.definition_sha256, "definition sha256"),
|
||||||
|
(self.source_adapter_sha256, "source adapter sha256"),
|
||||||
|
(self.model_manifest_sha256, "model manifest sha256"),
|
||||||
|
(self.resource_profile_sha256, "resource profile sha256"),
|
||||||
|
(self.result_contract_sha256, "result contract sha256"),
|
||||||
|
(self.candidate_sha256, "runtime candidate sha256"),
|
||||||
|
):
|
||||||
|
_digest(value, label)
|
||||||
|
if self.state not in ("blocked", "ready"):
|
||||||
|
raise PortableWorkerRuntimeRegistryError("runtime candidate state is invalid")
|
||||||
|
asset_ids = [asset.asset_id for asset in self.reusable_assets]
|
||||||
|
if asset_ids != sorted(asset_ids) or len(asset_ids) != len(set(asset_ids)):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"runtime assets must be unique and canonically ordered"
|
||||||
|
)
|
||||||
|
phase_ids = [phase.phase_id for phase in self.phases]
|
||||||
|
if not phase_ids or len(phase_ids) != len(set(phase_ids)):
|
||||||
|
raise PortableWorkerRuntimeRegistryError("runtime phases must be non-empty and unique")
|
||||||
|
if self.blockers != tuple(sorted(self.blockers)) or len(self.blockers) != len(
|
||||||
|
set(self.blockers)
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"runtime blockers must be unique and canonically ordered"
|
||||||
|
)
|
||||||
|
for blocker in self.blockers:
|
||||||
|
_pattern(blocker, _IDENTIFIER, "runtime blocker")
|
||||||
|
missing_phases = tuple(
|
||||||
|
phase.phase_id for phase in self.phases if phase.state == "missing"
|
||||||
|
)
|
||||||
|
if self.state == "ready":
|
||||||
|
if self.executor is None or self.blockers or missing_phases:
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"ready runtime requires a sealed executor and complete phases"
|
||||||
|
)
|
||||||
|
elif self.executor is not None or not self.blockers or not missing_phases:
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"blocked runtime must keep its executor unsealed and missing phases explicit"
|
||||||
|
)
|
||||||
|
if self.candidate_sha256 != canonical_sha256(self.identity_document()):
|
||||||
|
raise PortableWorkerRuntimeRegistryError("runtime candidate digest changed")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ready(self) -> bool:
|
||||||
|
return self.state == "ready"
|
||||||
|
|
||||||
|
def identity_document(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"schema_version": PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA,
|
||||||
|
"adapter_id": self.adapter_id,
|
||||||
|
"setup_id": self.setup_id,
|
||||||
|
"definition_id": self.definition_id,
|
||||||
|
"definition_version": self.definition_version,
|
||||||
|
"definition_sha256": self.definition_sha256,
|
||||||
|
"source_adapter_sha256": self.source_adapter_sha256,
|
||||||
|
"model_manifest_sha256": self.model_manifest_sha256,
|
||||||
|
"resource_profile_sha256": self.resource_profile_sha256,
|
||||||
|
"result_contract_sha256": self.result_contract_sha256,
|
||||||
|
"state": self.state,
|
||||||
|
"executor": self.executor.as_dict() if self.executor is not None else None,
|
||||||
|
"reusable_assets": [asset.as_dict() for asset in self.reusable_assets],
|
||||||
|
"phases": [phase.as_dict() for phase in self.phases],
|
||||||
|
"blockers": list(self.blockers),
|
||||||
|
"authority": dict(_AUTHORITY),
|
||||||
|
}
|
||||||
|
|
||||||
|
def bind_definition(self, definition: PortableRunDefinition) -> None:
|
||||||
|
if (
|
||||||
|
definition.setup_id != self.setup_id
|
||||||
|
or definition.definition_id != self.definition_id
|
||||||
|
or definition.version != self.definition_version
|
||||||
|
or definition.definition_sha256 != self.definition_sha256
|
||||||
|
or definition.source_adapter.contract_sha256 != self.source_adapter_sha256
|
||||||
|
or definition.model_manifest_sha256 != self.model_manifest_sha256
|
||||||
|
or definition.resource_profile.profile_sha256 != self.resource_profile_sha256
|
||||||
|
or definition.result_contract.contract_sha256 != self.result_contract_sha256
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"runtime candidate and portable RunDefinition identities disagree"
|
||||||
|
)
|
||||||
|
components = {component.component_id: component for component in definition.components}
|
||||||
|
models = {model.release_id: model for model in definition.models}
|
||||||
|
for requirement in self.reusable_assets:
|
||||||
|
if requirement.kind == "definition-component":
|
||||||
|
component = components.get(requirement.component_id or "")
|
||||||
|
if component is None or component.sha256 != requirement.sha256:
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"runtime component requirement differs from its RunDefinition"
|
||||||
|
)
|
||||||
|
elif requirement.kind == "model-artifact":
|
||||||
|
model = models.get(requirement.model_release_id or "")
|
||||||
|
artifacts = {
|
||||||
|
artifact.role: artifact for artifact in model.artifacts
|
||||||
|
} if model is not None else {}
|
||||||
|
artifact = artifacts.get(requirement.model_artifact_role or "")
|
||||||
|
if (
|
||||||
|
artifact is None
|
||||||
|
or artifact.sha256 != requirement.sha256
|
||||||
|
or artifact.byte_length != requirement.byte_length
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"runtime model artifact differs from its RunDefinition"
|
||||||
|
)
|
||||||
|
if self.state == "blocked":
|
||||||
|
if definition.executor.ready:
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"blocked local runtime cannot bind a ready RunDefinition"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if not definition.executor.ready or self.executor is None:
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"ready local runtime requires a ready RunDefinition"
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
definition.executor.release_id != self.executor.release_id
|
||||||
|
or definition.executor.release_sha256 != self.executor.release_sha256
|
||||||
|
or definition.executor.image_sha256 != self.executor.image_sha256
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"local executor seal differs from its RunDefinition"
|
||||||
|
)
|
||||||
|
|
||||||
|
def executor_identity(self) -> ObservatoryWorkerExecutorIdentity:
|
||||||
|
if not self.ready or self.executor is None:
|
||||||
|
raise PortableWorkerRuntimeUnavailableError(
|
||||||
|
"blocked runtime candidate has no executor identity"
|
||||||
|
)
|
||||||
|
return ObservatoryWorkerExecutorIdentity(
|
||||||
|
release_sha256=self.executor.release_sha256,
|
||||||
|
image_sha256=self.executor.image_sha256,
|
||||||
|
model_manifest_sha256=self.model_manifest_sha256,
|
||||||
|
resource_profile_sha256=self.resource_profile_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableWorkerRuntimeRegistry:
|
||||||
|
candidates: tuple[PortableWorkerRuntimeCandidate, ...]
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
if not self.candidates:
|
||||||
|
raise PortableWorkerRuntimeRegistryError("runtime registry cannot be empty")
|
||||||
|
for label, values in (
|
||||||
|
("runtime adapter ids", [candidate.adapter_id for candidate in self.candidates]),
|
||||||
|
("runtime setup ids", [candidate.setup_id for candidate in self.candidates]),
|
||||||
|
(
|
||||||
|
"runtime candidate digests",
|
||||||
|
[candidate.candidate_sha256 for candidate in self.candidates],
|
||||||
|
),
|
||||||
|
):
|
||||||
|
if len(values) != len(set(values)):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(f"{label} must be unique")
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_file(
|
||||||
|
cls,
|
||||||
|
path: Path,
|
||||||
|
*,
|
||||||
|
definitions: PortableRunDefinitionRegistry,
|
||||||
|
) -> PortableWorkerRuntimeRegistry:
|
||||||
|
candidate_path = path.expanduser().absolute()
|
||||||
|
if candidate_path.is_symlink() or not candidate_path.is_file():
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"runtime registry must be a regular file"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
if candidate_path.stat().st_size > _MAX_REGISTRY_BYTES:
|
||||||
|
raise PortableWorkerRuntimeRegistryError("runtime registry is too large")
|
||||||
|
document: object = json.loads(candidate_path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise PortableWorkerRuntimeRegistryError("runtime registry is unreadable") from exc
|
||||||
|
_reject_unsafe_keys(document)
|
||||||
|
root = _object(document, "runtime registry")
|
||||||
|
_exact_keys(root, {"schema_version", "candidates"}, "runtime registry")
|
||||||
|
if root["schema_version"] != PORTABLE_WORKER_RUNTIME_REGISTRY_SCHEMA:
|
||||||
|
raise PortableWorkerRuntimeRegistryError("runtime registry schema is invalid")
|
||||||
|
rows = _array(root["candidates"], "runtime candidates")
|
||||||
|
registry = cls(tuple(_candidate(row) for row in rows))
|
||||||
|
for candidate in registry.candidates:
|
||||||
|
candidate.bind_definition(
|
||||||
|
definitions.resolve(candidate.setup_id, candidate.definition_sha256)
|
||||||
|
)
|
||||||
|
return registry
|
||||||
|
|
||||||
|
def resolve(self, setup_id: str, definition_sha256: str) -> PortableWorkerRuntimeCandidate:
|
||||||
|
_pattern(setup_id, _IDENTIFIER, "setup id")
|
||||||
|
_digest(definition_sha256, "definition sha256")
|
||||||
|
for candidate in self.candidates:
|
||||||
|
if (
|
||||||
|
candidate.setup_id == setup_id
|
||||||
|
and candidate.definition_sha256 == definition_sha256
|
||||||
|
):
|
||||||
|
return candidate
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"portable Worker candidate identity is not allowlisted"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableWorkerLocalAssetBinding:
|
||||||
|
"""Worker-local binding populated by reviewed local configuration only."""
|
||||||
|
|
||||||
|
asset_id: str
|
||||||
|
file_path: Path | None = None
|
||||||
|
image_sha256: str | None = None
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_pattern(self.asset_id, _ASSET_ID, "local asset id")
|
||||||
|
if (self.file_path is None) == (self.image_sha256 is None):
|
||||||
|
raise ValueError("local asset binding must select exactly one local locator")
|
||||||
|
if self.image_sha256 is not None:
|
||||||
|
_digest(self.image_sha256, "local image sha256")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableWorkerAssetVerification:
|
||||||
|
asset_id: str
|
||||||
|
state: AssetVerificationState
|
||||||
|
reason: str | None
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_pattern(self.asset_id, _ASSET_ID, "verified asset id")
|
||||||
|
if self.state not in ("matched", "missing", "mismatched"):
|
||||||
|
raise ValueError("verified asset state is invalid")
|
||||||
|
if (self.state == "matched") != (self.reason is None):
|
||||||
|
raise ValueError("verified asset reason disagrees with its state")
|
||||||
|
|
||||||
|
|
||||||
|
def verify_local_assets(
|
||||||
|
candidate: PortableWorkerRuntimeCandidate,
|
||||||
|
bindings: Mapping[str, PortableWorkerLocalAssetBinding],
|
||||||
|
) -> tuple[PortableWorkerAssetVerification, ...]:
|
||||||
|
"""Hash locally bound assets without accepting locators from a job."""
|
||||||
|
|
||||||
|
checks: list[PortableWorkerAssetVerification] = []
|
||||||
|
for requirement in candidate.reusable_assets:
|
||||||
|
binding = bindings.get(requirement.asset_id)
|
||||||
|
if binding is None or binding.asset_id != requirement.asset_id:
|
||||||
|
checks.append(
|
||||||
|
PortableWorkerAssetVerification(requirement.asset_id, "missing", "not-bound")
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
if requirement.kind == "container-image":
|
||||||
|
matched = binding.file_path is None and binding.image_sha256 == requirement.sha256
|
||||||
|
else:
|
||||||
|
matched = _matches_file(requirement, binding.file_path)
|
||||||
|
checks.append(
|
||||||
|
PortableWorkerAssetVerification(
|
||||||
|
requirement.asset_id,
|
||||||
|
"matched" if matched else "mismatched",
|
||||||
|
None if matched else "identity-mismatch",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(checks)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableWorkerRuntimeAdmission:
|
||||||
|
candidate_sha256: str
|
||||||
|
ready: bool
|
||||||
|
blockers: tuple[str, ...]
|
||||||
|
assets: tuple[PortableWorkerAssetVerification, ...]
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_digest(self.candidate_sha256, "runtime admission candidate sha256")
|
||||||
|
if self.blockers != tuple(sorted(self.blockers)) or len(self.blockers) != len(
|
||||||
|
set(self.blockers)
|
||||||
|
):
|
||||||
|
raise ValueError("runtime admission blockers must be canonical")
|
||||||
|
if self.ready and (
|
||||||
|
self.blockers or any(item.state != "matched" for item in self.assets)
|
||||||
|
):
|
||||||
|
raise ValueError("ready runtime admission cannot contain an unresolved asset")
|
||||||
|
|
||||||
|
|
||||||
|
def inspect_runtime_candidate(
|
||||||
|
candidate: PortableWorkerRuntimeCandidate,
|
||||||
|
bindings: Mapping[str, PortableWorkerLocalAssetBinding],
|
||||||
|
) -> PortableWorkerRuntimeAdmission:
|
||||||
|
assets = verify_local_assets(candidate, bindings)
|
||||||
|
asset_blockers = tuple(
|
||||||
|
sorted(f"asset-{item.asset_id}-{item.state}" for item in assets if item.state != "matched")
|
||||||
|
)
|
||||||
|
blockers = tuple(sorted((*candidate.blockers, *asset_blockers)))
|
||||||
|
return PortableWorkerRuntimeAdmission(
|
||||||
|
candidate_sha256=candidate.candidate_sha256,
|
||||||
|
ready=candidate.ready and not blockers,
|
||||||
|
blockers=blockers,
|
||||||
|
assets=assets,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableWorkerSourceStage:
|
||||||
|
"""A locally materialized source; transport/materialization owns its path."""
|
||||||
|
|
||||||
|
root: Path
|
||||||
|
source_bundle_sha256: str
|
||||||
|
source_capability_manifest_sha256: str
|
||||||
|
source_adapter_sha256: str
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
for value, label in (
|
||||||
|
(self.source_bundle_sha256, "source bundle sha256"),
|
||||||
|
(self.source_capability_manifest_sha256, "source capability sha256"),
|
||||||
|
(self.source_adapter_sha256, "source adapter sha256"),
|
||||||
|
):
|
||||||
|
_digest(value, label)
|
||||||
|
if self.root.is_symlink() or not self.root.is_dir():
|
||||||
|
raise PortableWorkerRuntimeUnavailableError(
|
||||||
|
"portable source stage must be a real local directory"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableWorkerResultDraft:
|
||||||
|
root: Path
|
||||||
|
result_id: str
|
||||||
|
result_sha256: str
|
||||||
|
result_contract_sha256: str
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_pattern(self.result_id, _SESSION_ID, "result id")
|
||||||
|
_digest(self.result_sha256, "result sha256")
|
||||||
|
_digest(self.result_contract_sha256, "result contract sha256")
|
||||||
|
if self.root.is_symlink() or not self.root.is_dir():
|
||||||
|
raise PortableWorkerRuntimeUnavailableError(
|
||||||
|
"portable result draft must be a real local directory"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableWorkerRuntimePlan:
|
||||||
|
job_id: str
|
||||||
|
adapter_id: str
|
||||||
|
candidate_sha256: str
|
||||||
|
setup_id: str
|
||||||
|
definition_sha256: str
|
||||||
|
source_bundle_sha256: str
|
||||||
|
source_capability_manifest_sha256: str
|
||||||
|
result_contract_sha256: str
|
||||||
|
phases: tuple[str, ...]
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_pattern(self.job_id, _JOB_ID, "runtime plan job id")
|
||||||
|
for value, label in (
|
||||||
|
(self.adapter_id, "runtime plan adapter id"),
|
||||||
|
(self.setup_id, "runtime plan setup id"),
|
||||||
|
):
|
||||||
|
_pattern(value, _IDENTIFIER, label)
|
||||||
|
for value, label in (
|
||||||
|
(self.candidate_sha256, "runtime plan candidate sha256"),
|
||||||
|
(self.definition_sha256, "runtime plan definition sha256"),
|
||||||
|
(self.source_bundle_sha256, "runtime plan source bundle sha256"),
|
||||||
|
(
|
||||||
|
self.source_capability_manifest_sha256,
|
||||||
|
"runtime plan source capability sha256",
|
||||||
|
),
|
||||||
|
(self.result_contract_sha256, "runtime plan result contract sha256"),
|
||||||
|
):
|
||||||
|
_digest(value, label)
|
||||||
|
if not self.phases or len(self.phases) != len(set(self.phases)):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"runtime plan phases must be non-empty and unique"
|
||||||
|
)
|
||||||
|
for phase in self.phases:
|
||||||
|
_pattern(phase, _IDENTIFIER, "runtime plan phase id")
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"schema_version": PORTABLE_WORKER_RUNTIME_PLAN_SCHEMA,
|
||||||
|
"job_id": self.job_id,
|
||||||
|
"adapter_id": self.adapter_id,
|
||||||
|
"candidate_sha256": self.candidate_sha256,
|
||||||
|
"setup_id": self.setup_id,
|
||||||
|
"definition_sha256": self.definition_sha256,
|
||||||
|
"source_bundle_sha256": self.source_bundle_sha256,
|
||||||
|
"source_capability_manifest_sha256": self.source_capability_manifest_sha256,
|
||||||
|
"result_contract_sha256": self.result_contract_sha256,
|
||||||
|
"phases": list(self.phases),
|
||||||
|
"authority": dict(_AUTHORITY),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class PortableWorkerSourceMaterializer(Protocol):
|
||||||
|
def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage: ...
|
||||||
|
|
||||||
|
|
||||||
|
class PortableWorkerProfileRunner(Protocol):
|
||||||
|
def run(
|
||||||
|
self,
|
||||||
|
plan: PortableWorkerRuntimePlan,
|
||||||
|
source: PortableWorkerSourceStage,
|
||||||
|
) -> PortableWorkerResultDraft: ...
|
||||||
|
|
||||||
|
|
||||||
|
class PortableWorkerResultPublisher(Protocol):
|
||||||
|
def publish(
|
||||||
|
self,
|
||||||
|
job: SealedObservatoryRecordedJob,
|
||||||
|
draft: PortableWorkerResultDraft,
|
||||||
|
) -> ObservatoryWorkerExecutionResult: ...
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableWorkerExecutorAdapter:
|
||||||
|
"""Local adapter composition; none of its dependencies come from a job."""
|
||||||
|
|
||||||
|
candidate: PortableWorkerRuntimeCandidate
|
||||||
|
definition: PortableRunDefinition
|
||||||
|
admission: PortableWorkerRuntimeAdmission
|
||||||
|
source_materializer: PortableWorkerSourceMaterializer
|
||||||
|
runner: PortableWorkerProfileRunner
|
||||||
|
publisher: PortableWorkerResultPublisher
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
self.candidate.bind_definition(self.definition)
|
||||||
|
if not self.candidate.ready or not self.admission.ready:
|
||||||
|
raise PortableWorkerRuntimeUnavailableError(
|
||||||
|
"portable Worker adapter cannot bind a blocked runtime candidate"
|
||||||
|
)
|
||||||
|
if self.admission.candidate_sha256 != self.candidate.candidate_sha256:
|
||||||
|
raise PortableWorkerRuntimeUnavailableError(
|
||||||
|
"runtime admission belongs to another candidate"
|
||||||
|
)
|
||||||
|
expected_assets = tuple(
|
||||||
|
requirement.asset_id for requirement in self.candidate.reusable_assets
|
||||||
|
)
|
||||||
|
admitted_assets = tuple(item.asset_id for item in self.admission.assets)
|
||||||
|
if admitted_assets != expected_assets or any(
|
||||||
|
item.state != "matched" for item in self.admission.assets
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeUnavailableError(
|
||||||
|
"runtime admission does not prove every candidate asset"
|
||||||
|
)
|
||||||
|
|
||||||
|
def execute(
|
||||||
|
self,
|
||||||
|
job: SealedObservatoryRecordedJob,
|
||||||
|
) -> ObservatoryWorkerExecutionResult:
|
||||||
|
self._verify_job(job)
|
||||||
|
source = self.source_materializer.materialize(job)
|
||||||
|
if (
|
||||||
|
source.source_bundle_sha256 != job.source_bundle_sha256
|
||||||
|
or source.source_capability_manifest_sha256
|
||||||
|
!= job.source_capability_manifest_sha256
|
||||||
|
or source.source_adapter_sha256 != job.source_adapter_sha256
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeJobRejectedError(
|
||||||
|
"materialized source differs from the sealed job"
|
||||||
|
)
|
||||||
|
plan = PortableWorkerRuntimePlan(
|
||||||
|
job_id=job.job_id,
|
||||||
|
adapter_id=self.candidate.adapter_id,
|
||||||
|
candidate_sha256=self.candidate.candidate_sha256,
|
||||||
|
setup_id=job.setup_id,
|
||||||
|
definition_sha256=job.definition_sha256,
|
||||||
|
source_bundle_sha256=job.source_bundle_sha256,
|
||||||
|
source_capability_manifest_sha256=job.source_capability_manifest_sha256,
|
||||||
|
result_contract_sha256=self.candidate.result_contract_sha256,
|
||||||
|
phases=tuple(phase.phase_id for phase in self.candidate.phases),
|
||||||
|
)
|
||||||
|
draft = self.runner.run(plan, source)
|
||||||
|
if draft.result_contract_sha256 != self.candidate.result_contract_sha256:
|
||||||
|
raise PortableWorkerRuntimeJobRejectedError(
|
||||||
|
"runtime result uses another result contract"
|
||||||
|
)
|
||||||
|
published = self.publisher.publish(job, draft)
|
||||||
|
if (
|
||||||
|
published.result_id != draft.result_id
|
||||||
|
or published.result_sha256 != draft.result_sha256
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeJobRejectedError(
|
||||||
|
"publisher receipt differs from the validated result draft"
|
||||||
|
)
|
||||||
|
return published
|
||||||
|
|
||||||
|
def _verify_job(self, job: SealedObservatoryRecordedJob) -> None:
|
||||||
|
executor = self.candidate.executor
|
||||||
|
expected_identity = self.candidate.executor_identity()
|
||||||
|
if (
|
||||||
|
executor is None
|
||||||
|
or job.setup_id != self.definition.setup_id
|
||||||
|
or job.definition_id != self.definition.definition_id
|
||||||
|
or job.definition_version != self.definition.version
|
||||||
|
or job.definition_sha256 != self.definition.definition_sha256
|
||||||
|
or job.source_adapter_id != self.definition.source_adapter.adapter_id
|
||||||
|
or job.source_adapter_version != self.definition.source_adapter.version
|
||||||
|
or job.source_adapter_sha256 != self.definition.source_adapter.contract_sha256
|
||||||
|
or job.executor_release_id != executor.release_id
|
||||||
|
or job.executor_identity != expected_identity
|
||||||
|
or job.model_release_ids != self.definition.learned_models
|
||||||
|
or job.resource_profile_id != self.definition.resource_profile.profile_id
|
||||||
|
or job.checkpoint_policy != self.definition.resource_profile.checkpoint_policy
|
||||||
|
or job.allowed_checkpoints != self.definition.resource_profile.allowed_checkpoints
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeJobRejectedError(
|
||||||
|
"Worker job differs from the exact local runtime identity"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _matches_file(
|
||||||
|
requirement: PortableWorkerAssetRequirement,
|
||||||
|
path: Path | None,
|
||||||
|
) -> bool:
|
||||||
|
if path is None:
|
||||||
|
return False
|
||||||
|
candidate = path.expanduser().absolute()
|
||||||
|
try:
|
||||||
|
if candidate.is_symlink() or not candidate.is_file():
|
||||||
|
return False
|
||||||
|
if requirement.byte_length is not None and candidate.stat().st_size != (
|
||||||
|
requirement.byte_length
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with candidate.open("rb") as stream:
|
||||||
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest() == requirement.sha256
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _candidate(value: object) -> PortableWorkerRuntimeCandidate:
|
||||||
|
row = _object(value, "runtime candidate")
|
||||||
|
_exact_keys(
|
||||||
|
row,
|
||||||
|
{
|
||||||
|
"schema_version",
|
||||||
|
"adapter_id",
|
||||||
|
"setup_id",
|
||||||
|
"definition_id",
|
||||||
|
"definition_version",
|
||||||
|
"definition_sha256",
|
||||||
|
"source_adapter_sha256",
|
||||||
|
"model_manifest_sha256",
|
||||||
|
"resource_profile_sha256",
|
||||||
|
"result_contract_sha256",
|
||||||
|
"state",
|
||||||
|
"executor",
|
||||||
|
"reusable_assets",
|
||||||
|
"phases",
|
||||||
|
"blockers",
|
||||||
|
"authority",
|
||||||
|
"candidate_sha256",
|
||||||
|
},
|
||||||
|
"runtime candidate",
|
||||||
|
)
|
||||||
|
if row["schema_version"] != PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA:
|
||||||
|
raise PortableWorkerRuntimeRegistryError("runtime candidate schema is invalid")
|
||||||
|
if row["authority"] != _AUTHORITY:
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"runtime candidate authority must remain observation-only"
|
||||||
|
)
|
||||||
|
state = row["state"]
|
||||||
|
if state not in ("blocked", "ready"):
|
||||||
|
raise PortableWorkerRuntimeRegistryError("runtime candidate state is invalid")
|
||||||
|
executor_row = row["executor"]
|
||||||
|
executor = None if executor_row is None else _executor(executor_row)
|
||||||
|
assets = _array(row["reusable_assets"], "runtime reusable assets")
|
||||||
|
phases = _array(row["phases"], "runtime phases")
|
||||||
|
blockers = _array(row["blockers"], "runtime blockers")
|
||||||
|
return PortableWorkerRuntimeCandidate(
|
||||||
|
adapter_id=_string(row["adapter_id"], "runtime adapter id"),
|
||||||
|
setup_id=_string(row["setup_id"], "setup id"),
|
||||||
|
definition_id=_string(row["definition_id"], "definition id"),
|
||||||
|
definition_version=_integer(row["definition_version"], "definition version"),
|
||||||
|
definition_sha256=_string(row["definition_sha256"], "definition sha256"),
|
||||||
|
source_adapter_sha256=_string(
|
||||||
|
row["source_adapter_sha256"], "source adapter sha256"
|
||||||
|
),
|
||||||
|
model_manifest_sha256=_string(
|
||||||
|
row["model_manifest_sha256"], "model manifest sha256"
|
||||||
|
),
|
||||||
|
resource_profile_sha256=_string(
|
||||||
|
row["resource_profile_sha256"], "resource profile sha256"
|
||||||
|
),
|
||||||
|
result_contract_sha256=_string(
|
||||||
|
row["result_contract_sha256"], "result contract sha256"
|
||||||
|
),
|
||||||
|
state=state,
|
||||||
|
executor=executor,
|
||||||
|
reusable_assets=tuple(_asset(item) for item in assets),
|
||||||
|
phases=tuple(_phase(item) for item in phases),
|
||||||
|
blockers=tuple(_string(item, "runtime blocker") for item in blockers),
|
||||||
|
candidate_sha256=_string(row["candidate_sha256"], "runtime candidate sha256"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _asset(value: object) -> PortableWorkerAssetRequirement:
|
||||||
|
row = _object(value, "runtime asset")
|
||||||
|
_exact_keys(
|
||||||
|
row,
|
||||||
|
{
|
||||||
|
"asset_id",
|
||||||
|
"kind",
|
||||||
|
"sha256",
|
||||||
|
"byte_length",
|
||||||
|
"component_id",
|
||||||
|
"model_release_id",
|
||||||
|
"model_artifact_role",
|
||||||
|
},
|
||||||
|
"runtime asset",
|
||||||
|
)
|
||||||
|
kind = row["kind"]
|
||||||
|
if kind not in (
|
||||||
|
"container-image",
|
||||||
|
"definition-component",
|
||||||
|
"local-file",
|
||||||
|
"model-artifact",
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeRegistryError("runtime asset kind is invalid")
|
||||||
|
return PortableWorkerAssetRequirement(
|
||||||
|
asset_id=_string(row["asset_id"], "runtime asset id"),
|
||||||
|
kind=kind,
|
||||||
|
sha256=_string(row["sha256"], "runtime asset sha256"),
|
||||||
|
byte_length=_optional_integer(row["byte_length"], "runtime asset byte length"),
|
||||||
|
component_id=_optional_string(row["component_id"], "definition component id"),
|
||||||
|
model_release_id=_optional_string(row["model_release_id"], "model release id"),
|
||||||
|
model_artifact_role=_optional_string(
|
||||||
|
row["model_artifact_role"], "model artifact role"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _phase(value: object) -> PortableWorkerRuntimePhase:
|
||||||
|
row = _object(value, "runtime phase")
|
||||||
|
_exact_keys(row, {"phase_id", "state"}, "runtime phase")
|
||||||
|
state = row["state"]
|
||||||
|
if state not in ("implemented", "missing"):
|
||||||
|
raise PortableWorkerRuntimeRegistryError("runtime phase state is invalid")
|
||||||
|
return PortableWorkerRuntimePhase(
|
||||||
|
phase_id=_string(row["phase_id"], "runtime phase id"),
|
||||||
|
state=state,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _executor(value: object) -> PortableWorkerExecutorSeal:
|
||||||
|
row = _object(value, "runtime executor")
|
||||||
|
_exact_keys(
|
||||||
|
row,
|
||||||
|
{"release_id", "release_sha256", "image_sha256"},
|
||||||
|
"runtime executor",
|
||||||
|
)
|
||||||
|
return PortableWorkerExecutorSeal(
|
||||||
|
release_id=_string(row["release_id"], "executor release id"),
|
||||||
|
release_sha256=_string(row["release_sha256"], "executor release sha256"),
|
||||||
|
image_sha256=_string(row["image_sha256"], "executor image sha256"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _reject_unsafe_keys(value: object, *, parent: str = "registry") -> None:
|
||||||
|
"""Keep executable instructions and local locators out of shared config."""
|
||||||
|
|
||||||
|
if isinstance(value, dict):
|
||||||
|
for key, child in value.items():
|
||||||
|
if not isinstance(key, str):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
"runtime registry object keys must be strings"
|
||||||
|
)
|
||||||
|
normalized = key.lower().replace("-", "_")
|
||||||
|
if (
|
||||||
|
normalized == "path"
|
||||||
|
or normalized.endswith("_path")
|
||||||
|
or normalized in {"command", "commands", "argv", "env", "environment"}
|
||||||
|
or normalized.startswith("command_")
|
||||||
|
or normalized.endswith("_command")
|
||||||
|
or "priority" in normalized
|
||||||
|
):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(
|
||||||
|
f"runtime registry forbids {key!r} in {parent}"
|
||||||
|
)
|
||||||
|
_reject_unsafe_keys(child, parent=key)
|
||||||
|
elif isinstance(value, list):
|
||||||
|
for child in value:
|
||||||
|
_reject_unsafe_keys(child, parent=parent)
|
||||||
|
|
||||||
|
|
||||||
|
def _object(value: object, label: str) -> dict[str, object]:
|
||||||
|
if not isinstance(value, dict) or not all(isinstance(key, str) for key in value):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(f"{label} must be an object")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _array(value: object, label: str) -> list[object]:
|
||||||
|
if not isinstance(value, list):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(f"{label} must be an array")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _exact_keys(row: Mapping[str, object], expected: set[str], label: str) -> None:
|
||||||
|
if set(row) != expected:
|
||||||
|
raise PortableWorkerRuntimeRegistryError(f"{label} fields are invalid")
|
||||||
|
|
||||||
|
|
||||||
|
def _string(value: object, label: str) -> str:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(f"{label} must be a string")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_string(value: object, label: str) -> str | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
return _string(value, label)
|
||||||
|
|
||||||
|
|
||||||
|
def _integer(value: object, label: str) -> int:
|
||||||
|
if isinstance(value, bool) or not isinstance(value, int):
|
||||||
|
raise PortableWorkerRuntimeRegistryError(f"{label} must be an integer")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_integer(value: object, label: str) -> int | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
return _integer(value, label)
|
||||||
|
|
||||||
|
|
||||||
|
def _pattern(value: str, pattern: re.Pattern[str], label: str) -> None:
|
||||||
|
if pattern.fullmatch(value) is None:
|
||||||
|
raise PortableWorkerRuntimeRegistryError(f"{label} is invalid")
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_identifier(value: str | None, label: str) -> None:
|
||||||
|
if value is not None:
|
||||||
|
_pattern(value, _IDENTIFIER, label)
|
||||||
|
|
||||||
|
|
||||||
|
def _digest(value: object, label: str) -> None:
|
||||||
|
if not isinstance(value, str) or _SHA256.fullmatch(value) is None:
|
||||||
|
raise PortableWorkerRuntimeRegistryError(f"{label} is invalid")
|
||||||
@@ -45,6 +45,9 @@ MAX_LIVE_LEASES: Final = 10_000
|
|||||||
MAX_RECORDED_JOB_STORAGE_BYTES: Final = 128 * 1024 * 1024
|
MAX_RECORDED_JOB_STORAGE_BYTES: Final = 128 * 1024 * 1024
|
||||||
RECORDED_JOB_SQLITE_LOCK_TIMEOUT_SECONDS: Final = 0.1
|
RECORDED_JOB_SQLITE_LOCK_TIMEOUT_SECONDS: Final = 0.1
|
||||||
_SQLITE_BUSY_TIMEOUT_MILLISECONDS: Final = 100
|
_SQLITE_BUSY_TIMEOUT_MILLISECONDS: Final = 100
|
||||||
|
DEFAULT_RECORDED_CLAIM_LEASE_SECONDS: Final = 120
|
||||||
|
MIN_RECORDED_CLAIM_LEASE_SECONDS: Final = 5
|
||||||
|
MAX_RECORDED_CLAIM_LEASE_SECONDS: Final = 3_600
|
||||||
|
|
||||||
LIVE_K1_PRIORITY_RANK: Final = 0
|
LIVE_K1_PRIORITY_RANK: Final = 0
|
||||||
RECORDED_PRIORITY_RANK: Final = 100
|
RECORDED_PRIORITY_RANK: Final = 100
|
||||||
@@ -120,6 +123,11 @@ CREATE TABLE IF NOT EXISTS observatory_recorded_jobs (
|
|||||||
claim_generation INTEGER NOT NULL CHECK (claim_generation >= 0),
|
claim_generation INTEGER NOT NULL CHECK (claim_generation >= 0),
|
||||||
active_claim_token TEXT,
|
active_claim_token TEXT,
|
||||||
active_claimant_id TEXT,
|
active_claimant_id TEXT,
|
||||||
|
claimed_at_utc TEXT,
|
||||||
|
claim_expires_at_utc TEXT,
|
||||||
|
claim_heartbeat_at_utc TEXT,
|
||||||
|
claim_renewal_count INTEGER NOT NULL DEFAULT 0
|
||||||
|
CHECK (claim_renewal_count >= 0),
|
||||||
last_checkpoint_id TEXT,
|
last_checkpoint_id TEXT,
|
||||||
restart_from_zero INTEGER NOT NULL CHECK (restart_from_zero IN (0, 1)),
|
restart_from_zero INTEGER NOT NULL CHECK (restart_from_zero IN (0, 1)),
|
||||||
preemption_receipt_sha256 TEXT,
|
preemption_receipt_sha256 TEXT,
|
||||||
@@ -386,6 +394,10 @@ class ObservatoryRecordedJob:
|
|||||||
claim_generation: int
|
claim_generation: int
|
||||||
active_claim_token: str | None
|
active_claim_token: str | None
|
||||||
active_claimant_id: str | None
|
active_claimant_id: str | None
|
||||||
|
claimed_at_utc: str | None
|
||||||
|
claim_expires_at_utc: str | None
|
||||||
|
claim_heartbeat_at_utc: str | None
|
||||||
|
claim_renewal_count: int
|
||||||
last_checkpoint_id: str | None
|
last_checkpoint_id: str | None
|
||||||
restart_from_zero: bool
|
restart_from_zero: bool
|
||||||
preemption_receipt_sha256: str | None
|
preemption_receipt_sha256: str | None
|
||||||
@@ -461,6 +473,46 @@ class ObservatoryRecordedJob:
|
|||||||
raise ObservatoryRecordedQueueIntegrityError("recorded-job claim generation is invalid")
|
raise ObservatoryRecordedQueueIntegrityError("recorded-job claim generation is invalid")
|
||||||
_validate_optional_pattern(self.active_claim_token, _TOKEN, "active claim token")
|
_validate_optional_pattern(self.active_claim_token, _TOKEN, "active claim token")
|
||||||
_validate_optional_pattern(self.active_claimant_id, _IDENTIFIER, "claimant id")
|
_validate_optional_pattern(self.active_claimant_id, _IDENTIFIER, "claimant id")
|
||||||
|
lease_values = (
|
||||||
|
self.claimed_at_utc,
|
||||||
|
self.claim_expires_at_utc,
|
||||||
|
self.claim_heartbeat_at_utc,
|
||||||
|
)
|
||||||
|
if (self.active_claim_token is None) != (self.active_claimant_id is None):
|
||||||
|
raise ObservatoryRecordedQueueIntegrityError(
|
||||||
|
"recorded-job claim ownership is partial"
|
||||||
|
)
|
||||||
|
if self.active_claim_token is None:
|
||||||
|
if any(value is not None for value in lease_values) or self.claim_renewal_count != 0:
|
||||||
|
raise ObservatoryRecordedQueueIntegrityError(
|
||||||
|
"inactive recorded-job claim retains lease state"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
if self.state not in {"claimed", "running", "paused", "preemption-pending"}:
|
||||||
|
raise ObservatoryRecordedQueueIntegrityError(
|
||||||
|
"recorded-job claim is active outside an owned state"
|
||||||
|
)
|
||||||
|
if self.claim_generation < 1 or any(value is None for value in lease_values):
|
||||||
|
raise ObservatoryRecordedQueueIntegrityError(
|
||||||
|
"active recorded-job claim has no complete lease"
|
||||||
|
)
|
||||||
|
if self.claim_renewal_count < 0:
|
||||||
|
raise ObservatoryRecordedQueueIntegrityError(
|
||||||
|
"recorded-job claim renewal count is invalid"
|
||||||
|
)
|
||||||
|
assert self.claimed_at_utc is not None
|
||||||
|
assert self.claim_expires_at_utc is not None
|
||||||
|
assert self.claim_heartbeat_at_utc is not None
|
||||||
|
claimed_at = _parse_timestamp(self.claimed_at_utc, "claim timestamp")
|
||||||
|
expires_at = _parse_timestamp(self.claim_expires_at_utc, "claim expiry")
|
||||||
|
heartbeat_at = _parse_timestamp(
|
||||||
|
self.claim_heartbeat_at_utc,
|
||||||
|
"claim heartbeat timestamp",
|
||||||
|
)
|
||||||
|
if not claimed_at <= heartbeat_at < expires_at:
|
||||||
|
raise ObservatoryRecordedQueueIntegrityError(
|
||||||
|
"recorded-job claim lease chronology is invalid"
|
||||||
|
)
|
||||||
_validate_optional_pattern(self.last_checkpoint_id, _CHECKPOINT_ID, "last checkpoint id")
|
_validate_optional_pattern(self.last_checkpoint_id, _CHECKPOINT_ID, "last checkpoint id")
|
||||||
if not isinstance(self.restart_from_zero, bool):
|
if not isinstance(self.restart_from_zero, bool):
|
||||||
raise ObservatoryRecordedQueueIntegrityError("recorded-job restart marker is invalid")
|
raise ObservatoryRecordedQueueIntegrityError("recorded-job restart marker is invalid")
|
||||||
@@ -532,6 +584,16 @@ class ObservatoryRecordedJob:
|
|||||||
"restart_from_zero": self.restart_from_zero,
|
"restart_from_zero": self.restart_from_zero,
|
||||||
"preemption_receipt_sha256": self.preemption_receipt_sha256,
|
"preemption_receipt_sha256": self.preemption_receipt_sha256,
|
||||||
"claim_generation": self.claim_generation,
|
"claim_generation": self.claim_generation,
|
||||||
|
"claim_lease": (
|
||||||
|
None
|
||||||
|
if self.active_claim_token is None
|
||||||
|
else {
|
||||||
|
"claimed_at_utc": self.claimed_at_utc,
|
||||||
|
"expires_at_utc": self.claim_expires_at_utc,
|
||||||
|
"heartbeat_at_utc": self.claim_heartbeat_at_utc,
|
||||||
|
"renewal_count": self.claim_renewal_count,
|
||||||
|
}
|
||||||
|
),
|
||||||
"result": (
|
"result": (
|
||||||
None
|
None
|
||||||
if self.result_id is None
|
if self.result_id is None
|
||||||
@@ -794,10 +856,12 @@ class ObservatoryRecordedJobQueue:
|
|||||||
max_jobs: int = MAX_RECORDED_JOBS,
|
max_jobs: int = MAX_RECORDED_JOBS,
|
||||||
max_claim_receipts: int = MAX_RECORDED_CLAIM_RECEIPTS,
|
max_claim_receipts: int = MAX_RECORDED_CLAIM_RECEIPTS,
|
||||||
max_live_leases: int = MAX_LIVE_LEASES,
|
max_live_leases: int = MAX_LIVE_LEASES,
|
||||||
|
claim_lease_seconds: int = DEFAULT_RECORDED_CLAIM_LEASE_SECONDS,
|
||||||
) -> None:
|
) -> None:
|
||||||
_validate_quota(max_jobs, MAX_RECORDED_JOBS, "recorded job")
|
_validate_quota(max_jobs, MAX_RECORDED_JOBS, "recorded job")
|
||||||
_validate_quota(max_claim_receipts, MAX_RECORDED_CLAIM_RECEIPTS, "claim receipt")
|
_validate_quota(max_claim_receipts, MAX_RECORDED_CLAIM_RECEIPTS, "claim receipt")
|
||||||
_validate_quota(max_live_leases, MAX_LIVE_LEASES, "live lease")
|
_validate_quota(max_live_leases, MAX_LIVE_LEASES, "live lease")
|
||||||
|
_validate_claim_lease_seconds(claim_lease_seconds)
|
||||||
self.data_dir = data_dir.expanduser().resolve()
|
self.data_dir = data_dir.expanduser().resolve()
|
||||||
self.database_path = self.data_dir / RECORDED_JOB_DATABASE_NAME
|
self.database_path = self.data_dir / RECORDED_JOB_DATABASE_NAME
|
||||||
self._definitions = definitions
|
self._definitions = definitions
|
||||||
@@ -806,6 +870,7 @@ class ObservatoryRecordedJobQueue:
|
|||||||
self._max_jobs = max_jobs
|
self._max_jobs = max_jobs
|
||||||
self._max_claim_receipts = max_claim_receipts
|
self._max_claim_receipts = max_claim_receipts
|
||||||
self._max_live_leases = max_live_leases
|
self._max_live_leases = max_live_leases
|
||||||
|
self._claim_lease_seconds = claim_lease_seconds
|
||||||
self._lock = threading.RLock()
|
self._lock = threading.RLock()
|
||||||
self._initialize()
|
self._initialize()
|
||||||
|
|
||||||
@@ -946,6 +1011,8 @@ class ObservatoryRecordedJobQueue:
|
|||||||
_validate_pattern(claim_request_id, _IDEMPOTENCY_KEY, "claim request id")
|
_validate_pattern(claim_request_id, _IDEMPOTENCY_KEY, "claim request id")
|
||||||
request_sha256 = _claim_request_sha256(claimant_id, claim_request_id)
|
request_sha256 = _claim_request_sha256(claimant_id, claim_request_id)
|
||||||
with self._transaction() as connection:
|
with self._transaction() as connection:
|
||||||
|
now = self._timestamp()
|
||||||
|
self._recover_stale_claims(connection, now=now)
|
||||||
receipt = connection.execute(
|
receipt = connection.execute(
|
||||||
"SELECT * FROM observatory_recorded_claim_receipts WHERE claim_request_id = ?",
|
"SELECT * FROM observatory_recorded_claim_receipts WHERE claim_request_id = ?",
|
||||||
(claim_request_id,),
|
(claim_request_id,),
|
||||||
@@ -976,7 +1043,6 @@ class ObservatoryRecordedJobQueue:
|
|||||||
"WHERE state = 'queued' "
|
"WHERE state = 'queued' "
|
||||||
"ORDER BY priority_rank, created_at_utc, job_id LIMIT 1"
|
"ORDER BY priority_rank, created_at_utc, job_id LIMIT 1"
|
||||||
).fetchone()
|
).fetchone()
|
||||||
now = self._timestamp()
|
|
||||||
if row is None:
|
if row is None:
|
||||||
connection.execute(
|
connection.execute(
|
||||||
"INSERT INTO observatory_recorded_claim_receipts "
|
"INSERT INTO observatory_recorded_claim_receipts "
|
||||||
@@ -989,12 +1055,26 @@ class ObservatoryRecordedJobQueue:
|
|||||||
claim_token = hashlib.sha256(
|
claim_token = hashlib.sha256(
|
||||||
f"{uuid4().hex}:{job_id}:{claim_request_id}".encode()
|
f"{uuid4().hex}:{job_id}:{claim_request_id}".encode()
|
||||||
).hexdigest()
|
).hexdigest()
|
||||||
|
claim_expires_at = _timestamp_after_seconds(
|
||||||
|
now,
|
||||||
|
self._claim_lease_seconds,
|
||||||
|
)
|
||||||
updated = connection.execute(
|
updated = connection.execute(
|
||||||
"UPDATE observatory_recorded_jobs SET state = 'claimed', "
|
"UPDATE observatory_recorded_jobs SET state = 'claimed', "
|
||||||
"claim_generation = claim_generation + 1, active_claim_token = ?, "
|
"claim_generation = claim_generation + 1, active_claim_token = ?, "
|
||||||
"active_claimant_id = ?, preemption_requested = 0, "
|
"active_claimant_id = ?, claimed_at_utc = ?, "
|
||||||
|
"claim_expires_at_utc = ?, claim_heartbeat_at_utc = ?, "
|
||||||
|
"claim_renewal_count = 0, preemption_requested = 0, "
|
||||||
"updated_at_utc = ? WHERE job_id = ? AND state = 'queued'",
|
"updated_at_utc = ? WHERE job_id = ? AND state = 'queued'",
|
||||||
(claim_token, claimant_id, now, job_id),
|
(
|
||||||
|
claim_token,
|
||||||
|
claimant_id,
|
||||||
|
now,
|
||||||
|
claim_expires_at,
|
||||||
|
now,
|
||||||
|
now,
|
||||||
|
job_id,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
if updated.rowcount != 1:
|
if updated.rowcount != 1:
|
||||||
raise ObservatoryRecordedQueueIntegrityError(
|
raise ObservatoryRecordedQueueIntegrityError(
|
||||||
@@ -1021,12 +1101,125 @@ class ObservatoryRecordedJobQueue:
|
|||||||
job=self._get_job(connection, job_id),
|
job=self._get_job(connection, job_id),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def renew_claim(
|
||||||
|
self,
|
||||||
|
job_id: str,
|
||||||
|
*,
|
||||||
|
claim_token: str,
|
||||||
|
claim_generation: int,
|
||||||
|
heartbeat_sequence: int,
|
||||||
|
) -> ObservatoryRecordedJob:
|
||||||
|
"""Renew one exact active claim using an idempotent heartbeat sequence."""
|
||||||
|
|
||||||
|
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||||
|
_validate_pattern(claim_token, _TOKEN, "claim token")
|
||||||
|
_validate_positive_int(claim_generation, "claim generation")
|
||||||
|
_validate_positive_int(heartbeat_sequence, "heartbeat sequence")
|
||||||
|
self.recover_stale_claims()
|
||||||
|
with self._transaction() as connection:
|
||||||
|
job = self._get_job(connection, job_id)
|
||||||
|
now = self._timestamp()
|
||||||
|
self._require_active_claim(job, claim_token, now=now)
|
||||||
|
if job.claim_generation != claim_generation:
|
||||||
|
raise ObservatoryRecordedQueueStaleClaimError(
|
||||||
|
"recorded-job claim generation is stale"
|
||||||
|
)
|
||||||
|
if job.state not in {"claimed", "running", "paused", "preemption-pending"}:
|
||||||
|
raise ObservatoryRecordedQueueConflictError(
|
||||||
|
f"cannot renew recorded-job claim from {job.state}"
|
||||||
|
)
|
||||||
|
if heartbeat_sequence == job.claim_renewal_count:
|
||||||
|
return job
|
||||||
|
if heartbeat_sequence != job.claim_renewal_count + 1:
|
||||||
|
raise ObservatoryRecordedQueueConflictError(
|
||||||
|
"recorded-job heartbeat sequence is not contiguous"
|
||||||
|
)
|
||||||
|
expires_at = _timestamp_after_seconds(now, self._claim_lease_seconds)
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE observatory_recorded_jobs SET claim_expires_at_utc = ?, "
|
||||||
|
"claim_heartbeat_at_utc = ?, claim_renewal_count = ?, "
|
||||||
|
"updated_at_utc = ? WHERE job_id = ? AND active_claim_token = ? "
|
||||||
|
"AND claim_generation = ?",
|
||||||
|
(
|
||||||
|
expires_at,
|
||||||
|
now,
|
||||||
|
heartbeat_sequence,
|
||||||
|
now,
|
||||||
|
job_id,
|
||||||
|
claim_token,
|
||||||
|
claim_generation,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return self._get_job(connection, job_id)
|
||||||
|
|
||||||
|
def authorize_claim_access(
|
||||||
|
self,
|
||||||
|
job_id: str,
|
||||||
|
*,
|
||||||
|
claim_token: str,
|
||||||
|
claim_generation: int,
|
||||||
|
claimant_id: str,
|
||||||
|
allowed_states: tuple[RecordedJobState, ...] = ("claimed", "running"),
|
||||||
|
) -> ObservatoryRecordedJob:
|
||||||
|
"""Authorize one bounded side-channel operation for the active lease.
|
||||||
|
|
||||||
|
Source downloads and result staging are deliberately not queue state
|
||||||
|
transitions, but they still must be fenced by the exact claimant,
|
||||||
|
token, generation, unexpired lease and an explicitly admitted queue
|
||||||
|
state. Keeping this check inside the queue transaction prevents an
|
||||||
|
artifact transport from reimplementing only part of claim semantics.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||||
|
_validate_pattern(claim_token, _TOKEN, "claim token")
|
||||||
|
_validate_positive_int(claim_generation, "claim generation")
|
||||||
|
_validate_pattern(claimant_id, _IDENTIFIER, "claimant id")
|
||||||
|
if (
|
||||||
|
not allowed_states
|
||||||
|
or len(set(allowed_states)) != len(allowed_states)
|
||||||
|
or any(state not in _recorded_job_states() for state in allowed_states)
|
||||||
|
):
|
||||||
|
raise ValueError("claim-access states are invalid")
|
||||||
|
self.recover_stale_claims()
|
||||||
|
with self._transaction() as connection:
|
||||||
|
job = self._get_job(connection, job_id)
|
||||||
|
self._require_active_claim(job, claim_token, now=self._timestamp())
|
||||||
|
if (
|
||||||
|
job.claim_generation != claim_generation
|
||||||
|
or job.active_claimant_id != claimant_id
|
||||||
|
):
|
||||||
|
raise ObservatoryRecordedQueueStaleClaimError(
|
||||||
|
"recorded-job claim ownership is stale"
|
||||||
|
)
|
||||||
|
if job.state not in allowed_states:
|
||||||
|
raise ObservatoryRecordedQueueConflictError(
|
||||||
|
f"claim-bound access is unavailable from {job.state}"
|
||||||
|
)
|
||||||
|
return job
|
||||||
|
|
||||||
|
def recover_stale_claims(self) -> tuple[ObservatoryRecordedJob, ...]:
|
||||||
|
"""Recover expired ownership without creating a second physical owner.
|
||||||
|
|
||||||
|
A never-started or durably paused claim is safe to requeue. Expired
|
||||||
|
running ownership is quarantined for scheduler reconciliation because
|
||||||
|
lease expiry alone does not prove that its Worker process stopped.
|
||||||
|
"""
|
||||||
|
|
||||||
|
with self._transaction() as connection:
|
||||||
|
recovered_ids = self._recover_stale_claims(
|
||||||
|
connection,
|
||||||
|
now=self._timestamp(),
|
||||||
|
)
|
||||||
|
return tuple(self._get_job(connection, job_id) for job_id in recovered_ids)
|
||||||
|
|
||||||
def start(self, job_id: str, *, claim_token: str) -> ObservatoryRecordedJob:
|
def start(self, job_id: str, *, claim_token: str) -> ObservatoryRecordedJob:
|
||||||
"""Enter running state, or yield before execution when live has priority."""
|
"""Enter running state, or yield before execution when live has priority."""
|
||||||
|
|
||||||
|
self.recover_stale_claims()
|
||||||
with self._transaction() as connection:
|
with self._transaction() as connection:
|
||||||
job = self._get_job(connection, job_id)
|
job = self._get_job(connection, job_id)
|
||||||
self._require_active_claim(job, claim_token)
|
now = self._timestamp()
|
||||||
|
self._require_active_claim(job, claim_token, now=now)
|
||||||
if job.state == "running":
|
if job.state == "running":
|
||||||
return job
|
return job
|
||||||
if job.state == "paused":
|
if job.state == "paused":
|
||||||
@@ -1039,7 +1232,7 @@ class ObservatoryRecordedJobQueue:
|
|||||||
connection.execute(
|
connection.execute(
|
||||||
"UPDATE observatory_recorded_jobs SET state = ?, "
|
"UPDATE observatory_recorded_jobs SET state = ?, "
|
||||||
"preemption_requested = ?, updated_at_utc = ? WHERE job_id = ?",
|
"preemption_requested = ?, updated_at_utc = ? WHERE job_id = ?",
|
||||||
(state, int(state == "paused"), self._timestamp(), job_id),
|
(state, int(state == "paused"), now, job_id),
|
||||||
)
|
)
|
||||||
return self._get_job(connection, job_id)
|
return self._get_job(connection, job_id)
|
||||||
|
|
||||||
@@ -1053,9 +1246,11 @@ class ObservatoryRecordedJobQueue:
|
|||||||
"""Record an allowlisted cooperative boundary and yield if live is open."""
|
"""Record an allowlisted cooperative boundary and yield if live is open."""
|
||||||
|
|
||||||
_validate_pattern(checkpoint_id, _CHECKPOINT_ID, "checkpoint id")
|
_validate_pattern(checkpoint_id, _CHECKPOINT_ID, "checkpoint id")
|
||||||
|
self.recover_stale_claims()
|
||||||
with self._transaction() as connection:
|
with self._transaction() as connection:
|
||||||
job = self._get_job(connection, job_id)
|
job = self._get_job(connection, job_id)
|
||||||
self._require_active_claim(job, claim_token)
|
now = self._timestamp()
|
||||||
|
self._require_active_claim(job, claim_token, now=now)
|
||||||
if job.checkpoint_policy != "cooperative":
|
if job.checkpoint_policy != "cooperative":
|
||||||
raise ObservatoryRecordedCheckpointError(
|
raise ObservatoryRecordedCheckpointError(
|
||||||
"recorded RunDefinition is non-checkpointable"
|
"recorded RunDefinition is non-checkpointable"
|
||||||
@@ -1079,7 +1274,7 @@ class ObservatoryRecordedJobQueue:
|
|||||||
connection.execute(
|
connection.execute(
|
||||||
"UPDATE observatory_recorded_jobs SET state = ?, "
|
"UPDATE observatory_recorded_jobs SET state = ?, "
|
||||||
"last_checkpoint_id = ?, updated_at_utc = ? WHERE job_id = ?",
|
"last_checkpoint_id = ?, updated_at_utc = ? WHERE job_id = ?",
|
||||||
(state, checkpoint_id, self._timestamp(), job_id),
|
(state, checkpoint_id, now, job_id),
|
||||||
)
|
)
|
||||||
return self._get_job(connection, job_id)
|
return self._get_job(connection, job_id)
|
||||||
|
|
||||||
@@ -1195,6 +1390,7 @@ class ObservatoryRecordedJobQueue:
|
|||||||
def request_live(self, intent: ObservatoryLiveLeaseIntent) -> tuple[ObservatoryLiveLease, bool]:
|
def request_live(self, intent: ObservatoryLiveLeaseIntent) -> tuple[ObservatoryLiveLease, bool]:
|
||||||
"""Close recorded admission without allowing a monolith to delay live K1."""
|
"""Close recorded admission without allowing a monolith to delay live K1."""
|
||||||
|
|
||||||
|
self.recover_stale_claims()
|
||||||
created = False
|
created = False
|
||||||
with self._transaction() as connection:
|
with self._transaction() as connection:
|
||||||
existing = connection.execute(
|
existing = connection.execute(
|
||||||
@@ -1341,6 +1537,7 @@ class ObservatoryRecordedJobQueue:
|
|||||||
"""Activate only after every recorded resource owner has yielded."""
|
"""Activate only after every recorded resource owner has yielded."""
|
||||||
|
|
||||||
_validate_pattern(lease_id, _LEASE_ID, "live lease id")
|
_validate_pattern(lease_id, _LEASE_ID, "live lease id")
|
||||||
|
self.recover_stale_claims()
|
||||||
with self._transaction() as connection:
|
with self._transaction() as connection:
|
||||||
lease = self._get_live_lease(connection, lease_id)
|
lease = self._get_live_lease(connection, lease_id)
|
||||||
if lease.state == "active":
|
if lease.state == "active":
|
||||||
@@ -1438,7 +1635,9 @@ class ObservatoryRecordedJobQueue:
|
|||||||
connection.execute(
|
connection.execute(
|
||||||
"UPDATE observatory_recorded_jobs SET state = 'queued', "
|
"UPDATE observatory_recorded_jobs SET state = 'queued', "
|
||||||
"preemption_requested = 0, active_claim_token = NULL, "
|
"preemption_requested = 0, active_claim_token = NULL, "
|
||||||
"active_claimant_id = NULL, updated_at_utc = ? "
|
"active_claimant_id = NULL, claimed_at_utc = NULL, "
|
||||||
|
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
|
||||||
|
"claim_renewal_count = 0, updated_at_utc = ? "
|
||||||
"WHERE state = 'paused'",
|
"WHERE state = 'paused'",
|
||||||
(now,),
|
(now,),
|
||||||
)
|
)
|
||||||
@@ -1488,22 +1687,34 @@ class ObservatoryRecordedJobQueue:
|
|||||||
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||||
_validate_pattern(claim_token, _TOKEN, "claim token")
|
_validate_pattern(claim_token, _TOKEN, "claim token")
|
||||||
token_sha256 = hashlib.sha256(claim_token.encode()).hexdigest()
|
token_sha256 = hashlib.sha256(claim_token.encode()).hexdigest()
|
||||||
|
self.recover_stale_claims()
|
||||||
with self._transaction() as connection:
|
with self._transaction() as connection:
|
||||||
job = self._get_job(connection, job_id)
|
job = self._get_job(connection, job_id)
|
||||||
if job.state in _TERMINAL_STATES:
|
if job.state in _TERMINAL_STATES:
|
||||||
|
exact_replay = (
|
||||||
|
job.state == state
|
||||||
|
and job.result_id == result_id
|
||||||
|
and job.result_sha256 == result_sha256
|
||||||
|
and job.terminal_code == terminal_code
|
||||||
|
and job.terminal_message == terminal_message
|
||||||
|
and job.terminal_claim_token_sha256 == token_sha256
|
||||||
|
)
|
||||||
|
if exact_replay:
|
||||||
|
return job
|
||||||
if (
|
if (
|
||||||
job.state != state
|
job.terminal_claim_token_sha256 != token_sha256
|
||||||
or job.result_id != result_id
|
or job.terminal_code
|
||||||
or job.result_sha256 != result_sha256
|
in {"claim-lease-expired", "claim-lease-migration"}
|
||||||
or job.terminal_code != terminal_code
|
|
||||||
or job.terminal_message != terminal_message
|
|
||||||
or job.terminal_claim_token_sha256 != token_sha256
|
|
||||||
):
|
):
|
||||||
|
raise ObservatoryRecordedQueueStaleClaimError(
|
||||||
|
"recorded-job terminal acknowledgement is stale"
|
||||||
|
)
|
||||||
|
else:
|
||||||
raise ObservatoryRecordedQueueConflictError(
|
raise ObservatoryRecordedQueueConflictError(
|
||||||
"recorded job is bound to another terminal outcome"
|
"recorded job is bound to another terminal outcome"
|
||||||
)
|
)
|
||||||
return job
|
now = self._timestamp()
|
||||||
self._require_active_claim(job, claim_token)
|
self._require_active_claim(job, claim_token, now=now)
|
||||||
allowed_states = (
|
allowed_states = (
|
||||||
("running",)
|
("running",)
|
||||||
if state == "succeeded"
|
if state == "succeeded"
|
||||||
@@ -1523,7 +1734,9 @@ class ObservatoryRecordedJobQueue:
|
|||||||
"UPDATE observatory_recorded_jobs SET state = ?, result_id = ?, "
|
"UPDATE observatory_recorded_jobs SET state = ?, result_id = ?, "
|
||||||
"result_sha256 = ?, terminal_code = ?, terminal_message = ?, "
|
"result_sha256 = ?, terminal_code = ?, terminal_message = ?, "
|
||||||
"terminal_claim_token_sha256 = ?, active_claim_token = NULL, "
|
"terminal_claim_token_sha256 = ?, active_claim_token = NULL, "
|
||||||
"active_claimant_id = NULL, updated_at_utc = ? WHERE job_id = ?",
|
"active_claimant_id = NULL, claimed_at_utc = NULL, "
|
||||||
|
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
|
||||||
|
"claim_renewal_count = 0, updated_at_utc = ? WHERE job_id = ?",
|
||||||
(
|
(
|
||||||
state,
|
state,
|
||||||
result_id,
|
result_id,
|
||||||
@@ -1531,7 +1744,7 @@ class ObservatoryRecordedJobQueue:
|
|||||||
terminal_code,
|
terminal_code,
|
||||||
terminal_message,
|
terminal_message,
|
||||||
token_sha256,
|
token_sha256,
|
||||||
self._timestamp(),
|
now,
|
||||||
job_id,
|
job_id,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -1548,18 +1761,105 @@ class ObservatoryRecordedJobQueue:
|
|||||||
raise ObservatoryRecordedQueueIntegrityError(
|
raise ObservatoryRecordedQueueIntegrityError(
|
||||||
"stored claim receipt has a partial job identity"
|
"stored claim receipt has a partial job identity"
|
||||||
)
|
)
|
||||||
|
job = self._get_job(connection, job_id)
|
||||||
|
if job.active_claim_token != claim_token:
|
||||||
|
raise ObservatoryRecordedQueueStaleClaimError(
|
||||||
|
"recorded-job claim receipt no longer owns the job"
|
||||||
|
)
|
||||||
return ObservatoryRecordedClaim(
|
return ObservatoryRecordedClaim(
|
||||||
claim_request_id=str(receipt["claim_request_id"]),
|
claim_request_id=str(receipt["claim_request_id"]),
|
||||||
request_sha256=str(receipt["request_sha256"]),
|
request_sha256=str(receipt["request_sha256"]),
|
||||||
claimant_id=str(receipt["claimant_id"]),
|
claimant_id=str(receipt["claimant_id"]),
|
||||||
claim_token=claim_token,
|
claim_token=claim_token,
|
||||||
job=self._get_job(connection, job_id),
|
job=job,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _require_active_claim(self, job: ObservatoryRecordedJob, claim_token: str) -> None:
|
def _require_active_claim(
|
||||||
|
self,
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
claim_token: str,
|
||||||
|
*,
|
||||||
|
now: str | None = None,
|
||||||
|
) -> None:
|
||||||
_validate_pattern(claim_token, _TOKEN, "claim token")
|
_validate_pattern(claim_token, _TOKEN, "claim token")
|
||||||
if job.active_claim_token != claim_token:
|
if job.active_claim_token != claim_token:
|
||||||
raise ObservatoryRecordedQueueStaleClaimError("recorded-job claim token is stale")
|
raise ObservatoryRecordedQueueStaleClaimError("recorded-job claim token is stale")
|
||||||
|
if now is not None and (
|
||||||
|
job.claim_expires_at_utc is None
|
||||||
|
or _parse_timestamp(now, "queue timestamp")
|
||||||
|
>= _parse_timestamp(job.claim_expires_at_utc, "claim expiry")
|
||||||
|
):
|
||||||
|
raise ObservatoryRecordedQueueStaleClaimError(
|
||||||
|
"recorded-job claim lease expired"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _recover_stale_claims(
|
||||||
|
self,
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
*,
|
||||||
|
now: str,
|
||||||
|
) -> tuple[str, ...]:
|
||||||
|
now_value = _parse_timestamp(now, "queue timestamp")
|
||||||
|
rows = connection.execute(
|
||||||
|
"SELECT * FROM observatory_recorded_jobs "
|
||||||
|
"WHERE active_claim_token IS NOT NULL "
|
||||||
|
"ORDER BY created_at_utc, job_id"
|
||||||
|
).fetchall()
|
||||||
|
recovered: list[str] = []
|
||||||
|
live_open = self._open_live_lease_row(connection) is not None
|
||||||
|
for row in rows:
|
||||||
|
job = _job_from_row(row)
|
||||||
|
if job.claim_expires_at_utc is None:
|
||||||
|
raise ObservatoryRecordedQueueIntegrityError(
|
||||||
|
"active recorded-job claim has no expiry"
|
||||||
|
)
|
||||||
|
if _parse_timestamp(job.claim_expires_at_utc, "claim expiry") > now_value:
|
||||||
|
continue
|
||||||
|
assert job.active_claim_token is not None
|
||||||
|
token_sha256 = hashlib.sha256(job.active_claim_token.encode()).hexdigest()
|
||||||
|
if job.state in {"claimed", "paused"}:
|
||||||
|
next_state = "paused" if job.state == "paused" and live_open else "queued"
|
||||||
|
restart_from_zero = job.restart_from_zero or job.state == "paused"
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE observatory_recorded_jobs SET state = ?, "
|
||||||
|
"preemption_requested = ?, active_claim_token = NULL, "
|
||||||
|
"active_claimant_id = NULL, claimed_at_utc = NULL, "
|
||||||
|
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
|
||||||
|
"claim_renewal_count = 0, last_checkpoint_id = ?, "
|
||||||
|
"restart_from_zero = ?, updated_at_utc = ? WHERE job_id = ?",
|
||||||
|
(
|
||||||
|
next_state,
|
||||||
|
int(next_state == "paused"),
|
||||||
|
None if restart_from_zero else job.last_checkpoint_id,
|
||||||
|
int(restart_from_zero),
|
||||||
|
now,
|
||||||
|
job.job_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
elif job.state in {"running", "preemption-pending"}:
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE observatory_recorded_jobs "
|
||||||
|
"SET state = 'reconciliation-required', result_id = NULL, "
|
||||||
|
"result_sha256 = NULL, terminal_code = 'claim-lease-expired', "
|
||||||
|
"terminal_message = ?, terminal_claim_token_sha256 = ?, "
|
||||||
|
"active_claim_token = NULL, active_claimant_id = NULL, "
|
||||||
|
"claimed_at_utc = NULL, claim_expires_at_utc = NULL, "
|
||||||
|
"claim_heartbeat_at_utc = NULL, claim_renewal_count = 0, "
|
||||||
|
"updated_at_utc = ? WHERE job_id = ?",
|
||||||
|
(
|
||||||
|
"Worker claim lease expired after execution started; "
|
||||||
|
"physical resource ownership requires reconciliation.",
|
||||||
|
token_sha256,
|
||||||
|
now,
|
||||||
|
job.job_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ObservatoryRecordedQueueIntegrityError(
|
||||||
|
"active recorded-job claim is stored in an invalid state"
|
||||||
|
)
|
||||||
|
recovered.append(job.job_id)
|
||||||
|
return tuple(recovered)
|
||||||
|
|
||||||
def _cancellation_request(
|
def _cancellation_request(
|
||||||
self,
|
self,
|
||||||
@@ -1626,7 +1926,9 @@ class ObservatoryRecordedJobQueue:
|
|||||||
connection.execute(
|
connection.execute(
|
||||||
"UPDATE observatory_recorded_jobs SET state = 'paused', "
|
"UPDATE observatory_recorded_jobs SET state = 'paused', "
|
||||||
"preemption_requested = 1, active_claim_token = NULL, "
|
"preemption_requested = 1, active_claim_token = NULL, "
|
||||||
"active_claimant_id = NULL, last_checkpoint_id = NULL, "
|
"active_claimant_id = NULL, claimed_at_utc = NULL, "
|
||||||
|
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
|
||||||
|
"claim_renewal_count = 0, last_checkpoint_id = NULL, "
|
||||||
"restart_from_zero = 1, preemption_receipt_sha256 = ?, "
|
"restart_from_zero = 1, preemption_receipt_sha256 = ?, "
|
||||||
"updated_at_utc = ? WHERE job_id = ?",
|
"updated_at_utc = ? WHERE job_id = ?",
|
||||||
(receipt.receipt_sha256, now, job.job_id),
|
(receipt.receipt_sha256, now, job.job_id),
|
||||||
@@ -1638,6 +1940,8 @@ class ObservatoryRecordedJobQueue:
|
|||||||
"result_sha256 = NULL, terminal_code = 'preemption-race', "
|
"result_sha256 = NULL, terminal_code = 'preemption-race', "
|
||||||
"terminal_message = ?, terminal_claim_token_sha256 = ?, "
|
"terminal_message = ?, terminal_claim_token_sha256 = ?, "
|
||||||
"active_claim_token = NULL, active_claimant_id = NULL, "
|
"active_claim_token = NULL, active_claimant_id = NULL, "
|
||||||
|
"claimed_at_utc = NULL, claim_expires_at_utc = NULL, "
|
||||||
|
"claim_heartbeat_at_utc = NULL, claim_renewal_count = 0, "
|
||||||
"preemption_receipt_sha256 = ?, updated_at_utc = ? "
|
"preemption_receipt_sha256 = ?, updated_at_utc = ? "
|
||||||
"WHERE job_id = ?",
|
"WHERE job_id = ?",
|
||||||
(
|
(
|
||||||
@@ -1707,6 +2011,7 @@ class ObservatoryRecordedJobQueue:
|
|||||||
self.data_dir.chmod(0o700)
|
self.data_dir.chmod(0o700)
|
||||||
with self._connect() as connection:
|
with self._connect() as connection:
|
||||||
connection.executescript(_SCHEMA_SQL)
|
connection.executescript(_SCHEMA_SQL)
|
||||||
|
self._migrate_claim_lease_schema(connection)
|
||||||
self._validate_schema(connection)
|
self._validate_schema(connection)
|
||||||
self._validate_existing_capacity(connection)
|
self._validate_existing_capacity(connection)
|
||||||
connection.commit()
|
connection.commit()
|
||||||
@@ -1723,7 +2028,7 @@ class ObservatoryRecordedJobQueue:
|
|||||||
|
|
||||||
def _validate_schema(self, connection: sqlite3.Connection) -> None:
|
def _validate_schema(self, connection: sqlite3.Connection) -> None:
|
||||||
expected = {
|
expected = {
|
||||||
"observatory_recorded_jobs": 42,
|
"observatory_recorded_jobs": 46,
|
||||||
"observatory_recorded_claim_receipts": 6,
|
"observatory_recorded_claim_receipts": 6,
|
||||||
"observatory_live_leases": 13,
|
"observatory_live_leases": 13,
|
||||||
"observatory_recorded_preemptions": 14,
|
"observatory_recorded_preemptions": 14,
|
||||||
@@ -1734,6 +2039,21 @@ class ObservatoryRecordedJobQueue:
|
|||||||
).fetchall()
|
).fetchall()
|
||||||
if len(columns) != column_count:
|
if len(columns) != column_count:
|
||||||
raise ObservatoryRecordedQueueIntegrityError(f"{table} schema is incompatible")
|
raise ObservatoryRecordedQueueIntegrityError(f"{table} schema is incompatible")
|
||||||
|
job_columns = {
|
||||||
|
str(row["name"])
|
||||||
|
for row in connection.execute(
|
||||||
|
"SELECT name FROM pragma_table_info('observatory_recorded_jobs')"
|
||||||
|
).fetchall()
|
||||||
|
}
|
||||||
|
if not {
|
||||||
|
"claimed_at_utc",
|
||||||
|
"claim_expires_at_utc",
|
||||||
|
"claim_heartbeat_at_utc",
|
||||||
|
"claim_renewal_count",
|
||||||
|
}.issubset(job_columns):
|
||||||
|
raise ObservatoryRecordedQueueIntegrityError(
|
||||||
|
"recorded-job claim lease schema is unavailable"
|
||||||
|
)
|
||||||
indexes = connection.execute(
|
indexes = connection.execute(
|
||||||
"SELECT name FROM sqlite_master WHERE type = 'index' "
|
"SELECT name FROM sqlite_master WHERE type = 'index' "
|
||||||
"AND name = 'observatory_one_open_live_lease'"
|
"AND name = 'observatory_one_open_live_lease'"
|
||||||
@@ -1743,6 +2063,75 @@ class ObservatoryRecordedJobQueue:
|
|||||||
"live K1 lease exclusivity index is unavailable"
|
"live K1 lease exclusivity index is unavailable"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _migrate_claim_lease_schema(self, connection: sqlite3.Connection) -> None:
|
||||||
|
"""Add renewable claim columns and safely fence legacy active owners."""
|
||||||
|
|
||||||
|
columns = {
|
||||||
|
str(row["name"])
|
||||||
|
for row in connection.execute(
|
||||||
|
"SELECT name FROM pragma_table_info('observatory_recorded_jobs')"
|
||||||
|
).fetchall()
|
||||||
|
}
|
||||||
|
additions = (
|
||||||
|
("claimed_at_utc", "TEXT"),
|
||||||
|
("claim_expires_at_utc", "TEXT"),
|
||||||
|
("claim_heartbeat_at_utc", "TEXT"),
|
||||||
|
(
|
||||||
|
"claim_renewal_count",
|
||||||
|
"INTEGER NOT NULL DEFAULT 0 CHECK (claim_renewal_count >= 0)",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
missing = [name for name, _definition in additions if name not in columns]
|
||||||
|
if not missing:
|
||||||
|
return
|
||||||
|
for name, definition in additions:
|
||||||
|
if name not in columns:
|
||||||
|
connection.execute(
|
||||||
|
f"ALTER TABLE observatory_recorded_jobs ADD COLUMN {name} {definition}"
|
||||||
|
)
|
||||||
|
|
||||||
|
now = self._timestamp()
|
||||||
|
rows = connection.execute(
|
||||||
|
"SELECT job_id, state, active_claim_token "
|
||||||
|
"FROM observatory_recorded_jobs WHERE active_claim_token IS NOT NULL"
|
||||||
|
).fetchall()
|
||||||
|
for row in rows:
|
||||||
|
job_id = str(row["job_id"])
|
||||||
|
state = str(row["state"])
|
||||||
|
claim_token = str(row["active_claim_token"])
|
||||||
|
if state in {"claimed", "paused"}:
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE observatory_recorded_jobs SET state = 'queued', "
|
||||||
|
"preemption_requested = 0, active_claim_token = NULL, "
|
||||||
|
"active_claimant_id = NULL, claimed_at_utc = NULL, "
|
||||||
|
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
|
||||||
|
"claim_renewal_count = 0, last_checkpoint_id = NULL, "
|
||||||
|
"restart_from_zero = ?, updated_at_utc = ? WHERE job_id = ?",
|
||||||
|
(int(state == "paused"), now, job_id),
|
||||||
|
)
|
||||||
|
elif state in {"running", "preemption-pending"}:
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE observatory_recorded_jobs "
|
||||||
|
"SET state = 'reconciliation-required', result_id = NULL, "
|
||||||
|
"result_sha256 = NULL, terminal_code = 'claim-lease-migration', "
|
||||||
|
"terminal_message = ?, terminal_claim_token_sha256 = ?, "
|
||||||
|
"active_claim_token = NULL, active_claimant_id = NULL, "
|
||||||
|
"claimed_at_utc = NULL, claim_expires_at_utc = NULL, "
|
||||||
|
"claim_heartbeat_at_utc = NULL, claim_renewal_count = 0, "
|
||||||
|
"updated_at_utc = ? WHERE job_id = ?",
|
||||||
|
(
|
||||||
|
"Legacy Worker execution had no expiring lease; physical "
|
||||||
|
"resource ownership requires reconciliation.",
|
||||||
|
hashlib.sha256(claim_token.encode()).hexdigest(),
|
||||||
|
now,
|
||||||
|
job_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ObservatoryRecordedQueueIntegrityError(
|
||||||
|
"legacy active claim is stored in an invalid state"
|
||||||
|
)
|
||||||
|
|
||||||
def _validate_existing_capacity(self, connection: sqlite3.Connection) -> None:
|
def _validate_existing_capacity(self, connection: sqlite3.Connection) -> None:
|
||||||
for table, limit, label in (
|
for table, limit, label in (
|
||||||
("observatory_recorded_jobs", self._max_jobs, "recorded job"),
|
("observatory_recorded_jobs", self._max_jobs, "recorded job"),
|
||||||
@@ -1901,6 +2290,10 @@ def _job_from_row(row: sqlite3.Row) -> ObservatoryRecordedJob:
|
|||||||
claim_generation=row["claim_generation"],
|
claim_generation=row["claim_generation"],
|
||||||
active_claim_token=row["active_claim_token"],
|
active_claim_token=row["active_claim_token"],
|
||||||
active_claimant_id=row["active_claimant_id"],
|
active_claimant_id=row["active_claimant_id"],
|
||||||
|
claimed_at_utc=row["claimed_at_utc"],
|
||||||
|
claim_expires_at_utc=row["claim_expires_at_utc"],
|
||||||
|
claim_heartbeat_at_utc=row["claim_heartbeat_at_utc"],
|
||||||
|
claim_renewal_count=row["claim_renewal_count"],
|
||||||
last_checkpoint_id=row["last_checkpoint_id"],
|
last_checkpoint_id=row["last_checkpoint_id"],
|
||||||
restart_from_zero=bool(row["restart_from_zero"]),
|
restart_from_zero=bool(row["restart_from_zero"]),
|
||||||
preemption_receipt_sha256=row["preemption_receipt_sha256"],
|
preemption_receipt_sha256=row["preemption_receipt_sha256"],
|
||||||
@@ -2029,6 +2422,17 @@ def _validate_quota(value: object, maximum: int, label: str) -> None:
|
|||||||
raise ValueError(f"{label} quota is invalid")
|
raise ValueError(f"{label} quota is invalid")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_claim_lease_seconds(value: object) -> None:
|
||||||
|
if (
|
||||||
|
not isinstance(value, int)
|
||||||
|
or isinstance(value, bool)
|
||||||
|
or not MIN_RECORDED_CLAIM_LEASE_SECONDS
|
||||||
|
<= value
|
||||||
|
<= MAX_RECORDED_CLAIM_LEASE_SECONDS
|
||||||
|
):
|
||||||
|
raise ValueError("recorded-job claim lease duration is invalid")
|
||||||
|
|
||||||
|
|
||||||
def _validate_positive_int(value: object, label: str) -> None:
|
def _validate_positive_int(value: object, label: str) -> None:
|
||||||
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
|
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
|
||||||
raise ValueError(f"{label} must be positive")
|
raise ValueError(f"{label} must be positive")
|
||||||
@@ -2059,6 +2463,10 @@ def _validate_text(value: object, label: str, *, max_length: int) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _validate_timestamp(value: object, label: str) -> None:
|
def _validate_timestamp(value: object, label: str) -> None:
|
||||||
|
_parse_timestamp(value, label)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_timestamp(value: object, label: str) -> datetime:
|
||||||
_validate_text(value, label, max_length=64)
|
_validate_text(value, label, max_length=64)
|
||||||
assert isinstance(value, str)
|
assert isinstance(value, str)
|
||||||
try:
|
try:
|
||||||
@@ -2067,6 +2475,12 @@ def _validate_timestamp(value: object, label: str) -> None:
|
|||||||
raise ValueError(f"{label} is invalid") from exc
|
raise ValueError(f"{label} is invalid") from exc
|
||||||
if parsed.tzinfo is None or parsed.utcoffset() != timedelta(0) or not value.endswith("Z"):
|
if parsed.tzinfo is None or parsed.utcoffset() != timedelta(0) or not value.endswith("Z"):
|
||||||
raise ValueError(f"{label} must use UTC")
|
raise ValueError(f"{label} must use UTC")
|
||||||
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
|
def _timestamp_after_seconds(value: str, seconds: int) -> str:
|
||||||
|
expires_at = _parse_timestamp(value, "queue timestamp") + timedelta(seconds=seconds)
|
||||||
|
return expires_at.isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
||||||
|
|
||||||
|
|
||||||
def _fsync_directory(path: Path) -> None:
|
def _fsync_directory(path: Path) -> None:
|
||||||
|
|||||||
@@ -12,8 +12,11 @@ import json
|
|||||||
import re
|
import re
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path, PurePosixPath
|
from pathlib import Path, PurePosixPath
|
||||||
from typing import Any, Final, Literal
|
from typing import Any, Final, Literal, cast
|
||||||
|
|
||||||
|
from k1link.observatory.canonical_result import (
|
||||||
|
is_admitted_observatory_recorded_result,
|
||||||
|
)
|
||||||
from k1link.sessions.models import SessionSummary
|
from k1link.sessions.models import SessionSummary
|
||||||
|
|
||||||
LABORATORY_SETUP_REGISTRY_SCHEMA: Final = (
|
LABORATORY_SETUP_REGISTRY_SCHEMA: Final = (
|
||||||
@@ -22,6 +25,9 @@ LABORATORY_SETUP_REGISTRY_SCHEMA: Final = (
|
|||||||
LABORATORY_SETUP_CATALOG_SCHEMA: Final = (
|
LABORATORY_SETUP_CATALOG_SCHEMA: Final = (
|
||||||
"missioncore.observatory-laboratory-setup-catalog/v1"
|
"missioncore.observatory-laboratory-setup-catalog/v1"
|
||||||
)
|
)
|
||||||
|
OBSERVATORY_CALCULATION_PROFILE_SCHEMA: Final = (
|
||||||
|
"missioncore.observatory-calculation-profile/v1"
|
||||||
|
)
|
||||||
_MAX_REGISTRY_BYTES: Final = 256 * 1024
|
_MAX_REGISTRY_BYTES: Final = 256 * 1024
|
||||||
_MAX_CONFIGURATION_BYTES: Final = 4 * 1024 * 1024
|
_MAX_CONFIGURATION_BYTES: Final = 4 * 1024 * 1024
|
||||||
_IDENTIFIER: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
_IDENTIFIER: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||||
@@ -282,6 +288,42 @@ class LaboratorySetupRegistry:
|
|||||||
return result.result_kind
|
return result.result_kind
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def observatory_calculation_profile(
|
||||||
|
self,
|
||||||
|
summary: SessionSummary,
|
||||||
|
) -> dict[str, object] | None:
|
||||||
|
"""Project an exact preserved legacy profile without inferring identity."""
|
||||||
|
|
||||||
|
for setup in self.setups:
|
||||||
|
for result in setup.preserved_results:
|
||||||
|
if (
|
||||||
|
result.access != "observatory"
|
||||||
|
or result.result_id != summary.session_id
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
if (
|
||||||
|
setup.origin != "existing-result"
|
||||||
|
or setup.run_definition is not None
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
if not is_admitted_observatory_recorded_result(
|
||||||
|
summary,
|
||||||
|
expected_result_id=result.result_id,
|
||||||
|
expected_source_session_id=setup.source_session_id,
|
||||||
|
expected_result_kind=result.result_kind,
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
return {
|
||||||
|
"schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||||
|
"setup_id": setup.setup_id,
|
||||||
|
"display_name": setup.display_name,
|
||||||
|
"origin": setup.origin,
|
||||||
|
"definition_id": None,
|
||||||
|
"definition_version": None,
|
||||||
|
"definition_sha256": None,
|
||||||
|
}
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _setup(value: object, *, repository_root: Path) -> LaboratorySetup:
|
def _setup(value: object, *, repository_root: Path) -> LaboratorySetup:
|
||||||
row = _object(value, "setup")
|
row = _object(value, "setup")
|
||||||
@@ -330,7 +372,7 @@ def _setup(value: object, *, repository_root: Path) -> LaboratorySetup:
|
|||||||
setup_id=_identifier(row["setup_id"], "setup_id"),
|
setup_id=_identifier(row["setup_id"], "setup_id"),
|
||||||
display_name=_text(row["display_name"], "display_name"),
|
display_name=_text(row["display_name"], "display_name"),
|
||||||
description=_text(row["description"], "description"),
|
description=_text(row["description"], "description"),
|
||||||
origin=origin,
|
origin=cast(SetupOrigin, origin),
|
||||||
source_session_id=_text(source["session_id"], "source session_id"),
|
source_session_id=_text(source["session_id"], "source session_id"),
|
||||||
source_label=_text(source["label"], "source label"),
|
source_label=_text(source["label"], "source label"),
|
||||||
required_modalities=modalities,
|
required_modalities=modalities,
|
||||||
@@ -427,7 +469,7 @@ def _preserved_result(value: object) -> _PreservedResult:
|
|||||||
result_id=result_id,
|
result_id=result_id,
|
||||||
result_kind=_identifier(row["result_kind"], "result_kind"),
|
result_kind=_identifier(row["result_kind"], "result_kind"),
|
||||||
relation=_identifier(row["relation"], "result relation"),
|
relation=_identifier(row["relation"], "result relation"),
|
||||||
access=access,
|
access=cast(ResultAccess, access),
|
||||||
created_at_utc=_text(row["created_at_utc"], "created_at_utc"),
|
created_at_utc=_text(row["created_at_utc"], "created_at_utc"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import re
|
|||||||
import secrets
|
import secrets
|
||||||
import stat
|
import stat
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass, replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Final
|
from typing import Final
|
||||||
|
|
||||||
@@ -29,6 +29,7 @@ from k1link.sessions.media import (
|
|||||||
)
|
)
|
||||||
from k1link.sessions.models import (
|
from k1link.sessions.models import (
|
||||||
RecordedMediaArtifact,
|
RecordedMediaArtifact,
|
||||||
|
ReplayArtifact,
|
||||||
ReplayCommand,
|
ReplayCommand,
|
||||||
SessionArtifact,
|
SessionArtifact,
|
||||||
SessionDetail,
|
SessionDetail,
|
||||||
@@ -40,6 +41,8 @@ PORTABLE_SOURCE_BUNDLE_SCHEMA: Final = "missioncore.portable-recorded-source-bun
|
|||||||
PORTABLE_SOURCE_CAPABILITY_SCHEMA: Final = "missioncore.portable-recorded-source-capability/v1"
|
PORTABLE_SOURCE_CAPABILITY_SCHEMA: Final = "missioncore.portable-recorded-source-capability/v1"
|
||||||
PORTABLE_SOURCE_ADAPTER_SCHEMA: Final = "missioncore.portable-source-adapter/v1"
|
PORTABLE_SOURCE_ADAPTER_SCHEMA: Final = "missioncore.portable-source-adapter/v1"
|
||||||
PORTABLE_SOURCE_DOCUMENT_DIRECTORY: Final = "observatory-portable-source-contracts"
|
PORTABLE_SOURCE_DOCUMENT_DIRECTORY: Final = "observatory-portable-source-contracts"
|
||||||
|
PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID: Final = "raw-transport-index"
|
||||||
|
PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE: Final = "application/x-ndjson"
|
||||||
|
|
||||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9.-]{2,127}$")
|
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9.-]{2,127}$")
|
||||||
@@ -384,10 +387,14 @@ class RecordedK1SourceAdmissionService:
|
|||||||
camera_source=camera_source,
|
camera_source=camera_source,
|
||||||
recorded_media=recorded_media,
|
recorded_media=recorded_media,
|
||||||
)
|
)
|
||||||
|
sealed_replay = _seal_replay_artifact_digests(
|
||||||
|
detail=detail,
|
||||||
|
replay=replay,
|
||||||
|
)
|
||||||
self._verify_replay(
|
self._verify_replay(
|
||||||
detail=detail,
|
detail=detail,
|
||||||
selected_sources=selected_sources,
|
selected_sources=selected_sources,
|
||||||
replay=replay,
|
replay=sealed_replay,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
media = (
|
media = (
|
||||||
@@ -410,7 +417,7 @@ class RecordedK1SourceAdmissionService:
|
|||||||
detail=detail,
|
detail=detail,
|
||||||
catalog_sha256=catalog_sha256,
|
catalog_sha256=catalog_sha256,
|
||||||
selected_sources=selected_sources,
|
selected_sources=selected_sources,
|
||||||
replay=replay,
|
replay=sealed_replay,
|
||||||
media=media,
|
media=media,
|
||||||
)
|
)
|
||||||
source_bundle = _canonical_json(source_bundle_document)
|
source_bundle = _canonical_json(source_bundle_document)
|
||||||
@@ -592,13 +599,31 @@ class RecordedK1SourceAdmissionService:
|
|||||||
raise PortableSourceAdmissionIntegrityError("spatial replay members are not unique")
|
raise PortableSourceAdmissionIntegrityError("spatial replay members are not unique")
|
||||||
for replay_artifact in replay.artifacts:
|
for replay_artifact in replay.artifacts:
|
||||||
catalog_artifact = catalog_artifacts.get(replay_artifact.artifact_id)
|
catalog_artifact = catalog_artifacts.get(replay_artifact.artifact_id)
|
||||||
|
exact_metadata_member = (
|
||||||
|
catalog_artifact is not None
|
||||||
|
and replay_artifact.artifact_id
|
||||||
|
== PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||||
|
and catalog_artifact.kind
|
||||||
|
== PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||||
|
and replay_artifact.media_type
|
||||||
|
== PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||||
|
and catalog_artifact.media_type
|
||||||
|
== PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||||
|
)
|
||||||
|
digest_matches_catalog = (
|
||||||
|
replay_artifact.expected_sha256 == catalog_artifact.sha256
|
||||||
|
if catalog_artifact is not None and catalog_artifact.sha256 is not None
|
||||||
|
else exact_metadata_member
|
||||||
|
and isinstance(replay_artifact.expected_sha256, str)
|
||||||
|
and _SHA256.fullmatch(replay_artifact.expected_sha256) is not None
|
||||||
|
)
|
||||||
if (
|
if (
|
||||||
catalog_artifact is None
|
catalog_artifact is None
|
||||||
or catalog_artifact.integrity_status not in _SEALED_ARTIFACT_STATES
|
or catalog_artifact.integrity_status not in _SEALED_ARTIFACT_STATES
|
||||||
or replay_artifact.media_type != catalog_artifact.media_type
|
or replay_artifact.media_type != catalog_artifact.media_type
|
||||||
or replay_artifact.file_byte_length != catalog_artifact.byte_length
|
or replay_artifact.file_byte_length != catalog_artifact.byte_length
|
||||||
or not 1 <= replay_artifact.replay_byte_length <= replay_artifact.file_byte_length
|
or not 1 <= replay_artifact.replay_byte_length <= replay_artifact.file_byte_length
|
||||||
or replay_artifact.expected_sha256 != catalog_artifact.sha256
|
or not digest_matches_catalog
|
||||||
):
|
):
|
||||||
raise PortableSourceAdmissionIntegrityError(
|
raise PortableSourceAdmissionIntegrityError(
|
||||||
"spatial replay member disagrees with the catalog"
|
"spatial replay member disagrees with the catalog"
|
||||||
@@ -799,6 +824,130 @@ class RecordedK1SourceAdmissionService:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _seal_replay_artifact_digests(
|
||||||
|
*,
|
||||||
|
detail: SessionDetail,
|
||||||
|
replay: ReplayCommand,
|
||||||
|
) -> ReplayCommand:
|
||||||
|
"""Seal the one legacy replay member whose catalog has no stored digest.
|
||||||
|
|
||||||
|
Historical K1 catalogs explicitly register ``mqtt.metadata.jsonl`` as the
|
||||||
|
``raw-transport-index`` replay artifact, but the catalog row predates a
|
||||||
|
persisted SHA-256 column value. Portable admission may derive that one
|
||||||
|
digest from the already confined ReplayCommand handle. No sibling-name or
|
||||||
|
directory discovery is permitted, and every other missing digest remains
|
||||||
|
an integrity failure in ``_verify_replay``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
catalog_artifacts = {artifact.artifact_id: artifact for artifact in detail.artifacts}
|
||||||
|
sealed: list[ReplayArtifact] = []
|
||||||
|
sealed_metadata_count = 0
|
||||||
|
for artifact in replay.artifacts:
|
||||||
|
if artifact.expected_sha256 is not None:
|
||||||
|
sealed.append(artifact)
|
||||||
|
continue
|
||||||
|
catalog_artifact = catalog_artifacts.get(artifact.artifact_id)
|
||||||
|
if (
|
||||||
|
catalog_artifact is None
|
||||||
|
or artifact.artifact_id
|
||||||
|
!= PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||||
|
or catalog_artifact.kind
|
||||||
|
!= PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||||
|
or artifact.media_type != PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||||
|
or catalog_artifact.media_type
|
||||||
|
!= PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||||
|
or catalog_artifact.sha256 is not None
|
||||||
|
or catalog_artifact.integrity_status not in _SEALED_ARTIFACT_STATES
|
||||||
|
or artifact.file_byte_length != catalog_artifact.byte_length
|
||||||
|
or artifact.replay_byte_length != catalog_artifact.byte_length
|
||||||
|
):
|
||||||
|
sealed.append(artifact)
|
||||||
|
continue
|
||||||
|
sealed_metadata_count += 1
|
||||||
|
if sealed_metadata_count != 1:
|
||||||
|
raise PortableSourceAdmissionIntegrityError(
|
||||||
|
"spatial replay metadata member is not unique"
|
||||||
|
)
|
||||||
|
sha256 = _hash_confined_replay_artifact(replay=replay, artifact=artifact)
|
||||||
|
sealed.append(replace(artifact, expected_sha256=sha256))
|
||||||
|
return replace(replay, artifacts=tuple(sealed))
|
||||||
|
|
||||||
|
|
||||||
|
def _hash_confined_replay_artifact(
|
||||||
|
*,
|
||||||
|
replay: ReplayCommand,
|
||||||
|
artifact: ReplayArtifact,
|
||||||
|
) -> str:
|
||||||
|
descriptor = -1
|
||||||
|
try:
|
||||||
|
allowed_root = replay.allowed_root.resolve(strict=True)
|
||||||
|
session_root = replay.session_root.resolve(strict=True)
|
||||||
|
source_metadata = artifact.path.lstat()
|
||||||
|
source_path = artifact.path.resolve(strict=True)
|
||||||
|
if (
|
||||||
|
not allowed_root.is_dir()
|
||||||
|
or not session_root.is_dir()
|
||||||
|
or not session_root.is_relative_to(allowed_root)
|
||||||
|
or not source_path.is_relative_to(session_root)
|
||||||
|
or stat.S_ISLNK(source_metadata.st_mode)
|
||||||
|
or not stat.S_ISREG(source_metadata.st_mode)
|
||||||
|
or not 1 <= artifact.file_byte_length <= MAX_SAFE_INTEGER
|
||||||
|
):
|
||||||
|
raise PortableSourceAdmissionIntegrityError(
|
||||||
|
"spatial replay metadata escapes its admitted session root"
|
||||||
|
)
|
||||||
|
descriptor = os.open(
|
||||||
|
source_path,
|
||||||
|
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
|
||||||
|
)
|
||||||
|
before = os.fstat(descriptor)
|
||||||
|
if (
|
||||||
|
not stat.S_ISREG(before.st_mode)
|
||||||
|
or before.st_size != artifact.file_byte_length
|
||||||
|
or before.st_size != artifact.replay_byte_length
|
||||||
|
):
|
||||||
|
raise PortableSourceAdmissionIntegrityError(
|
||||||
|
"spatial replay metadata is outside admitted bounds"
|
||||||
|
)
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
byte_length = 0
|
||||||
|
while chunk := os.read(descriptor, 1024 * 1024):
|
||||||
|
byte_length += len(chunk)
|
||||||
|
if byte_length > artifact.file_byte_length:
|
||||||
|
raise PortableSourceAdmissionIntegrityError(
|
||||||
|
"spatial replay metadata grew while it was sealed"
|
||||||
|
)
|
||||||
|
digest.update(chunk)
|
||||||
|
after = os.fstat(descriptor)
|
||||||
|
stable_identity = (
|
||||||
|
before.st_dev,
|
||||||
|
before.st_ino,
|
||||||
|
before.st_size,
|
||||||
|
before.st_mtime_ns,
|
||||||
|
before.st_ctime_ns,
|
||||||
|
) == (
|
||||||
|
after.st_dev,
|
||||||
|
after.st_ino,
|
||||||
|
after.st_size,
|
||||||
|
after.st_mtime_ns,
|
||||||
|
after.st_ctime_ns,
|
||||||
|
)
|
||||||
|
if byte_length != artifact.file_byte_length or not stable_identity:
|
||||||
|
raise PortableSourceAdmissionIntegrityError(
|
||||||
|
"spatial replay metadata changed while it was sealed"
|
||||||
|
)
|
||||||
|
return digest.hexdigest()
|
||||||
|
except PortableSourceAdmissionIntegrityError:
|
||||||
|
raise
|
||||||
|
except OSError as exc:
|
||||||
|
raise PortableSourceAdmissionIntegrityError(
|
||||||
|
"spatial replay metadata is unavailable"
|
||||||
|
) from exc
|
||||||
|
finally:
|
||||||
|
if descriptor >= 0:
|
||||||
|
os.close(descriptor)
|
||||||
|
|
||||||
|
|
||||||
def _read_canonical_camera_summary(
|
def _read_canonical_camera_summary(
|
||||||
source_path: Path,
|
source_path: Path,
|
||||||
) -> tuple[dict[str, object], int]:
|
) -> tuple[dict[str, object], int]:
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import re
|
|||||||
import threading
|
import threading
|
||||||
from collections.abc import Callable, Mapping
|
from collections.abc import Callable, Mapping
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime
|
||||||
from typing import Annotated, Final, Literal, Protocol
|
from typing import Annotated, Final, Literal, Protocol
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
@@ -51,6 +52,7 @@ type WorkerCycleState = Literal[
|
|||||||
"succeeded",
|
"succeeded",
|
||||||
"failed",
|
"failed",
|
||||||
"rejected",
|
"rejected",
|
||||||
|
"lease-lost",
|
||||||
]
|
]
|
||||||
type RecordedJobWireState = Literal[
|
type RecordedJobWireState = Literal[
|
||||||
"accepted",
|
"accepted",
|
||||||
@@ -122,6 +124,7 @@ class SealedObservatoryRecordedJob:
|
|||||||
job_id: str
|
job_id: str
|
||||||
request_sha256: str
|
request_sha256: str
|
||||||
identity_sha256: str
|
identity_sha256: str
|
||||||
|
submission_receipt_sha256: str
|
||||||
source_session_id: str
|
source_session_id: str
|
||||||
source_catalog_sha256: str
|
source_catalog_sha256: str
|
||||||
source_bundle_sha256: str
|
source_bundle_sha256: str
|
||||||
@@ -140,6 +143,10 @@ class SealedObservatoryRecordedJob:
|
|||||||
checkpoint_policy: Literal["cooperative", "non-checkpointable"]
|
checkpoint_policy: Literal["cooperative", "non-checkpointable"]
|
||||||
allowed_checkpoints: tuple[str, ...]
|
allowed_checkpoints: tuple[str, ...]
|
||||||
claim_generation: int
|
claim_generation: int
|
||||||
|
claim_claimed_at_utc: str | None
|
||||||
|
claim_expires_at_utc: str | None
|
||||||
|
claim_heartbeat_at_utc: str | None
|
||||||
|
claim_renewal_count: int
|
||||||
restart_from_zero: bool
|
restart_from_zero: bool
|
||||||
|
|
||||||
|
|
||||||
@@ -213,6 +220,16 @@ class ObservatoryWorkerTransport(Protocol):
|
|||||||
claim_token: str,
|
claim_token: str,
|
||||||
) -> Mapping[str, object]: ...
|
) -> Mapping[str, object]: ...
|
||||||
|
|
||||||
|
def renew_claim(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
claimant_id: str,
|
||||||
|
job_id: str,
|
||||||
|
claim_token: str,
|
||||||
|
claim_generation: int,
|
||||||
|
heartbeat_sequence: int,
|
||||||
|
) -> Mapping[str, object]: ...
|
||||||
|
|
||||||
def succeed(
|
def succeed(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -308,6 +325,13 @@ class _TerminalPayload(_StrictPayload):
|
|||||||
message: str = Field(min_length=1, max_length=1_000)
|
message: str = Field(min_length=1, max_length=1_000)
|
||||||
|
|
||||||
|
|
||||||
|
class _ClaimLeasePayload(_StrictPayload):
|
||||||
|
claimed_at_utc: Timestamp
|
||||||
|
expires_at_utc: Timestamp
|
||||||
|
heartbeat_at_utc: Timestamp
|
||||||
|
renewal_count: int = Field(ge=0)
|
||||||
|
|
||||||
|
|
||||||
class _RecordedJobPayload(_StrictPayload):
|
class _RecordedJobPayload(_StrictPayload):
|
||||||
schema_version: Literal["missioncore.observatory-recorded-job/v1"]
|
schema_version: Literal["missioncore.observatory-recorded-job/v1"]
|
||||||
job_id: str = Field(pattern=_JOB_ID_PATTERN)
|
job_id: str = Field(pattern=_JOB_ID_PATTERN)
|
||||||
@@ -329,6 +353,7 @@ class _RecordedJobPayload(_StrictPayload):
|
|||||||
restart_from_zero: bool
|
restart_from_zero: bool
|
||||||
preemption_receipt_sha256: Sha256 | None
|
preemption_receipt_sha256: Sha256 | None
|
||||||
claim_generation: int = Field(ge=0)
|
claim_generation: int = Field(ge=0)
|
||||||
|
claim_lease: _ClaimLeasePayload | None
|
||||||
result: _ResultPayload | None
|
result: _ResultPayload | None
|
||||||
terminal: _TerminalPayload | None
|
terminal: _TerminalPayload | None
|
||||||
created_at_utc: Timestamp
|
created_at_utc: Timestamp
|
||||||
@@ -365,10 +390,20 @@ class ObservatoryWorkerAgent:
|
|||||||
transport: ObservatoryWorkerTransport,
|
transport: ObservatoryWorkerTransport,
|
||||||
executors: ObservatoryWorkerExecutorRegistry,
|
executors: ObservatoryWorkerExecutorRegistry,
|
||||||
claim_request_id_factory: Callable[[], str] | None = None,
|
claim_request_id_factory: Callable[[], str] | None = None,
|
||||||
|
heartbeat_interval_seconds: float | None = None,
|
||||||
|
heartbeat_stop_timeout_seconds: float = 5.0,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
if heartbeat_interval_seconds is not None and not (
|
||||||
|
0.01 <= heartbeat_interval_seconds <= 300.0
|
||||||
|
):
|
||||||
|
raise ValueError("Worker heartbeat interval is invalid")
|
||||||
|
if not 0.1 <= heartbeat_stop_timeout_seconds <= 300.0:
|
||||||
|
raise ValueError("Worker heartbeat stop timeout is invalid")
|
||||||
self._transport = transport
|
self._transport = transport
|
||||||
self._executors = executors
|
self._executors = executors
|
||||||
self._claim_request_id_factory = claim_request_id_factory or _default_claim_request_id
|
self._claim_request_id_factory = claim_request_id_factory or _default_claim_request_id
|
||||||
|
self._heartbeat_interval_seconds = heartbeat_interval_seconds
|
||||||
|
self._heartbeat_stop_timeout_seconds = heartbeat_stop_timeout_seconds
|
||||||
self._cycle_lock = threading.Lock()
|
self._cycle_lock = threading.Lock()
|
||||||
|
|
||||||
def run_once(self) -> ObservatoryWorkerCycleReport:
|
def run_once(self) -> ObservatoryWorkerCycleReport:
|
||||||
@@ -443,11 +478,33 @@ class ObservatoryWorkerAgent:
|
|||||||
job_id=claim.job.job_id,
|
job_id=claim.job.job_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
active_job = _seal_job(started)
|
||||||
|
heartbeat = _ClaimHeartbeat(
|
||||||
|
transport=self._transport,
|
||||||
|
job=active_job,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
interval_seconds=(
|
||||||
|
self._heartbeat_interval_seconds
|
||||||
|
if self._heartbeat_interval_seconds is not None
|
||||||
|
else _default_heartbeat_interval(active_job)
|
||||||
|
),
|
||||||
|
stop_timeout_seconds=self._heartbeat_stop_timeout_seconds,
|
||||||
|
)
|
||||||
|
heartbeat.start()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = adapter.execute(claim.job)
|
result = adapter.execute(active_job)
|
||||||
if not isinstance(result, ObservatoryWorkerExecutionResult):
|
if not isinstance(result, ObservatoryWorkerExecutionResult):
|
||||||
raise TypeError("executor returned an unknown result contract")
|
raise TypeError("executor returned an unknown result contract")
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
heartbeat.stop()
|
||||||
|
if heartbeat.failed:
|
||||||
|
return ObservatoryWorkerCycleReport(
|
||||||
|
state="lease-lost",
|
||||||
|
claim_request_id=claim_request_id,
|
||||||
|
job_id=claim.job.job_id,
|
||||||
|
failure_code="claim-heartbeat-lost",
|
||||||
|
)
|
||||||
failure_code = "executor-error"
|
failure_code = "executor-error"
|
||||||
acknowledgement = self._transport.fail(
|
acknowledgement = self._transport.fail(
|
||||||
claimant_id=WORKER_006_CONTOUR_ID,
|
claimant_id=WORKER_006_CONTOUR_ID,
|
||||||
@@ -468,6 +525,15 @@ class ObservatoryWorkerAgent:
|
|||||||
failure_code=failure_code,
|
failure_code=failure_code,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
heartbeat.stop()
|
||||||
|
if heartbeat.failed:
|
||||||
|
return ObservatoryWorkerCycleReport(
|
||||||
|
state="lease-lost",
|
||||||
|
claim_request_id=claim_request_id,
|
||||||
|
job_id=claim.job.job_id,
|
||||||
|
failure_code="claim-heartbeat-lost",
|
||||||
|
)
|
||||||
|
|
||||||
acknowledgement = self._transport.succeed(
|
acknowledgement = self._transport.succeed(
|
||||||
claimant_id=WORKER_006_CONTOUR_ID,
|
claimant_id=WORKER_006_CONTOUR_ID,
|
||||||
job_id=claim.job.job_id,
|
job_id=claim.job.job_id,
|
||||||
@@ -495,6 +561,92 @@ class ObservatoryWorkerAgent:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _ClaimHeartbeat:
|
||||||
|
"""Renew one exact generation while an executor owns Worker resources."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
transport: ObservatoryWorkerTransport,
|
||||||
|
job: SealedObservatoryRecordedJob,
|
||||||
|
claim_token: str,
|
||||||
|
interval_seconds: float,
|
||||||
|
stop_timeout_seconds: float,
|
||||||
|
) -> None:
|
||||||
|
if interval_seconds <= 0:
|
||||||
|
raise ValueError("Worker heartbeat interval must be positive")
|
||||||
|
self._transport = transport
|
||||||
|
self._job = job
|
||||||
|
self._claim_token = claim_token
|
||||||
|
self._interval_seconds = interval_seconds
|
||||||
|
self._stop_timeout_seconds = stop_timeout_seconds
|
||||||
|
self._stop = threading.Event()
|
||||||
|
self._state_lock = threading.Lock()
|
||||||
|
self._failure: Exception | None = None
|
||||||
|
self._thread = threading.Thread(
|
||||||
|
target=self._run,
|
||||||
|
name=f"observatory-heartbeat-{job.job_id}",
|
||||||
|
daemon=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def failed(self) -> bool:
|
||||||
|
with self._state_lock:
|
||||||
|
return self._failure is not None
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
self._thread.start()
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
self._stop.set()
|
||||||
|
self._thread.join(timeout=self._stop_timeout_seconds)
|
||||||
|
if self._thread.is_alive():
|
||||||
|
self._record_failure(
|
||||||
|
ObservatoryWorkerClaimRejectedError(
|
||||||
|
"Worker claim heartbeat did not stop within its bound"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _run(self) -> None:
|
||||||
|
sequence = self._job.claim_renewal_count + 1
|
||||||
|
# Renew immediately once start has been acknowledged. Waiting a full
|
||||||
|
# interval here would assume that claim/start transport latency consumed
|
||||||
|
# none of the original lease window.
|
||||||
|
while not self._stop.is_set():
|
||||||
|
try:
|
||||||
|
acknowledgement = self._transport.renew_claim(
|
||||||
|
claimant_id=WORKER_006_CONTOUR_ID,
|
||||||
|
job_id=self._job.job_id,
|
||||||
|
claim_token=self._claim_token,
|
||||||
|
claim_generation=self._job.claim_generation,
|
||||||
|
heartbeat_sequence=sequence,
|
||||||
|
)
|
||||||
|
renewed = _validate_transition_acknowledgement(
|
||||||
|
acknowledgement,
|
||||||
|
expected_job=self._job,
|
||||||
|
expected_state=("claimed", "running"),
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
renewed.claim_lease is None
|
||||||
|
or renewed.claim_lease.renewal_count != sequence
|
||||||
|
):
|
||||||
|
raise ObservatoryWorkerClaimRejectedError(
|
||||||
|
"Worker heartbeat acknowledgement changed its sequence"
|
||||||
|
)
|
||||||
|
sequence += 1
|
||||||
|
except Exception as exc:
|
||||||
|
self._record_failure(exc)
|
||||||
|
self._stop.set()
|
||||||
|
return
|
||||||
|
if self._stop.wait(self._interval_seconds):
|
||||||
|
return
|
||||||
|
|
||||||
|
def _record_failure(self, exc: Exception) -> None:
|
||||||
|
with self._state_lock:
|
||||||
|
if self._failure is None:
|
||||||
|
self._failure = exc
|
||||||
|
|
||||||
|
|
||||||
def _validate_claim(
|
def _validate_claim(
|
||||||
payload: Mapping[str, object],
|
payload: Mapping[str, object],
|
||||||
*,
|
*,
|
||||||
@@ -519,6 +671,10 @@ def _validate_claim(
|
|||||||
raise ObservatoryWorkerClaimRejectedError(
|
raise ObservatoryWorkerClaimRejectedError(
|
||||||
"Worker claim job is not in a claimed generation"
|
"Worker claim job is not in a claimed generation"
|
||||||
)
|
)
|
||||||
|
if claim.job.claim_lease is None:
|
||||||
|
raise ObservatoryWorkerClaimRejectedError(
|
||||||
|
"Worker claim has no renewable lease"
|
||||||
|
)
|
||||||
if claim.job.result is not None or claim.job.terminal is not None:
|
if claim.job.result is not None or claim.job.terminal is not None:
|
||||||
raise ObservatoryWorkerClaimRejectedError(
|
raise ObservatoryWorkerClaimRejectedError(
|
||||||
"Worker claim already carries a terminal outcome"
|
"Worker claim already carries a terminal outcome"
|
||||||
@@ -556,6 +712,14 @@ def _seal_job(payload: _RecordedJobPayload) -> SealedObservatoryRecordedJob:
|
|||||||
and payload.checkpoint_policy.allowed_checkpoints
|
and payload.checkpoint_policy.allowed_checkpoints
|
||||||
):
|
):
|
||||||
raise ObservatoryWorkerClaimRejectedError("Worker claim checkpoint policy is inconsistent")
|
raise ObservatoryWorkerClaimRejectedError("Worker claim checkpoint policy is inconsistent")
|
||||||
|
if payload.claim_lease is not None:
|
||||||
|
claimed_at = _parse_timestamp(payload.claim_lease.claimed_at_utc)
|
||||||
|
expires_at = _parse_timestamp(payload.claim_lease.expires_at_utc)
|
||||||
|
heartbeat_at = _parse_timestamp(payload.claim_lease.heartbeat_at_utc)
|
||||||
|
if not claimed_at <= heartbeat_at < expires_at:
|
||||||
|
raise ObservatoryWorkerClaimRejectedError(
|
||||||
|
"Worker claim lease chronology is inconsistent"
|
||||||
|
)
|
||||||
|
|
||||||
expected_request_sha256 = _sha256_document(
|
expected_request_sha256 = _sha256_document(
|
||||||
{
|
{
|
||||||
@@ -615,6 +779,7 @@ def _seal_job(payload: _RecordedJobPayload) -> SealedObservatoryRecordedJob:
|
|||||||
job_id=payload.job_id,
|
job_id=payload.job_id,
|
||||||
request_sha256=payload.request_sha256,
|
request_sha256=payload.request_sha256,
|
||||||
identity_sha256=payload.identity_sha256,
|
identity_sha256=payload.identity_sha256,
|
||||||
|
submission_receipt_sha256=payload.submission_receipt_sha256,
|
||||||
source_session_id=payload.source.session_id,
|
source_session_id=payload.source.session_id,
|
||||||
source_catalog_sha256=payload.source.catalog_sha256,
|
source_catalog_sha256=payload.source.catalog_sha256,
|
||||||
source_bundle_sha256=payload.source.bundle_sha256,
|
source_bundle_sha256=payload.source.bundle_sha256,
|
||||||
@@ -638,6 +803,18 @@ def _seal_job(payload: _RecordedJobPayload) -> SealedObservatoryRecordedJob:
|
|||||||
checkpoint_policy=payload.checkpoint_policy.mode,
|
checkpoint_policy=payload.checkpoint_policy.mode,
|
||||||
allowed_checkpoints=tuple(payload.checkpoint_policy.allowed_checkpoints),
|
allowed_checkpoints=tuple(payload.checkpoint_policy.allowed_checkpoints),
|
||||||
claim_generation=payload.claim_generation,
|
claim_generation=payload.claim_generation,
|
||||||
|
claim_claimed_at_utc=(
|
||||||
|
None if payload.claim_lease is None else payload.claim_lease.claimed_at_utc
|
||||||
|
),
|
||||||
|
claim_expires_at_utc=(
|
||||||
|
None if payload.claim_lease is None else payload.claim_lease.expires_at_utc
|
||||||
|
),
|
||||||
|
claim_heartbeat_at_utc=(
|
||||||
|
None if payload.claim_lease is None else payload.claim_lease.heartbeat_at_utc
|
||||||
|
),
|
||||||
|
claim_renewal_count=(
|
||||||
|
0 if payload.claim_lease is None else payload.claim_lease.renewal_count
|
||||||
|
),
|
||||||
restart_from_zero=payload.restart_from_zero,
|
restart_from_zero=payload.restart_from_zero,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -662,6 +839,12 @@ def _validate_transition_acknowledgement(
|
|||||||
raise ObservatoryWorkerClaimRejectedError(
|
raise ObservatoryWorkerClaimRejectedError(
|
||||||
"Worker transition acknowledgement has an unexpected state"
|
"Worker transition acknowledgement has an unexpected state"
|
||||||
)
|
)
|
||||||
|
if acknowledgement.state in {"claimed", "running", "preemption-pending"} and (
|
||||||
|
acknowledgement.claim_lease is None
|
||||||
|
):
|
||||||
|
raise ObservatoryWorkerClaimRejectedError(
|
||||||
|
"Worker transition acknowledgement lost the active claim lease"
|
||||||
|
)
|
||||||
if (
|
if (
|
||||||
sealed.job_id != expected_job.job_id
|
sealed.job_id != expected_job.job_id
|
||||||
or sealed.identity_sha256 != expected_job.identity_sha256
|
or sealed.identity_sha256 != expected_job.identity_sha256
|
||||||
@@ -677,6 +860,20 @@ def _default_claim_request_id() -> str:
|
|||||||
return f"worker-006:{uuid4().hex}"
|
return f"worker-006:{uuid4().hex}"
|
||||||
|
|
||||||
|
|
||||||
|
def _default_heartbeat_interval(job: SealedObservatoryRecordedJob) -> float:
|
||||||
|
if job.claim_heartbeat_at_utc is None or job.claim_expires_at_utc is None:
|
||||||
|
raise ObservatoryWorkerClaimRejectedError(
|
||||||
|
"Worker claim has no heartbeat lease bounds"
|
||||||
|
)
|
||||||
|
remaining = (
|
||||||
|
_parse_timestamp(job.claim_expires_at_utc)
|
||||||
|
- _parse_timestamp(job.claim_heartbeat_at_utc)
|
||||||
|
).total_seconds()
|
||||||
|
if remaining <= 0:
|
||||||
|
raise ObservatoryWorkerClaimRejectedError("Worker claim lease already expired")
|
||||||
|
return max(0.5, min(30.0, remaining / 3.0))
|
||||||
|
|
||||||
|
|
||||||
def _bounded_executor_failure(exc: Exception) -> str:
|
def _bounded_executor_failure(exc: Exception) -> str:
|
||||||
detail = " ".join(str(exc).split())
|
detail = " ".join(str(exc).split())
|
||||||
message = f"Executor adapter raised {type(exc).__name__}."
|
message = f"Executor adapter raised {type(exc).__name__}."
|
||||||
@@ -693,3 +890,7 @@ def _sha256_document(document: Mapping[str, object]) -> str:
|
|||||||
separators=(",", ":"),
|
separators=(",", ":"),
|
||||||
).encode()
|
).encode()
|
||||||
return hashlib.sha256(payload).hexdigest()
|
return hashlib.sha256(payload).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_timestamp(value: str) -> datetime:
|
||||||
|
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,357 @@
|
|||||||
|
"""Install-time composition and bounded polling for portable Worker 006.
|
||||||
|
|
||||||
|
The durable service owns transport cadence only. A reviewed Worker release
|
||||||
|
must inject an in-memory executor registry whose four-digest identities cover
|
||||||
|
every portable RunDefinition advertised as ready. Configuration cannot name
|
||||||
|
Python modules, commands, images, or executable paths, so neither Mission Core
|
||||||
|
nor an environment variable can turn an unsealed candidate into code.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import stat
|
||||||
|
from collections.abc import Callable, Mapping
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from threading import Event
|
||||||
|
from typing import Final
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry
|
||||||
|
from k1link.observatory.worker_agent import (
|
||||||
|
ObservatoryWorkerAgent,
|
||||||
|
ObservatoryWorkerCycleReport,
|
||||||
|
ObservatoryWorkerExecutorIdentity,
|
||||||
|
ObservatoryWorkerExecutorRegistry,
|
||||||
|
ObservatoryWorkerExecutorUnavailableError,
|
||||||
|
)
|
||||||
|
from k1link.observatory.worker_http_transport import (
|
||||||
|
ObservatoryWorkerHttpError,
|
||||||
|
ObservatoryWorkerHttpGateway,
|
||||||
|
)
|
||||||
|
|
||||||
|
OBSERVATORY_WORKER_BASE_URL_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_BASE_URL"
|
||||||
|
OBSERVATORY_WORKER_TOKEN_FILE_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE"
|
||||||
|
OBSERVATORY_WORKER_WORK_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT"
|
||||||
|
OBSERVATORY_WORKER_IDLE_POLL_SECONDS_ENV: Final = (
|
||||||
|
"MISSIONCORE_OBSERVATORY_WORKER_IDLE_POLL_SECONDS"
|
||||||
|
)
|
||||||
|
OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS_ENV: Final = (
|
||||||
|
"MISSIONCORE_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS"
|
||||||
|
)
|
||||||
|
OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES_ENV: Final = (
|
||||||
|
"MISSIONCORE_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES"
|
||||||
|
)
|
||||||
|
|
||||||
|
DEFAULT_OBSERVATORY_WORKER_BASE_URL: Final = "http://127.0.0.1:18080"
|
||||||
|
DEFAULT_OBSERVATORY_WORKER_IDLE_POLL_SECONDS: Final = 1.0
|
||||||
|
DEFAULT_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS: Final = 5.0
|
||||||
|
DEFAULT_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES: Final = 12
|
||||||
|
|
||||||
|
_TOKEN = re.compile(r"^[A-Za-z0-9._:-]{32,512}$")
|
||||||
|
|
||||||
|
|
||||||
|
class ObservatoryWorkerServiceError(RuntimeError):
|
||||||
|
"""Worker 006 cannot start without its exact local operational boundary."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ObservatoryWorkerServiceConfiguration:
|
||||||
|
"""Path-only service configuration; it never contains a plaintext secret."""
|
||||||
|
|
||||||
|
base_url: str
|
||||||
|
bearer_token_file: Path
|
||||||
|
work_root: Path
|
||||||
|
idle_poll_seconds: float = DEFAULT_OBSERVATORY_WORKER_IDLE_POLL_SECONDS
|
||||||
|
transport_backoff_seconds: float = (
|
||||||
|
DEFAULT_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS
|
||||||
|
)
|
||||||
|
max_consecutive_transport_failures: int = (
|
||||||
|
DEFAULT_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES
|
||||||
|
)
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_validate_worker_base_url(self.base_url)
|
||||||
|
_absolute_path(self.bearer_token_file, "Worker bearer token file")
|
||||||
|
_absolute_path(self.work_root, "Worker work root")
|
||||||
|
if isinstance(self.idle_poll_seconds, bool) or not (
|
||||||
|
0.05 <= self.idle_poll_seconds <= 300.0
|
||||||
|
):
|
||||||
|
raise ValueError("Worker idle poll interval is invalid")
|
||||||
|
if isinstance(self.transport_backoff_seconds, bool) or not (
|
||||||
|
0.05 <= self.transport_backoff_seconds <= 300.0
|
||||||
|
):
|
||||||
|
raise ValueError("Worker transport backoff is invalid")
|
||||||
|
if (
|
||||||
|
isinstance(self.max_consecutive_transport_failures, bool)
|
||||||
|
or not isinstance(self.max_consecutive_transport_failures, int)
|
||||||
|
or not 1 <= self.max_consecutive_transport_failures <= 10_000
|
||||||
|
):
|
||||||
|
raise ValueError("Worker transport failure bound is invalid")
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_environment(
|
||||||
|
cls,
|
||||||
|
environment: Mapping[str, str] | None = None,
|
||||||
|
) -> ObservatoryWorkerServiceConfiguration:
|
||||||
|
values = os.environ if environment is None else environment
|
||||||
|
token_file = _required_path(values, OBSERVATORY_WORKER_TOKEN_FILE_ENV)
|
||||||
|
work_root = _required_path(values, OBSERVATORY_WORKER_WORK_ROOT_ENV)
|
||||||
|
return cls(
|
||||||
|
base_url=values.get(
|
||||||
|
OBSERVATORY_WORKER_BASE_URL_ENV,
|
||||||
|
DEFAULT_OBSERVATORY_WORKER_BASE_URL,
|
||||||
|
),
|
||||||
|
bearer_token_file=token_file,
|
||||||
|
work_root=work_root,
|
||||||
|
idle_poll_seconds=_environment_float(
|
||||||
|
values,
|
||||||
|
OBSERVATORY_WORKER_IDLE_POLL_SECONDS_ENV,
|
||||||
|
DEFAULT_OBSERVATORY_WORKER_IDLE_POLL_SECONDS,
|
||||||
|
),
|
||||||
|
transport_backoff_seconds=_environment_float(
|
||||||
|
values,
|
||||||
|
OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS_ENV,
|
||||||
|
DEFAULT_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS,
|
||||||
|
),
|
||||||
|
max_consecutive_transport_failures=_environment_int(
|
||||||
|
values,
|
||||||
|
OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES_ENV,
|
||||||
|
DEFAULT_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class InstalledObservatoryWorkerService:
|
||||||
|
"""A composed gateway and agent owned by one installed Worker release."""
|
||||||
|
|
||||||
|
configuration: ObservatoryWorkerServiceConfiguration
|
||||||
|
gateway: ObservatoryWorkerHttpGateway
|
||||||
|
agent: ObservatoryWorkerAgent
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self.gateway.close()
|
||||||
|
|
||||||
|
def run(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
stop: Event,
|
||||||
|
on_cycle: Callable[[ObservatoryWorkerCycleReport], None] | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Poll until stopped, with a bounded consecutive transport-failure gate."""
|
||||||
|
|
||||||
|
consecutive_transport_failures = 0
|
||||||
|
try:
|
||||||
|
while not stop.is_set():
|
||||||
|
try:
|
||||||
|
report = self.agent.run_once()
|
||||||
|
except ObservatoryWorkerHttpError as exc:
|
||||||
|
consecutive_transport_failures += 1
|
||||||
|
if (
|
||||||
|
consecutive_transport_failures
|
||||||
|
>= self.configuration.max_consecutive_transport_failures
|
||||||
|
):
|
||||||
|
raise ObservatoryWorkerServiceError(
|
||||||
|
"Worker transport exceeded its consecutive failure bound"
|
||||||
|
) from exc
|
||||||
|
stop.wait(self.configuration.transport_backoff_seconds)
|
||||||
|
continue
|
||||||
|
consecutive_transport_failures = 0
|
||||||
|
if on_cycle is not None:
|
||||||
|
on_cycle(report)
|
||||||
|
if report.state in {
|
||||||
|
"empty",
|
||||||
|
"deferred",
|
||||||
|
"rejected",
|
||||||
|
"lease-lost",
|
||||||
|
}:
|
||||||
|
stop.wait(self.configuration.idle_poll_seconds)
|
||||||
|
finally:
|
||||||
|
self.close()
|
||||||
|
|
||||||
|
|
||||||
|
def compose_installed_observatory_worker_service(
|
||||||
|
*,
|
||||||
|
configuration: ObservatoryWorkerServiceConfiguration,
|
||||||
|
definitions: PortableRunDefinitionRegistry,
|
||||||
|
executors: ObservatoryWorkerExecutorRegistry,
|
||||||
|
http_transport: httpx.BaseTransport | None = None,
|
||||||
|
) -> InstalledObservatoryWorkerService:
|
||||||
|
"""Bind transport to an install-time registry after exact coverage checks.
|
||||||
|
|
||||||
|
This is the fixed seam a reviewed Worker release calls. There is no
|
||||||
|
dynamic import/provider name in service configuration. Blocked catalog
|
||||||
|
candidates are ignored, while an empty ready set or any missing exact
|
||||||
|
local executor identity rejects service startup before the first claim.
|
||||||
|
"""
|
||||||
|
|
||||||
|
require_ready_executor_coverage(definitions=definitions, executors=executors)
|
||||||
|
bearer_token = load_observatory_worker_bearer_token(
|
||||||
|
configuration.bearer_token_file
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
gateway = ObservatoryWorkerHttpGateway(
|
||||||
|
base_url=configuration.base_url,
|
||||||
|
bearer_token=bearer_token,
|
||||||
|
work_root=configuration.work_root,
|
||||||
|
transport=http_transport,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
# The immutable string remains owned by the gateway headers for the
|
||||||
|
# service lifetime; this local binding must not outlive composition.
|
||||||
|
del bearer_token
|
||||||
|
return InstalledObservatoryWorkerService(
|
||||||
|
configuration=configuration,
|
||||||
|
gateway=gateway,
|
||||||
|
agent=ObservatoryWorkerAgent(transport=gateway, executors=executors),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def require_ready_executor_coverage(
|
||||||
|
*,
|
||||||
|
definitions: PortableRunDefinitionRegistry,
|
||||||
|
executors: ObservatoryWorkerExecutorRegistry,
|
||||||
|
) -> tuple[ObservatoryWorkerExecutorIdentity, ...]:
|
||||||
|
"""Prove every server-advertisable definition has one local identity."""
|
||||||
|
|
||||||
|
ready = definitions.ready_recorded_definitions()
|
||||||
|
if not ready:
|
||||||
|
raise ObservatoryWorkerServiceError(
|
||||||
|
"no portable RunDefinition has a sealed ready executor"
|
||||||
|
)
|
||||||
|
identities: list[ObservatoryWorkerExecutorIdentity] = []
|
||||||
|
for definition in ready:
|
||||||
|
identity = ObservatoryWorkerExecutorIdentity(
|
||||||
|
release_sha256=definition.executor_release_sha256,
|
||||||
|
image_sha256=definition.executor_image_sha256,
|
||||||
|
model_manifest_sha256=definition.model_manifest_sha256,
|
||||||
|
resource_profile_sha256=definition.resource_profile_sha256,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
executors.resolve(identity)
|
||||||
|
except ObservatoryWorkerExecutorUnavailableError as exc:
|
||||||
|
raise ObservatoryWorkerServiceError(
|
||||||
|
"a ready portable RunDefinition has no exact local executor identity"
|
||||||
|
) from exc
|
||||||
|
identities.append(identity)
|
||||||
|
return tuple(identities)
|
||||||
|
|
||||||
|
|
||||||
|
def load_observatory_worker_bearer_token(path: Path) -> str:
|
||||||
|
"""Read one private regular ASCII token without accepting links/newlines."""
|
||||||
|
|
||||||
|
candidate = path.expanduser().absolute()
|
||||||
|
descriptor: int | None = None
|
||||||
|
try:
|
||||||
|
descriptor = os.open(
|
||||||
|
candidate,
|
||||||
|
os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0),
|
||||||
|
)
|
||||||
|
metadata = os.fstat(descriptor)
|
||||||
|
if not stat.S_ISREG(metadata.st_mode):
|
||||||
|
raise ObservatoryWorkerServiceError(
|
||||||
|
"Worker bearer credential must be a regular file"
|
||||||
|
)
|
||||||
|
if os.name != "posix":
|
||||||
|
raise ObservatoryWorkerServiceError(
|
||||||
|
"native Worker credential ACL verification is not available; "
|
||||||
|
"use the admitted POSIX Worker service runtime"
|
||||||
|
)
|
||||||
|
if metadata.st_mode & 0o077:
|
||||||
|
raise ObservatoryWorkerServiceError(
|
||||||
|
"Worker bearer credential permissions are too broad"
|
||||||
|
)
|
||||||
|
if not 32 <= metadata.st_size <= 512:
|
||||||
|
raise ObservatoryWorkerServiceError(
|
||||||
|
"Worker bearer credential format is invalid"
|
||||||
|
)
|
||||||
|
with os.fdopen(descriptor, "rb") as stream:
|
||||||
|
descriptor = None
|
||||||
|
payload = stream.read(513)
|
||||||
|
except ObservatoryWorkerServiceError:
|
||||||
|
raise
|
||||||
|
except OSError as exc:
|
||||||
|
raise ObservatoryWorkerServiceError(
|
||||||
|
"Worker bearer credential is unavailable"
|
||||||
|
) from exc
|
||||||
|
finally:
|
||||||
|
if descriptor is not None:
|
||||||
|
os.close(descriptor)
|
||||||
|
try:
|
||||||
|
token = payload.decode("ascii")
|
||||||
|
except UnicodeDecodeError as exc:
|
||||||
|
raise ObservatoryWorkerServiceError(
|
||||||
|
"Worker bearer credential is not ASCII"
|
||||||
|
) from exc
|
||||||
|
if _TOKEN.fullmatch(token) is None:
|
||||||
|
raise ObservatoryWorkerServiceError(
|
||||||
|
"Worker bearer credential format is invalid"
|
||||||
|
)
|
||||||
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_worker_base_url(value: str) -> None:
|
||||||
|
parsed = urlsplit(value)
|
||||||
|
if (
|
||||||
|
value != value.strip()
|
||||||
|
or parsed.query
|
||||||
|
or parsed.fragment
|
||||||
|
or parsed.username is not None
|
||||||
|
or parsed.password is not None
|
||||||
|
or parsed.path not in {"", "/"}
|
||||||
|
or parsed.scheme not in {"http", "https"}
|
||||||
|
or not parsed.hostname
|
||||||
|
):
|
||||||
|
raise ValueError("Worker Mission Core base URL is invalid")
|
||||||
|
if parsed.scheme == "http" and parsed.hostname not in {
|
||||||
|
"127.0.0.1",
|
||||||
|
"localhost",
|
||||||
|
"::1",
|
||||||
|
}:
|
||||||
|
raise ValueError("unencrypted Worker transport requires a loopback tunnel")
|
||||||
|
|
||||||
|
|
||||||
|
def _absolute_path(path: Path, label: str) -> None:
|
||||||
|
if not path.is_absolute() or str(path) != str(path).strip():
|
||||||
|
raise ValueError(f"{label} must be an absolute path")
|
||||||
|
|
||||||
|
|
||||||
|
def _required_path(environment: Mapping[str, str], name: str) -> Path:
|
||||||
|
value = environment.get(name, "")
|
||||||
|
if not value or value != value.strip():
|
||||||
|
raise ObservatoryWorkerServiceError(f"{name} is required")
|
||||||
|
path = Path(value)
|
||||||
|
_absolute_path(path, name)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _environment_float(
|
||||||
|
environment: Mapping[str, str],
|
||||||
|
name: str,
|
||||||
|
default: float,
|
||||||
|
) -> float:
|
||||||
|
value = environment.get(name, "")
|
||||||
|
if not value:
|
||||||
|
return default
|
||||||
|
try:
|
||||||
|
return float(value)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise ObservatoryWorkerServiceError(f"{name} is invalid") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _environment_int(
|
||||||
|
environment: Mapping[str, str],
|
||||||
|
name: str,
|
||||||
|
default: int,
|
||||||
|
) -> int:
|
||||||
|
value = environment.get(name, "")
|
||||||
|
if not value:
|
||||||
|
return default
|
||||||
|
if not value.isascii() or not value.isdecimal():
|
||||||
|
raise ObservatoryWorkerServiceError(f"{name} is invalid")
|
||||||
|
return int(value)
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
"""Pure launchd plan for the Mac-owned Worker 006 reverse SSH tunnel."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import plistlib
|
||||||
|
import stat
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Final
|
||||||
|
|
||||||
|
OBSERVATORY_WORKER_TUNNEL_LABEL: Final = (
|
||||||
|
"com.nodedc.observatory-worker-tunnel.local"
|
||||||
|
)
|
||||||
|
OBSERVATORY_WORKER_TUNNEL_SCHEMA: Final = (
|
||||||
|
"missioncore.observatory-worker-tunnel-plan/v1"
|
||||||
|
)
|
||||||
|
OBSERVATORY_WORKER_TUNNEL_PORT: Final = 18080
|
||||||
|
|
||||||
|
|
||||||
|
class ObservatoryWorkerTunnelPlanError(RuntimeError):
|
||||||
|
"""The reverse tunnel cannot be declared without weakening its boundary."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ObservatoryWorkerTunnelLaunchAgentPlan:
|
||||||
|
agent_path: Path
|
||||||
|
data_directory: Path
|
||||||
|
desired_sha256: str
|
||||||
|
desired_payload: bytes
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"schema_version": OBSERVATORY_WORKER_TUNNEL_SCHEMA,
|
||||||
|
"label": OBSERVATORY_WORKER_TUNNEL_LABEL,
|
||||||
|
"agent_path": str(self.agent_path),
|
||||||
|
"data_directory": str(self.data_directory),
|
||||||
|
"desired_sha256": self.desired_sha256,
|
||||||
|
"transport": {
|
||||||
|
"owner": "mac-launchd",
|
||||||
|
"ssh_alias": "mission-gpu",
|
||||||
|
"worker_listener": "127.0.0.1:18080",
|
||||||
|
"mission_core_target": "127.0.0.1:8000",
|
||||||
|
"encrypted": True,
|
||||||
|
"worker_listener_loopback_only": True,
|
||||||
|
"bearer_credential_in_arguments": False,
|
||||||
|
},
|
||||||
|
"changes": {
|
||||||
|
"durable_mutation_performed": False,
|
||||||
|
"requires_hash-gated_install": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def plan_observatory_worker_tunnel_launch_agent(
|
||||||
|
*,
|
||||||
|
data_directory: Path,
|
||||||
|
agent_path: Path,
|
||||||
|
ssh_path: Path = Path("/usr/bin/ssh"),
|
||||||
|
) -> ObservatoryWorkerTunnelLaunchAgentPlan:
|
||||||
|
"""Build, but never install, the exact reverse-loopback declaration."""
|
||||||
|
|
||||||
|
data_root = _private_directory(data_directory)
|
||||||
|
ssh = _exact_executable(ssh_path)
|
||||||
|
target_path = agent_path.expanduser().absolute()
|
||||||
|
log_path = data_root / "observatory-worker-tunnel.log"
|
||||||
|
arguments = [
|
||||||
|
str(ssh),
|
||||||
|
"-o",
|
||||||
|
"BatchMode=yes",
|
||||||
|
"-o",
|
||||||
|
"ExitOnForwardFailure=yes",
|
||||||
|
"-o",
|
||||||
|
"ServerAliveInterval=15",
|
||||||
|
"-o",
|
||||||
|
"ServerAliveCountMax=3",
|
||||||
|
"-o",
|
||||||
|
"RequestTTY=no",
|
||||||
|
"-N",
|
||||||
|
"-T",
|
||||||
|
"-R",
|
||||||
|
"127.0.0.1:18080:127.0.0.1:8000",
|
||||||
|
"mission-gpu",
|
||||||
|
]
|
||||||
|
desired = {
|
||||||
|
"Label": OBSERVATORY_WORKER_TUNNEL_LABEL,
|
||||||
|
"ProgramArguments": arguments,
|
||||||
|
"KeepAlive": True,
|
||||||
|
"RunAtLoad": True,
|
||||||
|
"AbandonProcessGroup": False,
|
||||||
|
"ProcessType": "Background",
|
||||||
|
"ThrottleInterval": 5,
|
||||||
|
"ExitTimeOut": 10,
|
||||||
|
"StandardOutPath": str(log_path),
|
||||||
|
"StandardErrorPath": str(log_path),
|
||||||
|
}
|
||||||
|
payload = plistlib.dumps(desired, fmt=plistlib.FMT_XML, sort_keys=True)
|
||||||
|
return ObservatoryWorkerTunnelLaunchAgentPlan(
|
||||||
|
agent_path=target_path,
|
||||||
|
data_directory=data_root,
|
||||||
|
desired_sha256=hashlib.sha256(payload).hexdigest(),
|
||||||
|
desired_payload=payload,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _private_directory(path: Path) -> Path:
|
||||||
|
candidate = path.expanduser().absolute()
|
||||||
|
try:
|
||||||
|
metadata = candidate.lstat()
|
||||||
|
resolved = candidate.resolve(strict=True)
|
||||||
|
except OSError as exc:
|
||||||
|
raise ObservatoryWorkerTunnelPlanError(
|
||||||
|
"Mission Core data directory is unavailable"
|
||||||
|
) from exc
|
||||||
|
if (
|
||||||
|
resolved != candidate
|
||||||
|
or not stat.S_ISDIR(metadata.st_mode)
|
||||||
|
or stat.S_IMODE(metadata.st_mode) != 0o700
|
||||||
|
or metadata.st_uid != os.getuid()
|
||||||
|
):
|
||||||
|
raise ObservatoryWorkerTunnelPlanError(
|
||||||
|
"Mission Core data directory is not private and canonical"
|
||||||
|
)
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def _exact_executable(path: Path) -> Path:
|
||||||
|
candidate = path.expanduser().absolute()
|
||||||
|
try:
|
||||||
|
metadata = candidate.lstat()
|
||||||
|
resolved = candidate.resolve(strict=True)
|
||||||
|
except OSError as exc:
|
||||||
|
raise ObservatoryWorkerTunnelPlanError("SSH executable is unavailable") from exc
|
||||||
|
if (
|
||||||
|
resolved != candidate
|
||||||
|
or stat.S_ISLNK(metadata.st_mode)
|
||||||
|
or not stat.S_ISREG(metadata.st_mode)
|
||||||
|
or not os.access(candidate, os.X_OK)
|
||||||
|
):
|
||||||
|
raise ObservatoryWorkerTunnelPlanError("SSH executable is not exact")
|
||||||
|
return resolved
|
||||||
+204
-31
@@ -47,20 +47,38 @@ from k1link.observatory.m49_queue_binding import (
|
|||||||
M49QueueBindingError,
|
M49QueueBindingError,
|
||||||
M49RecordedQueueBindingService,
|
M49RecordedQueueBindingService,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.portable_queue_binding import (
|
||||||
|
PortableQueueBindingError,
|
||||||
|
PortableRecordedQueueBindingService,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_result_contract import (
|
||||||
|
PortableCalculationProfileRegistry,
|
||||||
|
PortableResultContractValidatorRegistry,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_result_publisher import (
|
||||||
|
resolve_published_portable_calculation_profile,
|
||||||
|
)
|
||||||
from k1link.observatory.portable_run_definitions import (
|
from k1link.observatory.portable_run_definitions import (
|
||||||
PortableRunDefinitionRegistry,
|
PortableRunDefinitionRegistry,
|
||||||
PortableRunDefinitionRegistryError,
|
PortableRunDefinitionRegistryError,
|
||||||
)
|
)
|
||||||
from k1link.observatory.portable_setup_projection import (
|
from k1link.observatory.portable_setup_projection import (
|
||||||
PORTABLE_LAB_V1_SETUP_ID,
|
|
||||||
PortableLabV1SetupProjector,
|
|
||||||
PortableSetupProjectionError,
|
PortableSetupProjectionError,
|
||||||
|
PortableSetupProjector,
|
||||||
|
portable_calculation_profile_registry,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_worker_integration import (
|
||||||
|
PortableObservatoryWorkerIntegration,
|
||||||
|
PortableWorkerIntegrationError,
|
||||||
|
PortableWorkerStorageRoots,
|
||||||
|
build_portable_observatory_worker_integration,
|
||||||
|
portable_result_validator_registry,
|
||||||
)
|
)
|
||||||
from k1link.observatory.recorded_jobs import (
|
from k1link.observatory.recorded_jobs import (
|
||||||
ObservatoryRecordedJobQueue,
|
ObservatoryRecordedJobQueue,
|
||||||
ObservatoryRecordedQueueError,
|
ObservatoryRecordedQueueError,
|
||||||
|
RecordedRunDefinitionRegistry,
|
||||||
)
|
)
|
||||||
from k1link.observatory.source_admission import RecordedK1SourceAdmissionService
|
|
||||||
from k1link.sessions import (
|
from k1link.sessions import (
|
||||||
MaterializedRecording,
|
MaterializedRecording,
|
||||||
RecordedCameraFrameService,
|
RecordedCameraFrameService,
|
||||||
@@ -73,6 +91,7 @@ from k1link.sessions import (
|
|||||||
SessionRecordingPreparationManager,
|
SessionRecordingPreparationManager,
|
||||||
SessionStore,
|
SessionStore,
|
||||||
)
|
)
|
||||||
|
from k1link.sessions.models import SessionSummary
|
||||||
from k1link.simulation.projects import SimulationProjectService, SimulationProjectStore
|
from k1link.simulation.projects import SimulationProjectService, SimulationProjectStore
|
||||||
from k1link.web.advanced_laboratory_api import build_advanced_laboratory_router
|
from k1link.web.advanced_laboratory_api import build_advanced_laboratory_router
|
||||||
from k1link.web.artifact_health_api import build_artifact_health_router
|
from k1link.web.artifact_health_api import build_artifact_health_router
|
||||||
@@ -261,6 +280,55 @@ plugin_catalog: DevicePluginCatalog = plugin_environment.catalog
|
|||||||
plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher
|
plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher
|
||||||
session_store = SessionStore(REPOSITORY_ROOT)
|
session_store = SessionStore(REPOSITORY_ROOT)
|
||||||
|
|
||||||
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY: PortableRunDefinitionRegistry | None
|
||||||
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR: str | None
|
||||||
|
OBSERVATORY_PORTABLE_CALCULATION_PROFILES: PortableCalculationProfileRegistry | None
|
||||||
|
OBSERVATORY_PORTABLE_RESULT_VALIDATORS: PortableResultContractValidatorRegistry | None
|
||||||
|
try:
|
||||||
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY = PortableRunDefinitionRegistry.from_file(
|
||||||
|
REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||||
|
)
|
||||||
|
OBSERVATORY_PORTABLE_CALCULATION_PROFILES = portable_calculation_profile_registry(
|
||||||
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY
|
||||||
|
)
|
||||||
|
OBSERVATORY_PORTABLE_RESULT_VALIDATORS = portable_result_validator_registry(
|
||||||
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY
|
||||||
|
)
|
||||||
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR = None
|
||||||
|
except (
|
||||||
|
PortableRunDefinitionRegistryError,
|
||||||
|
PortableWorkerIntegrationError,
|
||||||
|
OSError,
|
||||||
|
ValueError,
|
||||||
|
) as exc:
|
||||||
|
# Portable definitions are an optional observation-only slice. Registry
|
||||||
|
# drift cannot affect K1, Simulation, legacy LAB, or the exact M49 binding.
|
||||||
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY = None
|
||||||
|
OBSERVATORY_PORTABLE_CALCULATION_PROFILES = None
|
||||||
|
OBSERVATORY_PORTABLE_RESULT_VALIDATORS = None
|
||||||
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR = str(exc)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_observatory_calculation_profile(
|
||||||
|
summary: SessionSummary,
|
||||||
|
) -> dict[str, object] | None:
|
||||||
|
if OBSERVATORY_LABORATORY_SETUP_REGISTRY is not None:
|
||||||
|
legacy = OBSERVATORY_LABORATORY_SETUP_REGISTRY.observatory_calculation_profile(
|
||||||
|
summary
|
||||||
|
)
|
||||||
|
if legacy is not None:
|
||||||
|
return legacy
|
||||||
|
if (
|
||||||
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None
|
||||||
|
or OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
return resolve_published_portable_calculation_profile(
|
||||||
|
summary,
|
||||||
|
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||||
|
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _load_optional_observatory_worker_authentication(
|
def _load_optional_observatory_worker_authentication(
|
||||||
recorded_job_queue: ObservatoryRecordedJobQueue | None,
|
recorded_job_queue: ObservatoryRecordedJobQueue | None,
|
||||||
@@ -299,9 +367,14 @@ try:
|
|||||||
REPOSITORY_ROOT / "config" / "observatory-m49-recorded-queue-binding.json"
|
REPOSITORY_ROOT / "config" / "observatory-m49-recorded-queue-binding.json"
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
recorded_definitions = list(OBSERVATORY_RECORDED_BINDING_SERVICE.definitions.definitions)
|
||||||
|
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is not None:
|
||||||
|
recorded_definitions.extend(
|
||||||
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY.ready_recorded_definitions()
|
||||||
|
)
|
||||||
OBSERVATORY_RECORDED_JOB_QUEUE = ObservatoryRecordedJobQueue(
|
OBSERVATORY_RECORDED_JOB_QUEUE = ObservatoryRecordedJobQueue(
|
||||||
session_store.data_dir,
|
session_store.data_dir,
|
||||||
definitions=OBSERVATORY_RECORDED_BINDING_SERVICE.definitions,
|
definitions=RecordedRunDefinitionRegistry(tuple(recorded_definitions)),
|
||||||
)
|
)
|
||||||
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = None
|
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = None
|
||||||
except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError) as exc:
|
except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError) as exc:
|
||||||
@@ -316,11 +389,14 @@ OBSERVATORY_WORKER_CLAIM_LEASE_READY = False
|
|||||||
OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = False
|
OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = False
|
||||||
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED = False
|
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED = False
|
||||||
OBSERVATORY_WORKER_AUTHENTICATION: ObservatoryWorkerAuthentication | None
|
OBSERVATORY_WORKER_AUTHENTICATION: ObservatoryWorkerAuthentication | None
|
||||||
|
OBSERVATORY_WORKER_AUTHENTICATION_ERROR: str | None
|
||||||
OBSERVATORY_WORKER_API_ERROR: str | None
|
OBSERVATORY_WORKER_API_ERROR: str | None
|
||||||
OBSERVATORY_WORKER_AUTHENTICATION = None
|
(
|
||||||
OBSERVATORY_WORKER_API_ERROR = (
|
OBSERVATORY_WORKER_AUTHENTICATION,
|
||||||
"Worker pull API is hard-disabled until claim leases and a verified "
|
OBSERVATORY_WORKER_AUTHENTICATION_ERROR,
|
||||||
"Observatory result publisher are implemented and accepted"
|
) = _load_optional_observatory_worker_authentication(
|
||||||
|
OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||||
|
token_path=OBSERVATORY_WORKER_TOKEN_PATH,
|
||||||
)
|
)
|
||||||
simulation_project_store = SimulationProjectStore(session_store.data_dir)
|
simulation_project_store = SimulationProjectStore(session_store.data_dir)
|
||||||
simulation_project_service = SimulationProjectService(simulation_project_store)
|
simulation_project_service = SimulationProjectService(simulation_project_store)
|
||||||
@@ -336,37 +412,122 @@ session_recording_materializer = SessionRecordingMaterializer(
|
|||||||
session_recorded_media_inspector = RecordedMediaInspector(
|
session_recorded_media_inspector = RecordedMediaInspector(
|
||||||
session_store.data_dir / "recorded-media-preparations"
|
session_store.data_dir / "recorded-media-preparations"
|
||||||
)
|
)
|
||||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableLabV1SetupProjector | None
|
OBSERVATORY_PORTABLE_WORKER_INTEGRATION: PortableObservatoryWorkerIntegration | None
|
||||||
|
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR: str | None
|
||||||
|
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS: PortableWorkerStorageRoots | None = None
|
||||||
|
try:
|
||||||
|
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None:
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR
|
||||||
|
or "portable definition registry is unavailable"
|
||||||
|
)
|
||||||
|
if OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None:
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
"portable calculation profile registry is unavailable"
|
||||||
|
)
|
||||||
|
if OBSERVATORY_PORTABLE_RESULT_VALIDATORS is None:
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
"portable result validator registry is unavailable"
|
||||||
|
)
|
||||||
|
if OBSERVATORY_RECORDED_JOB_QUEUE is None:
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR
|
||||||
|
or "Observatory recorded-job queue is unavailable"
|
||||||
|
)
|
||||||
|
if session_artifact_gateway is None:
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
"central artifact store is not configured"
|
||||||
|
)
|
||||||
|
if session_artifact_gateway.status().central_status != "ready":
|
||||||
|
raise PortableWorkerIntegrationError(
|
||||||
|
"central artifact store is unavailable"
|
||||||
|
)
|
||||||
|
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS = (
|
||||||
|
PortableWorkerStorageRoots.from_environment(
|
||||||
|
artifact_store_root=session_artifact_gateway.store.root,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
OBSERVATORY_PORTABLE_WORKER_INTEGRATION = (
|
||||||
|
build_portable_observatory_worker_integration(
|
||||||
|
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||||
|
session_store=session_store,
|
||||||
|
media_inspector=session_recorded_media_inspector,
|
||||||
|
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||||
|
artifact_store=session_artifact_gateway.store,
|
||||||
|
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
|
||||||
|
validators=OBSERVATORY_PORTABLE_RESULT_VALIDATORS,
|
||||||
|
source_cas_root=(
|
||||||
|
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.source_cas_root
|
||||||
|
),
|
||||||
|
result_staging_root=(
|
||||||
|
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.result_staging_root
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = None
|
||||||
|
except (PortableWorkerIntegrationError, OSError, ValueError) as exc:
|
||||||
|
# Constructing this dormant foundation does not enable the Worker router.
|
||||||
|
# Failure remains isolated from K1, Simulation and legacy LAB.
|
||||||
|
OBSERVATORY_PORTABLE_WORKER_INTEGRATION = None
|
||||||
|
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = str(exc)
|
||||||
|
OBSERVATORY_WORKER_API_ERROR = (
|
||||||
|
"Worker pull API is hard-disabled pending sealed installed executors, "
|
||||||
|
"a configured Worker credential, explicit integration acceptance and the "
|
||||||
|
"production gate"
|
||||||
|
+ (
|
||||||
|
""
|
||||||
|
if OBSERVATORY_WORKER_AUTHENTICATION_ERROR is None
|
||||||
|
else (
|
||||||
|
"; authentication unavailable: "
|
||||||
|
f"{OBSERVATORY_WORKER_AUTHENTICATION_ERROR}"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
+ (
|
||||||
|
""
|
||||||
|
if OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR is None
|
||||||
|
else f"; integration unavailable: {OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR}"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
OBSERVATORY_WORKER_DISPATCH_READY = (
|
||||||
|
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED
|
||||||
|
and OBSERVATORY_WORKER_CLAIM_LEASE_READY
|
||||||
|
and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY
|
||||||
|
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||||
|
and OBSERVATORY_WORKER_AUTHENTICATION is not None
|
||||||
|
and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
|
||||||
|
)
|
||||||
|
OBSERVATORY_PORTABLE_BINDING_SERVICE: PortableRecordedQueueBindingService | None
|
||||||
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableSetupProjector | None
|
||||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
|
||||||
try:
|
try:
|
||||||
portable_definition_registry = PortableRunDefinitionRegistry.from_file(
|
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None:
|
||||||
REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
raise PortableSetupProjectionError(
|
||||||
)
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR
|
||||||
portable_lab_v1_definition = next(
|
or "portable definition registry is unavailable"
|
||||||
definition
|
)
|
||||||
for definition in portable_definition_registry.definitions
|
OBSERVATORY_PORTABLE_BINDING_SERVICE = PortableRecordedQueueBindingService(
|
||||||
if definition.setup_id == PORTABLE_LAB_V1_SETUP_ID
|
|
||||||
)
|
|
||||||
portable_source_capability_service = RecordedK1SourceAdmissionService(
|
|
||||||
data_dir=session_store.data_dir,
|
data_dir=session_store.data_dir,
|
||||||
session_store=session_store,
|
session_store=session_store,
|
||||||
media_inspector=session_recorded_media_inspector,
|
media_inspector=session_recorded_media_inspector,
|
||||||
requirements=portable_lab_v1_definition.to_source_admission_requirements(),
|
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||||
|
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||||
)
|
)
|
||||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableLabV1SetupProjector(
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableSetupProjector(
|
||||||
registry=portable_definition_registry,
|
registry=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||||
capability_probe=portable_source_capability_service,
|
capability_probe=OBSERVATORY_PORTABLE_BINDING_SERVICE,
|
||||||
|
dispatch_available=OBSERVATORY_WORKER_DISPATCH_READY,
|
||||||
)
|
)
|
||||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None
|
||||||
except (
|
except (
|
||||||
|
PortableQueueBindingError,
|
||||||
PortableRunDefinitionRegistryError,
|
PortableRunDefinitionRegistryError,
|
||||||
PortableSetupProjectionError,
|
PortableSetupProjectionError,
|
||||||
OSError,
|
OSError,
|
||||||
StopIteration,
|
|
||||||
ValueError,
|
ValueError,
|
||||||
) as exc:
|
) as exc:
|
||||||
# Portable LAB V1 is an optional observation-only slice. A drifted
|
# Portable setup execution is an optional observation-only slice. A drifted
|
||||||
# registry cannot affect K1, Simulation, legacy LAB, or the exact M49 queue.
|
# registry cannot affect K1, Simulation, legacy LAB, or the exact M49 queue.
|
||||||
|
OBSERVATORY_PORTABLE_BINDING_SERVICE = None
|
||||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = None
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = None
|
||||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = str(exc)
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = str(exc)
|
||||||
_ffmpeg = _resolve_media_tool("ffmpeg")
|
_ffmpeg = _resolve_media_tool("ffmpeg")
|
||||||
@@ -829,6 +990,14 @@ app.include_router(
|
|||||||
perception_overlay_provider=session_perception_overlay_store,
|
perception_overlay_provider=session_perception_overlay_store,
|
||||||
perception_media_provider=session_perception_epoch_store,
|
perception_media_provider=session_perception_epoch_store,
|
||||||
point_color_renderers=plugin_environment.point_color_renderers,
|
point_color_renderers=plugin_environment.point_color_renderers,
|
||||||
|
lab_calculation_profile_resolver=(
|
||||||
|
None
|
||||||
|
if (
|
||||||
|
OBSERVATORY_LABORATORY_SETUP_REGISTRY is None
|
||||||
|
and OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None
|
||||||
|
)
|
||||||
|
else _resolve_observatory_calculation_profile
|
||||||
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
app.include_router(
|
app.include_router(
|
||||||
@@ -843,19 +1012,23 @@ app.include_router(
|
|||||||
recorded_job_queue_error=OBSERVATORY_RECORDED_JOB_QUEUE_ERROR,
|
recorded_job_queue_error=OBSERVATORY_RECORDED_JOB_QUEUE_ERROR,
|
||||||
portable_setup_projector=OBSERVATORY_PORTABLE_SETUP_PROJECTOR,
|
portable_setup_projector=OBSERVATORY_PORTABLE_SETUP_PROJECTOR,
|
||||||
portable_setup_projector_error=OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR,
|
portable_setup_projector_error=OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR,
|
||||||
|
portable_binding_service=OBSERVATORY_PORTABLE_BINDING_SERVICE,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if (
|
if OBSERVATORY_WORKER_DISPATCH_READY:
|
||||||
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED
|
assert OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||||
and OBSERVATORY_WORKER_CLAIM_LEASE_READY
|
assert OBSERVATORY_WORKER_AUTHENTICATION is not None
|
||||||
and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY
|
assert OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
|
||||||
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
|
||||||
and OBSERVATORY_WORKER_AUTHENTICATION is not None
|
|
||||||
):
|
|
||||||
app.include_router(
|
app.include_router(
|
||||||
build_observatory_worker_router(
|
build_observatory_worker_router(
|
||||||
OBSERVATORY_RECORDED_JOB_QUEUE,
|
OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||||
authentication=OBSERVATORY_WORKER_AUTHENTICATION,
|
authentication=OBSERVATORY_WORKER_AUTHENTICATION,
|
||||||
|
artifact_transport=(
|
||||||
|
OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport
|
||||||
|
),
|
||||||
|
result_publisher=(
|
||||||
|
OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher
|
||||||
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
app.include_router(
|
app.include_router(
|
||||||
|
|||||||
@@ -22,9 +22,19 @@ from k1link.observatory.m49_queue_binding import (
|
|||||||
M49QueueBindingIntegrityError,
|
M49QueueBindingIntegrityError,
|
||||||
M49RecordedQueueBindingService,
|
M49RecordedQueueBindingService,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.portable_queue_binding import (
|
||||||
|
PortableQueueBindingError,
|
||||||
|
PortableQueueBindingIntegrityError,
|
||||||
|
PortableQueueBindingStaleCheckError,
|
||||||
|
PortableRecordedQueueBindingService,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_run_definitions import (
|
||||||
|
PortableRunDefinitionUnavailableError,
|
||||||
|
)
|
||||||
from k1link.observatory.portable_setup_projection import (
|
from k1link.observatory.portable_setup_projection import (
|
||||||
PortableLabV1SetupProjector,
|
PortableLabV1SetupProjector,
|
||||||
PortableSetupProjectionError,
|
PortableSetupProjectionError,
|
||||||
|
PortableSetupProjector,
|
||||||
)
|
)
|
||||||
from k1link.observatory.recorded_jobs import (
|
from k1link.observatory.recorded_jobs import (
|
||||||
ObservatoryRecordedJobQueue,
|
ObservatoryRecordedJobQueue,
|
||||||
@@ -33,6 +43,7 @@ from k1link.observatory.recorded_jobs import (
|
|||||||
ObservatoryRecordedQueueError,
|
ObservatoryRecordedQueueError,
|
||||||
ObservatoryRecordedQueueNotFoundError,
|
ObservatoryRecordedQueueNotFoundError,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.source_admission import PortableSourceAdmissionError
|
||||||
from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore
|
from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore
|
||||||
from k1link.sessions.models import SessionSummary
|
from k1link.sessions.models import SessionSummary
|
||||||
|
|
||||||
@@ -130,6 +141,8 @@ class ObservatoryRecordedRunSubmitRequest(_StrictApiModel):
|
|||||||
max_length=96,
|
max_length=96,
|
||||||
pattern=r"^[a-z][a-z0-9-]{2,95}$",
|
pattern=r"^[a-z][a-z0-9-]{2,95}$",
|
||||||
)
|
)
|
||||||
|
definition_sha256: str | None = Field(default=None, pattern=r"^[a-f0-9]{64}$")
|
||||||
|
check_sha256: str | None = Field(default=None, pattern=r"^[a-f0-9]{64}$")
|
||||||
|
|
||||||
|
|
||||||
def build_observatory_router(
|
def build_observatory_router(
|
||||||
@@ -142,8 +155,9 @@ def build_observatory_router(
|
|||||||
recorded_binding_service: M49RecordedQueueBindingService | None = None,
|
recorded_binding_service: M49RecordedQueueBindingService | None = None,
|
||||||
recorded_job_queue: ObservatoryRecordedJobQueue | None = None,
|
recorded_job_queue: ObservatoryRecordedJobQueue | None = None,
|
||||||
recorded_job_queue_error: str | None = None,
|
recorded_job_queue_error: str | None = None,
|
||||||
portable_setup_projector: PortableLabV1SetupProjector | None = None,
|
portable_setup_projector: PortableSetupProjector | PortableLabV1SetupProjector | None = None,
|
||||||
portable_setup_projector_error: str | None = None,
|
portable_setup_projector_error: str | None = None,
|
||||||
|
portable_binding_service: PortableRecordedQueueBindingService | None = None,
|
||||||
) -> APIRouter:
|
) -> APIRouter:
|
||||||
"""Build bounded catalog-only mutations for typed Observatory projections."""
|
"""Build bounded catalog-only mutations for typed Observatory projections."""
|
||||||
|
|
||||||
@@ -214,6 +228,126 @@ def build_observatory_router(
|
|||||||
available.add(result_id)
|
available.add(result_id)
|
||||||
return frozenset(available)
|
return frozenset(available)
|
||||||
|
|
||||||
|
def portable_run_preflight(
|
||||||
|
source: SessionSummary,
|
||||||
|
request: ObservatoryRunPreflightRequest,
|
||||||
|
) -> dict[str, object] | None:
|
||||||
|
projector = portable_setup_projector
|
||||||
|
if projector is None or not projector.has_setup(request.setup_id):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
projected = projector.project(source, setup_id=request.setup_id)
|
||||||
|
except PortableSetupProjectionError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Portable-каталог сетапов нарушил контракт целостности.",
|
||||||
|
) from exc
|
||||||
|
definition = projected.get("run_definition")
|
||||||
|
compatibility = projected.get("source_compatibility")
|
||||||
|
executor = projected.get("executor")
|
||||||
|
if (
|
||||||
|
not isinstance(definition, dict)
|
||||||
|
or not isinstance(compatibility, dict)
|
||||||
|
or not isinstance(executor, dict)
|
||||||
|
):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Portable-каталог сетапов нарушил контракт целостности.",
|
||||||
|
)
|
||||||
|
expected_digest = definition.get("definition_sha256")
|
||||||
|
if request.definition_sha256 != expected_digest:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Идентичность RunDefinition изменилась; обновите каталог.",
|
||||||
|
)
|
||||||
|
compatible = compatibility.get("compatible") is True
|
||||||
|
executor_ready = executor.get("state") == "ready" and executor.get("ready") is True
|
||||||
|
checked = None
|
||||||
|
check_reason: str | None = None
|
||||||
|
if (
|
||||||
|
compatible
|
||||||
|
and executor_ready
|
||||||
|
and portable_binding_service is not None
|
||||||
|
and recorded_job_queue is not None
|
||||||
|
and isinstance(expected_digest, str)
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
checked = portable_binding_service.check(
|
||||||
|
source_session_id=request.source_session_id,
|
||||||
|
setup_id=request.setup_id,
|
||||||
|
definition_sha256=expected_digest,
|
||||||
|
)
|
||||||
|
except PortableRunDefinitionUnavailableError as exc:
|
||||||
|
check_reason = str(exc)
|
||||||
|
except (PortableQueueBindingError, PortableSourceAdmissionError, ValueError):
|
||||||
|
check_reason = (
|
||||||
|
"Источник или исполняемый portable-релиз не прошёл проверку целостности."
|
||||||
|
)
|
||||||
|
elif compatible and executor_ready:
|
||||||
|
check_reason = "Portable dispatch-контур или durable-очередь недоступны."
|
||||||
|
check_sha256 = None if checked is None else checked.check_sha256
|
||||||
|
queueable = check_sha256 is not None
|
||||||
|
checks: list[dict[str, Any]] = [
|
||||||
|
{
|
||||||
|
"check_id": "source-compatibility",
|
||||||
|
"outcome": "pass" if compatible else "fail",
|
||||||
|
"reason_code": "source-compatible" if compatible else "source-incompatible",
|
||||||
|
"message": str(compatibility.get("reason")),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"check_id": "executor",
|
||||||
|
"outcome": "pass" if executor_ready else "fail",
|
||||||
|
"reason_code": (
|
||||||
|
"executor-release-sealed"
|
||||||
|
if executor_ready
|
||||||
|
else str(executor.get("reason_code"))
|
||||||
|
),
|
||||||
|
"message": (
|
||||||
|
"Исполняемый portable-релиз и image запечатаны."
|
||||||
|
if executor_ready
|
||||||
|
else str(executor.get("reason"))
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"check_id": "definition-check",
|
||||||
|
"outcome": "pass" if checked is not None else "fail",
|
||||||
|
"reason_code": (
|
||||||
|
"portable-check-sealed" if checked is not None else "portable-check-unavailable"
|
||||||
|
),
|
||||||
|
"message": (
|
||||||
|
"Источник и RunDefinition связаны одноразовым check SHA."
|
||||||
|
if checked is not None
|
||||||
|
else check_reason
|
||||||
|
or "Portable-проверка недоступна до установки executor-релиза."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"check_id": "durable-queue",
|
||||||
|
"outcome": "pass" if queueable else "fail",
|
||||||
|
"reason_code": (
|
||||||
|
"durable-queue-ready" if queueable else "durable-queue-unavailable"
|
||||||
|
),
|
||||||
|
"message": (
|
||||||
|
"Durable-очередь готова принять расчёт по check SHA."
|
||||||
|
if queueable
|
||||||
|
else "Расчёт нельзя поставить в очередь."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
return {
|
||||||
|
"schema_version": OBSERVATORY_RUN_PREFLIGHT_SCHEMA,
|
||||||
|
"source_session_id": request.source_session_id,
|
||||||
|
"setup_id": request.setup_id,
|
||||||
|
"definition_sha256": expected_digest,
|
||||||
|
"check_sha256": check_sha256,
|
||||||
|
"outcome": "queueable" if queueable else "blocked",
|
||||||
|
"submission_allowed": queueable,
|
||||||
|
"checks": checks,
|
||||||
|
"existing_result_ids": [],
|
||||||
|
"executor": executor,
|
||||||
|
"authority": projected.get("authority", dict(_OBSERVATION_ONLY_AUTHORITY)),
|
||||||
|
}
|
||||||
|
|
||||||
if portable_setup_projector is not None:
|
if portable_setup_projector is not None:
|
||||||
|
|
||||||
@router.get("/api/v1/observatory/portable-laboratory-setups")
|
@router.get("/api/v1/observatory/portable-laboratory-setups")
|
||||||
@@ -230,7 +364,7 @@ def build_observatory_router(
|
|||||||
except PortableSetupProjectionError as exc:
|
except PortableSetupProjectionError as exc:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=503,
|
status_code=503,
|
||||||
detail="Portable-каталог LAB V1 нарушил контракт целостности.",
|
detail="Portable-каталог сетапов нарушил контракт целостности.",
|
||||||
) from exc
|
) from exc
|
||||||
|
|
||||||
elif portable_setup_projector_error is not None:
|
elif portable_setup_projector_error is not None:
|
||||||
@@ -246,10 +380,10 @@ def build_observatory_router(
|
|||||||
del source_session_id
|
del source_session_id
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=503,
|
status_code=503,
|
||||||
detail="Portable-каталог LAB V1 недоступен.",
|
detail="Portable-каталог сетапов недоступен.",
|
||||||
)
|
)
|
||||||
|
|
||||||
if setup_registry is not None:
|
if setup_registry is not None or portable_setup_projector is not None:
|
||||||
|
|
||||||
@router.get("/api/v1/observatory/laboratory-setups")
|
@router.get("/api/v1/observatory/laboratory-setups")
|
||||||
def list_observatory_laboratory_setups(
|
def list_observatory_laboratory_setups(
|
||||||
@@ -259,6 +393,11 @@ def build_observatory_router(
|
|||||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||||
),
|
),
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
|
if setup_registry is None:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Каталог legacy-сетапов Обсерватории недоступен.",
|
||||||
|
)
|
||||||
source = source_summary(source_session_id)
|
source = source_summary(source_session_id)
|
||||||
return setup_registry.catalog(
|
return setup_registry.catalog(
|
||||||
source,
|
source,
|
||||||
@@ -270,6 +409,11 @@ def build_observatory_router(
|
|||||||
request: ObservatoryRunPreflightRequest,
|
request: ObservatoryRunPreflightRequest,
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
source = source_summary(request.source_session_id)
|
source = source_summary(request.source_session_id)
|
||||||
|
portable = portable_run_preflight(source, request)
|
||||||
|
if portable is not None:
|
||||||
|
return portable
|
||||||
|
if setup_registry is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Сетап лаборатории не найден.")
|
||||||
try:
|
try:
|
||||||
setup_registry.setup(request.setup_id)
|
setup_registry.setup(request.setup_id)
|
||||||
except KeyError as exc:
|
except KeyError as exc:
|
||||||
@@ -639,16 +783,18 @@ def build_observatory_router(
|
|||||||
detail="Подготовка расчётов Обсерватории недоступна.",
|
detail="Подготовка расчётов Обсерватории недоступна.",
|
||||||
)
|
)
|
||||||
|
|
||||||
if (
|
if recorded_job_queue is not None and (
|
||||||
setup_registry is not None
|
recorded_binding_service is not None or portable_binding_service is not None
|
||||||
and recorded_binding_service is not None
|
|
||||||
and recorded_job_queue is not None
|
|
||||||
):
|
):
|
||||||
|
|
||||||
@router.post("/api/v1/observatory/runs", status_code=202)
|
@router.post("/api/v1/observatory/runs", status_code=202)
|
||||||
def submit_observatory_recorded_run(
|
def submit_observatory_recorded_run(
|
||||||
request: ObservatoryRecordedRunSubmitRequest,
|
request: ObservatoryRecordedRunSubmitRequest,
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
|
portable_request = (
|
||||||
|
portable_setup_projector is not None
|
||||||
|
and portable_setup_projector.has_setup(request.setup_id)
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
existing_job = recorded_job_queue.get_by_idempotency_key(request.idempotency_key)
|
existing_job = recorded_job_queue.get_by_idempotency_key(request.idempotency_key)
|
||||||
except ObservatoryRecordedQueueNotFoundError:
|
except ObservatoryRecordedQueueNotFoundError:
|
||||||
@@ -662,6 +808,10 @@ def build_observatory_router(
|
|||||||
if (
|
if (
|
||||||
existing_job.source_session_id != request.source_session_id
|
existing_job.source_session_id != request.source_session_id
|
||||||
or existing_job.setup_id != request.setup_id
|
or existing_job.setup_id != request.setup_id
|
||||||
|
or (
|
||||||
|
portable_request
|
||||||
|
and existing_job.definition_sha256 != request.definition_sha256
|
||||||
|
)
|
||||||
):
|
):
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=409,
|
status_code=409,
|
||||||
@@ -670,6 +820,113 @@ def build_observatory_router(
|
|||||||
return existing_job.as_dict()
|
return existing_job.as_dict()
|
||||||
|
|
||||||
source = source_summary(request.source_session_id)
|
source = source_summary(request.source_session_id)
|
||||||
|
if portable_request:
|
||||||
|
if portable_binding_service is None:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Portable dispatch-контур недоступен.",
|
||||||
|
)
|
||||||
|
if request.definition_sha256 is None or request.check_sha256 is None:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail=(
|
||||||
|
"Для portable-расчёта требуются актуальные definition SHA "
|
||||||
|
"и check SHA из preflight."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert portable_setup_projector is not None
|
||||||
|
try:
|
||||||
|
portable_projection = portable_setup_projector.project(
|
||||||
|
source,
|
||||||
|
setup_id=request.setup_id,
|
||||||
|
)
|
||||||
|
except PortableSetupProjectionError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Portable-каталог сетапов нарушил контракт целостности.",
|
||||||
|
) from exc
|
||||||
|
projected_definition = portable_projection.get("run_definition")
|
||||||
|
projected_executor = portable_projection.get("executor")
|
||||||
|
if not isinstance(projected_definition, dict) or not isinstance(
|
||||||
|
projected_executor, dict
|
||||||
|
):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Portable-каталог сетапов нарушил контракт целостности.",
|
||||||
|
)
|
||||||
|
if projected_definition.get("definition_sha256") != request.definition_sha256:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Идентичность RunDefinition изменилась; повторите preflight.",
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
projected_executor.get("state") != "ready"
|
||||||
|
or projected_executor.get("ready") is not True
|
||||||
|
):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Portable executor-релиз не установлен.",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
job, _created = portable_binding_service.submit(
|
||||||
|
source_session_id=request.source_session_id,
|
||||||
|
setup_id=request.setup_id,
|
||||||
|
definition_sha256=request.definition_sha256,
|
||||||
|
expected_check_sha256=request.check_sha256,
|
||||||
|
idempotency_key=request.idempotency_key,
|
||||||
|
)
|
||||||
|
return job.as_dict()
|
||||||
|
except PortableQueueBindingStaleCheckError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Источник или RunDefinition изменились; повторите preflight.",
|
||||||
|
) from exc
|
||||||
|
except PortableRunDefinitionUnavailableError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Portable executor-релиз не установлен.",
|
||||||
|
) from exc
|
||||||
|
except (
|
||||||
|
PortableQueueBindingIntegrityError,
|
||||||
|
PortableSourceAdmissionError,
|
||||||
|
) as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Portable-привязка источника не прошла проверку целостности.",
|
||||||
|
) from exc
|
||||||
|
except ObservatoryRecordedQueueConflictError as exc:
|
||||||
|
try:
|
||||||
|
raced = recorded_job_queue.get_by_idempotency_key(request.idempotency_key)
|
||||||
|
except ObservatoryRecordedQueueNotFoundError:
|
||||||
|
raced = None
|
||||||
|
if (
|
||||||
|
raced is not None
|
||||||
|
and raced.source_session_id == request.source_session_id
|
||||||
|
and raced.setup_id == request.setup_id
|
||||||
|
and raced.definition_sha256 == request.definition_sha256
|
||||||
|
):
|
||||||
|
return raced.as_dict()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Ключ идемпотентности уже связан с другим расчётом.",
|
||||||
|
) from exc
|
||||||
|
except ObservatoryRecordedQueueCapacityError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Квота durable-очереди расчётов исчерпана.",
|
||||||
|
) from exc
|
||||||
|
except (
|
||||||
|
PortableQueueBindingError,
|
||||||
|
ObservatoryRecordedQueueError,
|
||||||
|
ValueError,
|
||||||
|
) as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Portable dispatch-контур недоступен.",
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
if setup_registry is None or recorded_binding_service is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Сетап лаборатории не найден.")
|
||||||
try:
|
try:
|
||||||
setup_registry.setup(request.setup_id)
|
setup_registry.setup(request.setup_id)
|
||||||
except KeyError as exc:
|
except KeyError as exc:
|
||||||
|
|||||||
@@ -18,11 +18,23 @@ from dataclasses import dataclass
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Annotated, Final, Literal
|
from typing import Annotated, Final, Literal
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Header, HTTPException, Response
|
from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response
|
||||||
from fastapi import Path as ApiPath
|
from fastapi import Path as ApiPath
|
||||||
|
from fastapi.responses import FileResponse
|
||||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||||
from pydantic import BaseModel, ConfigDict, Field
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
|
|
||||||
|
from k1link.observatory.portable_artifact_transport import (
|
||||||
|
MAX_RESULT_MANIFEST_BYTES,
|
||||||
|
PortableArtifactTransportError,
|
||||||
|
PortableArtifactTransportIntegrityError,
|
||||||
|
PortableArtifactTransportUnavailableError,
|
||||||
|
PortableObservatoryArtifactTransport,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_result_contract import PortableResultPublisherError
|
||||||
|
from k1link.observatory.portable_result_publisher import (
|
||||||
|
PortableObservatoryResultPublisher,
|
||||||
|
)
|
||||||
from k1link.observatory.recorded_jobs import (
|
from k1link.observatory.recorded_jobs import (
|
||||||
ObservatoryRecordedCheckpointError,
|
ObservatoryRecordedCheckpointError,
|
||||||
ObservatoryRecordedJobQueue,
|
ObservatoryRecordedJobQueue,
|
||||||
@@ -38,6 +50,7 @@ from k1link.observatory.recorded_jobs import (
|
|||||||
|
|
||||||
OBSERVATORY_WORKER_CLAIM_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-claim-request/v1"
|
OBSERVATORY_WORKER_CLAIM_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-claim-request/v1"
|
||||||
OBSERVATORY_WORKER_START_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-start-request/v1"
|
OBSERVATORY_WORKER_START_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-start-request/v1"
|
||||||
|
OBSERVATORY_WORKER_RENEW_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-renew-request/v1"
|
||||||
OBSERVATORY_WORKER_CHECKPOINT_REQUEST_SCHEMA: Final = (
|
OBSERVATORY_WORKER_CHECKPOINT_REQUEST_SCHEMA: Final = (
|
||||||
"missioncore.observatory-worker-checkpoint-request/v1"
|
"missioncore.observatory-worker-checkpoint-request/v1"
|
||||||
)
|
)
|
||||||
@@ -46,6 +59,10 @@ OBSERVATORY_WORKER_SUCCEED_REQUEST_SCHEMA: Final = (
|
|||||||
)
|
)
|
||||||
OBSERVATORY_WORKER_FAIL_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-fail-request/v1"
|
OBSERVATORY_WORKER_FAIL_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-fail-request/v1"
|
||||||
OBSERVATORY_WORKER_CONTOUR_HEADER: Final = "X-Mission-Core-Contour-Id"
|
OBSERVATORY_WORKER_CONTOUR_HEADER: Final = "X-Mission-Core-Contour-Id"
|
||||||
|
OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER: Final = "X-Mission-Core-Claim-Token"
|
||||||
|
OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER: Final = (
|
||||||
|
"X-Mission-Core-Claim-Generation"
|
||||||
|
)
|
||||||
|
|
||||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||||
@@ -138,6 +155,13 @@ class ObservatoryWorkerStartRequest(_StrictWorkerRequest):
|
|||||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||||
|
|
||||||
|
|
||||||
|
class ObservatoryWorkerRenewRequest(_StrictWorkerRequest):
|
||||||
|
schema_version: Literal["missioncore.observatory-worker-renew-request/v1"]
|
||||||
|
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||||
|
claim_generation: int = Field(ge=1)
|
||||||
|
heartbeat_sequence: int = Field(ge=1)
|
||||||
|
|
||||||
|
|
||||||
class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest):
|
class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest):
|
||||||
schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"]
|
schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"]
|
||||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||||
@@ -174,6 +198,8 @@ def build_observatory_worker_router(
|
|||||||
queue: ObservatoryRecordedJobQueue,
|
queue: ObservatoryRecordedJobQueue,
|
||||||
*,
|
*,
|
||||||
authentication: ObservatoryWorkerAuthentication,
|
authentication: ObservatoryWorkerAuthentication,
|
||||||
|
artifact_transport: PortableObservatoryArtifactTransport | None = None,
|
||||||
|
result_publisher: PortableObservatoryResultPublisher | None = None,
|
||||||
) -> APIRouter:
|
) -> APIRouter:
|
||||||
"""Build the bounded Worker pull/state-transition router.
|
"""Build the bounded Worker pull/state-transition router.
|
||||||
|
|
||||||
@@ -182,6 +208,9 @@ def build_observatory_worker_router(
|
|||||||
hashed, and is compared to the configured digest in constant time.
|
hashed, and is compared to the configured digest in constant time.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
if result_publisher is not None and artifact_transport is None:
|
||||||
|
raise ValueError("portable result publisher requires artifact transport")
|
||||||
|
|
||||||
def require_configured_worker(
|
def require_configured_worker(
|
||||||
credentials: Annotated[
|
credentials: Annotated[
|
||||||
HTTPAuthorizationCredentials | None,
|
HTTPAuthorizationCredentials | None,
|
||||||
@@ -245,6 +274,20 @@ def build_observatory_worker_router(
|
|||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
return _queue_call(lambda: queue.start(job_id, claim_token=request.claim_token)).as_dict()
|
return _queue_call(lambda: queue.start(job_id, claim_token=request.claim_token)).as_dict()
|
||||||
|
|
||||||
|
@router.post("/recorded-jobs/{job_id}/lease/renew")
|
||||||
|
def renew_job_claim(
|
||||||
|
request: ObservatoryWorkerRenewRequest,
|
||||||
|
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return _queue_call(
|
||||||
|
lambda: queue.renew_claim(
|
||||||
|
job_id,
|
||||||
|
claim_token=request.claim_token,
|
||||||
|
claim_generation=request.claim_generation,
|
||||||
|
heartbeat_sequence=request.heartbeat_sequence,
|
||||||
|
)
|
||||||
|
).as_dict()
|
||||||
|
|
||||||
@router.post("/recorded-jobs/{job_id}/checkpoint")
|
@router.post("/recorded-jobs/{job_id}/checkpoint")
|
||||||
def checkpoint_job(
|
def checkpoint_job(
|
||||||
request: ObservatoryWorkerCheckpointRequest,
|
request: ObservatoryWorkerCheckpointRequest,
|
||||||
@@ -263,14 +306,39 @@ def build_observatory_worker_router(
|
|||||||
request: ObservatoryWorkerSucceedRequest,
|
request: ObservatoryWorkerSucceedRequest,
|
||||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
return _queue_call(
|
if artifact_transport is not None:
|
||||||
|
_artifact_call(
|
||||||
|
lambda: artifact_transport.require_completed_for_success(
|
||||||
|
job_id=job_id,
|
||||||
|
result_id=request.result_id,
|
||||||
|
result_sha256=request.result_sha256,
|
||||||
|
claim_token=request.claim_token,
|
||||||
|
claimant_id=authentication.contour_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
succeeded = _queue_call(
|
||||||
lambda: queue.succeed(
|
lambda: queue.succeed(
|
||||||
job_id,
|
job_id,
|
||||||
claim_token=request.claim_token,
|
claim_token=request.claim_token,
|
||||||
result_id=request.result_id,
|
result_id=request.result_id,
|
||||||
result_sha256=request.result_sha256,
|
result_sha256=request.result_sha256,
|
||||||
)
|
)
|
||||||
).as_dict()
|
)
|
||||||
|
if artifact_transport is not None and result_publisher is not None:
|
||||||
|
package_root = _artifact_call(
|
||||||
|
lambda: artifact_transport.package_root_for_terminal(succeeded)
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
result_publisher.publish(job=succeeded, package_root=package_root)
|
||||||
|
except PortableResultPublisherError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail=(
|
||||||
|
"Recorded result is sealed but its verified publication "
|
||||||
|
"requires reconciliation."
|
||||||
|
),
|
||||||
|
) from exc
|
||||||
|
return succeeded.as_dict()
|
||||||
|
|
||||||
@router.post("/recorded-jobs/{job_id}/fail")
|
@router.post("/recorded-jobs/{job_id}/fail")
|
||||||
def fail_job(
|
def fail_job(
|
||||||
@@ -286,6 +354,164 @@ def build_observatory_worker_router(
|
|||||||
)
|
)
|
||||||
).as_dict()
|
).as_dict()
|
||||||
|
|
||||||
|
if artifact_transport is not None:
|
||||||
|
|
||||||
|
@router.get("/recorded-jobs/{job_id}/source-materialization")
|
||||||
|
def source_materialization(
|
||||||
|
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||||
|
claim_token: Annotated[
|
||||||
|
str,
|
||||||
|
Header(
|
||||||
|
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
|
||||||
|
pattern=_CLAIM_TOKEN_PATTERN,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
claim_generation: Annotated[
|
||||||
|
int,
|
||||||
|
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
|
||||||
|
],
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return _artifact_call(
|
||||||
|
lambda: artifact_transport.source_manifest(
|
||||||
|
job_id=job_id,
|
||||||
|
claim_token=claim_token,
|
||||||
|
claim_generation=claim_generation,
|
||||||
|
claimant_id=authentication.contour_id,
|
||||||
|
)
|
||||||
|
.as_dict()
|
||||||
|
)
|
||||||
|
|
||||||
|
@router.get("/recorded-jobs/{job_id}/source-members/{member_id}")
|
||||||
|
def source_member(
|
||||||
|
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||||
|
member_id: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
|
||||||
|
claim_token: Annotated[
|
||||||
|
str,
|
||||||
|
Header(
|
||||||
|
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
|
||||||
|
pattern=_CLAIM_TOKEN_PATTERN,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
claim_generation: Annotated[
|
||||||
|
int,
|
||||||
|
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
|
||||||
|
],
|
||||||
|
) -> FileResponse:
|
||||||
|
member, path = _artifact_call(
|
||||||
|
lambda: artifact_transport.materialize_source_member(
|
||||||
|
job_id=job_id,
|
||||||
|
member_id=member_id,
|
||||||
|
claim_token=claim_token,
|
||||||
|
claim_generation=claim_generation,
|
||||||
|
claimant_id=authentication.contour_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return FileResponse(
|
||||||
|
path,
|
||||||
|
media_type=member.media_type,
|
||||||
|
headers={
|
||||||
|
"ETag": f'"{member.sha256}"',
|
||||||
|
"Cache-Control": "private, no-store",
|
||||||
|
"X-Content-Type-Options": "nosniff",
|
||||||
|
"X-Mission-Core-Content-Sha256": member.sha256,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
@router.put(
|
||||||
|
"/recorded-jobs/{job_id}/result-packages/{result_sha256}/manifest"
|
||||||
|
)
|
||||||
|
async def stage_result_manifest(
|
||||||
|
request: Request,
|
||||||
|
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||||
|
result_sha256: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
|
||||||
|
claim_token: Annotated[
|
||||||
|
str,
|
||||||
|
Header(
|
||||||
|
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
|
||||||
|
pattern=_CLAIM_TOKEN_PATTERN,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
claim_generation: Annotated[
|
||||||
|
int,
|
||||||
|
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
|
||||||
|
],
|
||||||
|
) -> dict[str, object]:
|
||||||
|
payload = await _read_bounded_body(request, MAX_RESULT_MANIFEST_BYTES)
|
||||||
|
return _artifact_call(
|
||||||
|
lambda: artifact_transport.stage_result_manifest(
|
||||||
|
job_id=job_id,
|
||||||
|
result_sha256=result_sha256,
|
||||||
|
manifest_payload=payload,
|
||||||
|
claim_token=claim_token,
|
||||||
|
claim_generation=claim_generation,
|
||||||
|
claimant_id=authentication.contour_id,
|
||||||
|
)
|
||||||
|
.as_dict()
|
||||||
|
)
|
||||||
|
|
||||||
|
@router.put(
|
||||||
|
"/recorded-jobs/{job_id}/result-packages/{result_sha256}/members/{member_id}"
|
||||||
|
)
|
||||||
|
async def upload_result_member(
|
||||||
|
request: Request,
|
||||||
|
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||||
|
result_sha256: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
|
||||||
|
member_id: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
|
||||||
|
claim_token: Annotated[
|
||||||
|
str,
|
||||||
|
Header(
|
||||||
|
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
|
||||||
|
pattern=_CLAIM_TOKEN_PATTERN,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
claim_generation: Annotated[
|
||||||
|
int,
|
||||||
|
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
|
||||||
|
],
|
||||||
|
) -> dict[str, object]:
|
||||||
|
try:
|
||||||
|
plan = await artifact_transport.upload_result_member(
|
||||||
|
job_id=job_id,
|
||||||
|
result_sha256=result_sha256,
|
||||||
|
member_id=member_id,
|
||||||
|
chunks=request.stream(),
|
||||||
|
claim_token=claim_token,
|
||||||
|
claim_generation=claim_generation,
|
||||||
|
claimant_id=authentication.contour_id,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
_raise_artifact_or_queue_error(exc)
|
||||||
|
return plan.as_dict()
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/recorded-jobs/{job_id}/result-packages/{result_sha256}/complete"
|
||||||
|
)
|
||||||
|
def complete_result_package(
|
||||||
|
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||||
|
result_sha256: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
|
||||||
|
claim_token: Annotated[
|
||||||
|
str,
|
||||||
|
Header(
|
||||||
|
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
|
||||||
|
pattern=_CLAIM_TOKEN_PATTERN,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
claim_generation: Annotated[
|
||||||
|
int,
|
||||||
|
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
|
||||||
|
],
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return _artifact_call(
|
||||||
|
lambda: artifact_transport.complete_result_upload(
|
||||||
|
job_id=job_id,
|
||||||
|
result_sha256=result_sha256,
|
||||||
|
claim_token=claim_token,
|
||||||
|
claim_generation=claim_generation,
|
||||||
|
claimant_id=authentication.contour_id,
|
||||||
|
)
|
||||||
|
.as_dict()
|
||||||
|
)
|
||||||
|
|
||||||
return router
|
return router
|
||||||
|
|
||||||
|
|
||||||
@@ -341,3 +567,66 @@ def _queue_call[T](operation: Callable[[], T]) -> T:
|
|||||||
status_code=503,
|
status_code=503,
|
||||||
detail="Recorded-job queue is unavailable.",
|
detail="Recorded-job queue is unavailable.",
|
||||||
) from exc
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _artifact_call[T](operation: Callable[[], T]) -> T:
|
||||||
|
try:
|
||||||
|
return _queue_call(operation)
|
||||||
|
except PortableArtifactTransportIntegrityError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Worker artifact identity was rejected.",
|
||||||
|
) from exc
|
||||||
|
except PortableArtifactTransportUnavailableError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Worker artifact member is unavailable for this claim.",
|
||||||
|
) from exc
|
||||||
|
except PortableArtifactTransportError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Worker artifact transport is unavailable.",
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _raise_artifact_or_queue_error(exc: Exception) -> None:
|
||||||
|
if isinstance(exc, PortableArtifactTransportIntegrityError):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Worker artifact identity was rejected.",
|
||||||
|
) from exc
|
||||||
|
if isinstance(exc, PortableArtifactTransportUnavailableError):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Worker artifact member is unavailable for this claim.",
|
||||||
|
) from exc
|
||||||
|
if isinstance(exc, PortableArtifactTransportError):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="Worker artifact transport is unavailable.",
|
||||||
|
) from exc
|
||||||
|
_queue_call(lambda: _raise(exc))
|
||||||
|
raise AssertionError("unreachable")
|
||||||
|
|
||||||
|
|
||||||
|
def _raise(exc: Exception) -> None:
|
||||||
|
raise exc
|
||||||
|
|
||||||
|
|
||||||
|
async def _read_bounded_body(request: Request, maximum_bytes: int) -> bytes:
|
||||||
|
content_length = request.headers.get("content-length")
|
||||||
|
if content_length is not None:
|
||||||
|
try:
|
||||||
|
declared = int(content_length)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise HTTPException(status_code=400, detail="Content-Length is invalid.") from exc
|
||||||
|
if declared < 1 or declared > maximum_bytes:
|
||||||
|
raise HTTPException(status_code=413, detail="Request body is outside bounds.")
|
||||||
|
payload = bytearray()
|
||||||
|
async for chunk in request.stream():
|
||||||
|
payload.extend(chunk)
|
||||||
|
if len(payload) > maximum_bytes:
|
||||||
|
raise HTTPException(status_code=413, detail="Request body is outside bounds.")
|
||||||
|
if not payload:
|
||||||
|
raise HTTPException(status_code=400, detail="Request body is empty.")
|
||||||
|
return bytes(payload)
|
||||||
|
|||||||
@@ -41,6 +41,7 @@ from k1link.sessions.canonical_lab_spatial import (
|
|||||||
CANONICAL_LAB_SPATIAL_PROFILE,
|
CANONICAL_LAB_SPATIAL_PROFILE,
|
||||||
canonical_lab_spatial_frame,
|
canonical_lab_spatial_frame,
|
||||||
)
|
)
|
||||||
|
from k1link.sessions.models import SessionSummary
|
||||||
from k1link.sessions.plugin_contract import RecordedPointColorRenderer
|
from k1link.sessions.plugin_contract import RecordedPointColorRenderer
|
||||||
from k1link.viewer.recorded import (
|
from k1link.viewer.recorded import (
|
||||||
APPLICATION_ID as RECORDED_APPLICATION_ID,
|
APPLICATION_ID as RECORDED_APPLICATION_ID,
|
||||||
@@ -328,6 +329,9 @@ def build_session_router(
|
|||||||
perception_overlay_provider: RecordedPerceptionOverlayProvider | None = None,
|
perception_overlay_provider: RecordedPerceptionOverlayProvider | None = None,
|
||||||
perception_media_provider: RecordedPerceptionMediaProvider | None = None,
|
perception_media_provider: RecordedPerceptionMediaProvider | None = None,
|
||||||
point_color_renderers: Mapping[str, RecordedPointColorRenderer] | None = None,
|
point_color_renderers: Mapping[str, RecordedPointColorRenderer] | None = None,
|
||||||
|
lab_calculation_profile_resolver: (
|
||||||
|
Callable[[SessionSummary], Mapping[str, object] | None] | None
|
||||||
|
) = None,
|
||||||
allow_synchronous_recording_fallback: bool = False,
|
allow_synchronous_recording_fallback: bool = False,
|
||||||
replay_action_id: str = DEFAULT_REPLAY_ACTION_ID,
|
replay_action_id: str = DEFAULT_REPLAY_ACTION_ID,
|
||||||
) -> APIRouter:
|
) -> APIRouter:
|
||||||
@@ -336,12 +340,29 @@ def build_session_router(
|
|||||||
router = APIRouter(tags=["observation-sessions"])
|
router = APIRouter(tags=["observation-sessions"])
|
||||||
recorded_media_inspector = media_inspector or RecordedMediaInspector()
|
recorded_media_inspector = media_inspector or RecordedMediaInspector()
|
||||||
|
|
||||||
|
def lab_catalog_document(
|
||||||
|
summary: SessionSummary,
|
||||||
|
contract: Literal["v1", "v2", "v3"],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
lab = summary.lab
|
||||||
|
if lab is None:
|
||||||
|
raise ValueError("LAB catalog document requires a LAB summary")
|
||||||
|
document = lab.as_dict(include_replay_capability=contract in ("v2", "v3"))
|
||||||
|
if contract == "v3":
|
||||||
|
profile = (
|
||||||
|
None
|
||||||
|
if lab_calculation_profile_resolver is None
|
||||||
|
else lab_calculation_profile_resolver(summary)
|
||||||
|
)
|
||||||
|
document["calculation_profile"] = None if profile is None else dict(profile)
|
||||||
|
return document
|
||||||
|
|
||||||
@router.get("/api/v1/observation-sessions")
|
@router.get("/api/v1/observation-sessions")
|
||||||
def list_observation_sessions(
|
def list_observation_sessions(
|
||||||
limit: int = Query(default=20, ge=1, le=100),
|
limit: int = Query(default=20, ge=1, le=100),
|
||||||
cursor: str | None = Query(default=None, max_length=128),
|
cursor: str | None = Query(default=None, max_length=128),
|
||||||
scope: Literal["all", "source", "laboratory"] = "all",
|
scope: Literal["all", "source", "laboratory"] = "all",
|
||||||
lab_contract: Literal["v1", "v2"] = "v1",
|
lab_contract: Literal["v1", "v2", "v3"] = "v1",
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
_refresh_catalog(catalog_refresher)
|
_refresh_catalog(catalog_refresher)
|
||||||
@@ -349,7 +370,7 @@ def build_session_router(
|
|||||||
limit=limit,
|
limit=limit,
|
||||||
cursor=cursor,
|
cursor=cursor,
|
||||||
scope=scope,
|
scope=scope,
|
||||||
include_capability_projections=lab_contract == "v2",
|
include_capability_projections=lab_contract in ("v2", "v3"),
|
||||||
)
|
)
|
||||||
return {
|
return {
|
||||||
"items": [
|
"items": [
|
||||||
@@ -364,9 +385,7 @@ def build_session_router(
|
|||||||
"replayable": item.replayable,
|
"replayable": item.replayable,
|
||||||
**(
|
**(
|
||||||
{
|
{
|
||||||
"lab": item.lab.as_dict(
|
"lab": lab_catalog_document(item, lab_contract)
|
||||||
include_replay_capability=lab_contract == "v2"
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
if item.lab is not None
|
if item.lab is not None
|
||||||
else {}
|
else {}
|
||||||
|
|||||||
@@ -0,0 +1,836 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from typing import cast
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import k1link.observatory.m49_portable_source as source_module
|
||||||
|
from k1link.observatory.m49_portable_executor import (
|
||||||
|
M49_PORTABLE_RUNTIME_PHASES,
|
||||||
|
M49PortableBoundSourceStage,
|
||||||
|
M49PortableProfileRunnerAdapter,
|
||||||
|
M49PortableRunnerInstallation,
|
||||||
|
M49PortableSourceMaterializerAdapter,
|
||||||
|
)
|
||||||
|
from k1link.observatory.m49_portable_result import (
|
||||||
|
M49PortableResultError,
|
||||||
|
validate_m49_portable_result,
|
||||||
|
)
|
||||||
|
from k1link.observatory.m49_portable_source import (
|
||||||
|
M49PortableSourceError,
|
||||||
|
M49PortableSourceIdentity,
|
||||||
|
materialize_m49_portable_source,
|
||||||
|
read_m49_source_index,
|
||||||
|
validate_m49_portable_source_stage,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_result_contract import (
|
||||||
|
PortableResultPackageManifest,
|
||||||
|
PortableResultValidationContext,
|
||||||
|
canonical_json,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_run_definitions import (
|
||||||
|
PortableRunDefinition,
|
||||||
|
PortableRunDefinitionRegistry,
|
||||||
|
canonical_sha256,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_worker_runtime import (
|
||||||
|
PortableWorkerRuntimePlan,
|
||||||
|
PortableWorkerSourceStage,
|
||||||
|
)
|
||||||
|
from k1link.observatory.recorded_jobs import (
|
||||||
|
ObservatoryRecordedJob,
|
||||||
|
ObservatoryRecordedJobIntent,
|
||||||
|
ObservatoryRecordedJobQueue,
|
||||||
|
RecordedRunDefinitionRegistry,
|
||||||
|
)
|
||||||
|
from k1link.observatory.source_admission import (
|
||||||
|
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||||
|
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||||
|
)
|
||||||
|
from k1link.observatory.worker_agent import (
|
||||||
|
ObservatoryWorkerExecutorIdentity,
|
||||||
|
SealedObservatoryRecordedJob,
|
||||||
|
)
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||||
|
PROFILE_PATH = REPOSITORY_ROOT / "config" / "perception" / "m49-tgs-portable-v2.json"
|
||||||
|
RUNNER_PATH = (
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "observatory_portable"
|
||||||
|
/ "run_m49_tgs_portable.cpp"
|
||||||
|
)
|
||||||
|
BUILDER_PATH = REPOSITORY_ROOT / "scripts" / "build_m49_portable_executor_release.py"
|
||||||
|
DOCKERFILE_PATH = (
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "observatory_portable"
|
||||||
|
/ "Dockerfile.m49-portable-executor"
|
||||||
|
)
|
||||||
|
INSTALLER_PATH = (
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "observatory_portable"
|
||||||
|
/ "Invoke-M49PortableExecutorCandidateInstall.ps1"
|
||||||
|
)
|
||||||
|
NOW = "2026-08-31T09:00:00.000Z"
|
||||||
|
SESSION_ID = "20260831T085500Z_viewer_live"
|
||||||
|
AUTHORITY = {
|
||||||
|
"commands_enabled": False,
|
||||||
|
"actuation_allowed": False,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
"production_accepted": False,
|
||||||
|
}
|
||||||
|
RAW_PAYLOAD = b"exact-k1-raw-replay"
|
||||||
|
METADATA_PAYLOAD = b'{"received_monotonic_ns":1}\n'
|
||||||
|
|
||||||
|
_builder_spec = importlib.util.spec_from_file_location(
|
||||||
|
"build_m49_portable_executor_release", BUILDER_PATH
|
||||||
|
)
|
||||||
|
assert _builder_spec is not None and _builder_spec.loader is not None
|
||||||
|
builder = importlib.util.module_from_spec(_builder_spec)
|
||||||
|
sys.modules[_builder_spec.name] = builder
|
||||||
|
_builder_spec.loader.exec_module(builder)
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeLidarPack:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
adapter_sha256: str,
|
||||||
|
raw_sha256: str,
|
||||||
|
metadata_sha256: str,
|
||||||
|
) -> None:
|
||||||
|
del adapter_sha256
|
||||||
|
self.pack_id = f"lidar-replay-pack-{'c' * 64}"
|
||||||
|
self.identity = {
|
||||||
|
"session_id": SESSION_ID,
|
||||||
|
"logical_content_sha256": "e" * 64,
|
||||||
|
"source_evidence": {
|
||||||
|
"raw": {
|
||||||
|
"byte_length": len(RAW_PAYLOAD),
|
||||||
|
"sha256": raw_sha256,
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"byte_length": len(METADATA_PAYLOAD),
|
||||||
|
"sha256": metadata_sha256,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
self.manifest = {"identity_sha256": "d" * 64}
|
||||||
|
self.arrays = {
|
||||||
|
"point_received_monotonic_ns": np.asarray([100_000_000, 1_100_000_000], dtype=np.int64),
|
||||||
|
"pose_received_monotonic_ns": np.asarray([50_000_000, 1_050_000_000], dtype=np.int64),
|
||||||
|
"pose_positions_map": np.zeros((2, 3), dtype=np.float64),
|
||||||
|
}
|
||||||
|
self._points = (
|
||||||
|
np.asarray(
|
||||||
|
[[2.0, 0.0, 0.0], [3.0, 0.0, 1.0], [4.0, 0.0, 0.2]],
|
||||||
|
dtype=np.float64,
|
||||||
|
),
|
||||||
|
np.asarray(
|
||||||
|
[[2.5, 0.0, 0.0], [3.5, 0.0, 1.0], [4.5, 0.0, 0.2]],
|
||||||
|
dtype=np.float64,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def point_frame(self, index: int) -> SimpleNamespace:
|
||||||
|
return SimpleNamespace(
|
||||||
|
xyz_map=self._points[index],
|
||||||
|
intensity=np.asarray([10, 20, 30], dtype=np.uint8),
|
||||||
|
)
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256(path: Path) -> str:
|
||||||
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _m49_definition() -> PortableRunDefinition:
|
||||||
|
return PortableRunDefinitionRegistry.from_file(REGISTRY_PATH).resolve_setup(
|
||||||
|
"m49-tgs-portable-v2"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _materialized_source(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
*,
|
||||||
|
include_metadata_member: bool = True,
|
||||||
|
) -> tuple[Path, M49PortableSourceIdentity]:
|
||||||
|
definition = _m49_definition()
|
||||||
|
adapter_sha256 = definition.source_adapter.contract_sha256
|
||||||
|
raw_sha256 = hashlib.sha256(RAW_PAYLOAD).hexdigest()
|
||||||
|
metadata_sha256 = hashlib.sha256(METADATA_PAYLOAD).hexdigest()
|
||||||
|
bundle = {
|
||||||
|
"schema_version": PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||||
|
"source_session_id": SESSION_ID,
|
||||||
|
"source_catalog_sha256": "1" * 64,
|
||||||
|
"plugin_id": definition.source_requirements.plugin_id,
|
||||||
|
"archive_id": definition.source_requirements.archive_id,
|
||||||
|
"source_adapter": {
|
||||||
|
"id": definition.source_adapter.adapter_id,
|
||||||
|
"version": definition.source_adapter.version,
|
||||||
|
"sha256": adapter_sha256,
|
||||||
|
},
|
||||||
|
"sources": [],
|
||||||
|
"spatial_replay": {
|
||||||
|
"timeline_origin_monotonic_ns": 0,
|
||||||
|
"members": [
|
||||||
|
{
|
||||||
|
"artifact_id": "raw-transport-primary",
|
||||||
|
"media_type": "application/x-nodedc-k1mqtt",
|
||||||
|
"byte_length": len(RAW_PAYLOAD),
|
||||||
|
"replay_byte_length": len(RAW_PAYLOAD),
|
||||||
|
"sha256": raw_sha256,
|
||||||
|
},
|
||||||
|
*(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"artifact_id": "raw-transport-index",
|
||||||
|
"media_type": "application/x-ndjson",
|
||||||
|
"byte_length": len(METADATA_PAYLOAD),
|
||||||
|
"replay_byte_length": len(METADATA_PAYLOAD),
|
||||||
|
"sha256": metadata_sha256,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
if include_metadata_member
|
||||||
|
else []
|
||||||
|
),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"camera": {
|
||||||
|
"generation_sha256": "2" * 64,
|
||||||
|
"epoch": {
|
||||||
|
"timeline_start_seconds": 0.0,
|
||||||
|
"timeline_end_seconds": 3.0,
|
||||||
|
"segments": [
|
||||||
|
{"sequence": 1, "end_time_seconds": 1.0},
|
||||||
|
{"sequence": 2, "end_time_seconds": 2.0},
|
||||||
|
{"sequence": 3, "end_time_seconds": 3.0},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"authority": AUTHORITY,
|
||||||
|
}
|
||||||
|
bundle_payload = canonical_json(bundle)
|
||||||
|
bundle_sha256 = hashlib.sha256(bundle_payload).hexdigest()
|
||||||
|
capability = {
|
||||||
|
"schema_version": PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||||
|
"source_session_id": SESSION_ID,
|
||||||
|
"source_catalog_sha256": "1" * 64,
|
||||||
|
"source_bundle_sha256": bundle_sha256,
|
||||||
|
"source_adapter_sha256": adapter_sha256,
|
||||||
|
"modalities": ["point-cloud", "trajectory", "video"],
|
||||||
|
"camera_profile": {
|
||||||
|
"generation_sha256": "2" * 64,
|
||||||
|
"frame_count": 3,
|
||||||
|
},
|
||||||
|
"calibration": {},
|
||||||
|
"authority": AUTHORITY,
|
||||||
|
}
|
||||||
|
capability_payload = canonical_json(capability)
|
||||||
|
capability_sha256 = hashlib.sha256(capability_payload).hexdigest()
|
||||||
|
bundle_path = tmp_path / "source-bundle.json"
|
||||||
|
capability_path = tmp_path / "source-capability.json"
|
||||||
|
bundle_path.write_bytes(bundle_payload)
|
||||||
|
capability_path.write_bytes(capability_payload)
|
||||||
|
expected = M49PortableSourceIdentity(
|
||||||
|
source_session_id=SESSION_ID,
|
||||||
|
source_catalog_sha256="1" * 64,
|
||||||
|
source_bundle_sha256=bundle_sha256,
|
||||||
|
source_capability_manifest_sha256=capability_sha256,
|
||||||
|
source_adapter_sha256=adapter_sha256,
|
||||||
|
raw_capture_sha256=raw_sha256,
|
||||||
|
metadata_sha256=metadata_sha256,
|
||||||
|
)
|
||||||
|
fake = _FakeLidarPack(
|
||||||
|
adapter_sha256=adapter_sha256,
|
||||||
|
raw_sha256=raw_sha256,
|
||||||
|
metadata_sha256=metadata_sha256,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(source_module, "LidarReplayPackV2", lambda _root: fake)
|
||||||
|
stage = materialize_m49_portable_source(
|
||||||
|
source_bundle_path=bundle_path,
|
||||||
|
source_capability_path=capability_path,
|
||||||
|
lidar_pack_root=tmp_path,
|
||||||
|
profile_path=PROFILE_PATH,
|
||||||
|
output_parent=tmp_path / "source-stages",
|
||||||
|
expected=expected,
|
||||||
|
)
|
||||||
|
return stage.root, expected
|
||||||
|
|
||||||
|
|
||||||
|
def _ready_m49_registry(tmp_path: Path) -> PortableRunDefinitionRegistry:
|
||||||
|
base_registry = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||||
|
base = base_registry.resolve_setup("m49-tgs-portable-v2")
|
||||||
|
document = cast(dict[str, object], json.loads(REGISTRY_PATH.read_text(encoding="utf-8")))
|
||||||
|
selected = copy.deepcopy(
|
||||||
|
next(
|
||||||
|
cast(dict[str, object], row)
|
||||||
|
for row in cast(list[object], document["definitions"])
|
||||||
|
if cast(dict[str, object], row)["setup_id"] == "m49-tgs-portable-v2"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
runner = {
|
||||||
|
"component_id": "m49-tgs-portable-runner-v1",
|
||||||
|
"kind": "runner",
|
||||||
|
"sha256": _sha256(RUNNER_PATH),
|
||||||
|
}
|
||||||
|
components = cast(list[object], selected["components"])
|
||||||
|
components.append(runner)
|
||||||
|
components.sort(key=lambda value: cast(str, cast(dict[str, object], value)["component_id"]))
|
||||||
|
executor = {
|
||||||
|
"contour_id": "worker-006",
|
||||||
|
"state": "ready",
|
||||||
|
"release_id": "m49-tgs-portable-executor-v1",
|
||||||
|
"release_sha256": "3" * 64,
|
||||||
|
"image_sha256": "4" * 64,
|
||||||
|
"reason_code": None,
|
||||||
|
"reason": None,
|
||||||
|
}
|
||||||
|
selected["executor"] = executor
|
||||||
|
identity = copy.deepcopy(base.identity_document())
|
||||||
|
identity["components"] = copy.deepcopy(components)
|
||||||
|
identity["executor"] = {
|
||||||
|
key: executor[key]
|
||||||
|
for key in ("contour_id", "state", "release_id", "release_sha256", "image_sha256")
|
||||||
|
}
|
||||||
|
selected["definition_sha256"] = canonical_sha256(identity)
|
||||||
|
path = tmp_path / "ready-m49-registry.json"
|
||||||
|
path.write_bytes(
|
||||||
|
canonical_json(
|
||||||
|
{
|
||||||
|
"schema_version": document["schema_version"],
|
||||||
|
"definitions": [selected],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return PortableRunDefinitionRegistry.from_file(path)
|
||||||
|
|
||||||
|
|
||||||
|
def _running_job(
|
||||||
|
tmp_path: Path,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
expected: M49PortableSourceIdentity,
|
||||||
|
) -> tuple[ObservatoryRecordedJobQueue, ObservatoryRecordedJob, str]:
|
||||||
|
queue = ObservatoryRecordedJobQueue(
|
||||||
|
tmp_path / "queue",
|
||||||
|
definitions=RecordedRunDefinitionRegistry((definition.to_recorded_run_definition(),)),
|
||||||
|
clock=lambda: NOW,
|
||||||
|
)
|
||||||
|
job, created = queue.submit(
|
||||||
|
ObservatoryRecordedJobIntent(
|
||||||
|
idempotency_key="m49-portable-executor-test-001",
|
||||||
|
source_session_id=SESSION_ID,
|
||||||
|
source_catalog_sha256=expected.source_catalog_sha256,
|
||||||
|
source_bundle_sha256=expected.source_bundle_sha256,
|
||||||
|
source_capability_manifest_sha256=(expected.source_capability_manifest_sha256),
|
||||||
|
setup_id=definition.setup_id,
|
||||||
|
definition_sha256=definition.definition_sha256,
|
||||||
|
),
|
||||||
|
enqueue=True,
|
||||||
|
)
|
||||||
|
assert created is True
|
||||||
|
claim = queue.claim_next(claimant_id="worker-006", claim_request_id="m49-portable-claim-001")
|
||||||
|
assert claim is not None
|
||||||
|
running = queue.start(job.job_id, claim_token=claim.claim_token)
|
||||||
|
return queue, running, claim.claim_token
|
||||||
|
|
||||||
|
|
||||||
|
def _sealed_worker_job(expected: M49PortableSourceIdentity) -> SealedObservatoryRecordedJob:
|
||||||
|
definition = _m49_definition()
|
||||||
|
return SealedObservatoryRecordedJob(
|
||||||
|
job_id=f"observatory-run-{'a' * 32}",
|
||||||
|
request_sha256="2" * 64,
|
||||||
|
identity_sha256="3" * 64,
|
||||||
|
submission_receipt_sha256="6" * 64,
|
||||||
|
source_session_id=SESSION_ID,
|
||||||
|
source_catalog_sha256=expected.source_catalog_sha256,
|
||||||
|
source_bundle_sha256=expected.source_bundle_sha256,
|
||||||
|
source_capability_manifest_sha256=(expected.source_capability_manifest_sha256),
|
||||||
|
source_adapter_id=definition.source_adapter.adapter_id,
|
||||||
|
source_adapter_version=definition.source_adapter.version,
|
||||||
|
source_adapter_sha256=expected.source_adapter_sha256,
|
||||||
|
setup_id=definition.setup_id,
|
||||||
|
definition_id=definition.definition_id,
|
||||||
|
definition_version=definition.version,
|
||||||
|
definition_sha256=definition.definition_sha256,
|
||||||
|
executor_release_id="m49-tgs-portable-executor-v1",
|
||||||
|
executor_identity=ObservatoryWorkerExecutorIdentity(
|
||||||
|
release_sha256="4" * 64,
|
||||||
|
image_sha256="5" * 64,
|
||||||
|
model_manifest_sha256=definition.model_manifest_sha256,
|
||||||
|
resource_profile_sha256=definition.resource_profile.profile_sha256,
|
||||||
|
),
|
||||||
|
model_release_ids=(),
|
||||||
|
resource_profile_id=definition.resource_profile.profile_id,
|
||||||
|
checkpoint_policy=definition.resource_profile.checkpoint_policy,
|
||||||
|
allowed_checkpoints=definition.resource_profile.allowed_checkpoints,
|
||||||
|
claim_generation=1,
|
||||||
|
claim_claimed_at_utc=NOW,
|
||||||
|
claim_expires_at_utc="2026-08-31T09:05:00.000Z",
|
||||||
|
claim_heartbeat_at_utc=NOW,
|
||||||
|
claim_renewal_count=0,
|
||||||
|
restart_from_zero=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _seal_running_job(job: ObservatoryRecordedJob) -> SealedObservatoryRecordedJob:
|
||||||
|
return SealedObservatoryRecordedJob(
|
||||||
|
job_id=job.job_id,
|
||||||
|
request_sha256=job.request_sha256,
|
||||||
|
identity_sha256=job.identity_sha256,
|
||||||
|
submission_receipt_sha256=job.submission_receipt_sha256,
|
||||||
|
source_session_id=job.source_session_id,
|
||||||
|
source_catalog_sha256=job.source_catalog_sha256,
|
||||||
|
source_bundle_sha256=job.source_bundle_sha256,
|
||||||
|
source_capability_manifest_sha256=job.source_capability_manifest_sha256,
|
||||||
|
source_adapter_id=job.source_adapter_id,
|
||||||
|
source_adapter_version=job.source_adapter_version,
|
||||||
|
source_adapter_sha256=job.source_adapter_sha256,
|
||||||
|
setup_id=job.setup_id,
|
||||||
|
definition_id=job.definition_id,
|
||||||
|
definition_version=job.definition_version,
|
||||||
|
definition_sha256=job.definition_sha256,
|
||||||
|
executor_release_id=job.executor_release_id,
|
||||||
|
executor_identity=ObservatoryWorkerExecutorIdentity(
|
||||||
|
release_sha256=job.executor_release_sha256,
|
||||||
|
image_sha256=job.executor_image_sha256,
|
||||||
|
model_manifest_sha256=job.model_manifest_sha256,
|
||||||
|
resource_profile_sha256=job.resource_profile_sha256,
|
||||||
|
),
|
||||||
|
model_release_ids=job.model_release_ids,
|
||||||
|
resource_profile_id=job.resource_profile_id,
|
||||||
|
checkpoint_policy=job.checkpoint_policy,
|
||||||
|
allowed_checkpoints=job.allowed_checkpoints,
|
||||||
|
claim_generation=job.claim_generation,
|
||||||
|
claim_claimed_at_utc=job.claimed_at_utc,
|
||||||
|
claim_expires_at_utc=job.claim_expires_at_utc,
|
||||||
|
claim_heartbeat_at_utc=job.claim_heartbeat_at_utc,
|
||||||
|
claim_renewal_count=job.claim_renewal_count,
|
||||||
|
restart_from_zero=job.restart_from_zero,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_source_member(
|
||||||
|
job: SealedObservatoryRecordedJob,
|
||||||
|
*,
|
||||||
|
kind: str,
|
||||||
|
payload: bytes,
|
||||||
|
media_type: str,
|
||||||
|
artifact_id: str | None = None,
|
||||||
|
primary: bool = False,
|
||||||
|
camera_epoch: int | None = None,
|
||||||
|
camera_sequence: int | None = None,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
sha256 = hashlib.sha256(payload).hexdigest()
|
||||||
|
identity = {
|
||||||
|
"job_identity_sha256": job.identity_sha256,
|
||||||
|
"source_bundle_sha256": job.source_bundle_sha256,
|
||||||
|
"kind": kind,
|
||||||
|
"artifact_id": artifact_id,
|
||||||
|
"primary": primary,
|
||||||
|
"camera_epoch": camera_epoch,
|
||||||
|
"camera_sequence": camera_sequence,
|
||||||
|
"media_type": media_type,
|
||||||
|
"byte_length": len(payload),
|
||||||
|
"sha256": sha256,
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"member_id": hashlib.sha256(canonical_json(identity)).hexdigest(),
|
||||||
|
"kind": kind,
|
||||||
|
"media_type": media_type,
|
||||||
|
"byte_length": len(payload),
|
||||||
|
"sha256": sha256,
|
||||||
|
"artifact_id": artifact_id,
|
||||||
|
"primary": primary,
|
||||||
|
"camera_epoch": camera_epoch,
|
||||||
|
"camera_sequence": camera_sequence,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _runner_outputs(stage_root: Path, tmp_path: Path) -> tuple[Path, Path]:
|
||||||
|
stage = validate_m49_portable_source_stage(stage_root)
|
||||||
|
rows = read_m49_source_index(
|
||||||
|
stage.root / "sequence-index.ndjson",
|
||||||
|
expected_frame_count=stage.timeline_frame_count,
|
||||||
|
)
|
||||||
|
output_root = tmp_path / "runner-outputs"
|
||||||
|
output_root.mkdir()
|
||||||
|
timing = [
|
||||||
|
"timeline_frame_index\tsource_frame_index\tsession_seconds\tsample_available"
|
||||||
|
"\tavailable_slot\tinput_points\tground_points\tnonground_points\ttgs_ms"
|
||||||
|
"\tstage_wall_ms"
|
||||||
|
]
|
||||||
|
for row in rows:
|
||||||
|
index = cast(int, row["timeline_frame_index"])
|
||||||
|
if row["sample_available"] is not True:
|
||||||
|
timing.append(
|
||||||
|
f"{index}\t{row['source_frame_index']}\t"
|
||||||
|
f"{cast(float, row['session_seconds']):.6f}"
|
||||||
|
"\t0\t-1\t0\t0\t0\t0.000000\t0.010000"
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
source = np.fromfile(stage.root / cast(str, row["relative_path"]), dtype="<f4").reshape(
|
||||||
|
-1, 4
|
||||||
|
)
|
||||||
|
ranges = np.linalg.norm(source[:, :2].astype(np.float64), axis=1)
|
||||||
|
eligible = source[(ranges > 1.0) & (ranges < 80.0)]
|
||||||
|
ground = eligible[:1]
|
||||||
|
nonground = eligible[1:2]
|
||||||
|
(output_root / f"{index}_ground.bin").write_bytes(ground.tobytes())
|
||||||
|
(output_root / f"{index}_nonground.bin").write_bytes(nonground.tobytes())
|
||||||
|
timing.append(
|
||||||
|
f"{index}\t{row['source_frame_index']}\t"
|
||||||
|
f"{cast(float, row['session_seconds']):.6f}"
|
||||||
|
f"\t1\t{row['available_slot']}\t{source.shape[0]}\t{ground.shape[0]}"
|
||||||
|
f"\t{nonground.shape[0]}\t1.250000\t1.500000"
|
||||||
|
)
|
||||||
|
timing_path = tmp_path / "tgs-timing.tsv"
|
||||||
|
timing_path.write_text("\n".join(timing) + "\n", encoding="utf-8")
|
||||||
|
return output_root, timing_path
|
||||||
|
|
||||||
|
|
||||||
|
def test_dynamic_source_materializer_is_source_derived_and_tamper_evident(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
stage_root, _expected = _materialized_source(tmp_path, monkeypatch)
|
||||||
|
stage = validate_m49_portable_source_stage(stage_root)
|
||||||
|
assert stage.timeline_frame_count == 3
|
||||||
|
assert stage.available_lidar_frame_count == 2
|
||||||
|
rows = read_m49_source_index(stage.root / "sequence-index.ndjson", expected_frame_count=3)
|
||||||
|
assert [row["sample_available"] for row in rows] == [False, True, True]
|
||||||
|
assert [row["source_frame_index"] for row in rows] == [0, 1, 2]
|
||||||
|
assert all(
|
||||||
|
"RAVNOVES" not in path.read_text(errors="ignore")
|
||||||
|
for path in (
|
||||||
|
stage.root / "manifest.json",
|
||||||
|
stage.root / "sequence-index.ndjson",
|
||||||
|
stage.root / "schedule.tsv",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
schedule = stage.root / "schedule.tsv"
|
||||||
|
schedule.write_text(schedule.read_text() + "0\t0\t0\t-1\t0\n", encoding="utf-8")
|
||||||
|
with pytest.raises(M49PortableSourceError):
|
||||||
|
validate_m49_portable_source_stage(stage.root)
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_materializer_rejects_unadmitted_adjacent_metadata(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
with pytest.raises(M49PortableSourceError, match="admitted raw and host-time"):
|
||||||
|
_materialized_source(
|
||||||
|
tmp_path,
|
||||||
|
monkeypatch,
|
||||||
|
include_metadata_member=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_fixed_worker_stage_consumer_requires_manifested_metadata_member(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
direct_root = tmp_path / "direct"
|
||||||
|
direct_root.mkdir()
|
||||||
|
_direct_stage, expected = _materialized_source(direct_root, monkeypatch)
|
||||||
|
job = _sealed_worker_job(expected)
|
||||||
|
worker_root = tmp_path / "worker-source"
|
||||||
|
(worker_root / "camera" / "epoch-1" / "segments").mkdir(parents=True)
|
||||||
|
bundle = (direct_root / "source-bundle.json").read_bytes()
|
||||||
|
capability = (direct_root / "source-capability.json").read_bytes()
|
||||||
|
init = b"exact-init"
|
||||||
|
segments = (b"segment-1", b"segment-2", b"segment-3")
|
||||||
|
(worker_root / "source-bundle.json").write_bytes(bundle)
|
||||||
|
(worker_root / "source-capability.json").write_bytes(capability)
|
||||||
|
(worker_root / "mqtt.raw.k1mqtt").write_bytes(RAW_PAYLOAD)
|
||||||
|
(worker_root / "mqtt.metadata.jsonl").write_bytes(METADATA_PAYLOAD)
|
||||||
|
(worker_root / "camera" / "epoch-1" / "init.mp4").write_bytes(init)
|
||||||
|
for index, payload in enumerate(segments, start=1):
|
||||||
|
(worker_root / "camera" / "epoch-1" / "segments" / f"{index}.m4s").write_bytes(payload)
|
||||||
|
members = [
|
||||||
|
_worker_source_member(
|
||||||
|
job,
|
||||||
|
kind="source-bundle",
|
||||||
|
payload=bundle,
|
||||||
|
media_type="application/json",
|
||||||
|
),
|
||||||
|
_worker_source_member(
|
||||||
|
job,
|
||||||
|
kind="source-capability",
|
||||||
|
payload=capability,
|
||||||
|
media_type="application/json",
|
||||||
|
),
|
||||||
|
_worker_source_member(
|
||||||
|
job,
|
||||||
|
kind="spatial-replay",
|
||||||
|
payload=RAW_PAYLOAD,
|
||||||
|
media_type="application/x-nodedc-k1mqtt",
|
||||||
|
artifact_id="raw-transport-primary",
|
||||||
|
primary=True,
|
||||||
|
),
|
||||||
|
_worker_source_member(
|
||||||
|
job,
|
||||||
|
kind="spatial-replay-metadata",
|
||||||
|
payload=METADATA_PAYLOAD,
|
||||||
|
media_type="application/x-ndjson",
|
||||||
|
artifact_id="raw-transport-index",
|
||||||
|
),
|
||||||
|
_worker_source_member(
|
||||||
|
job,
|
||||||
|
kind="camera-init",
|
||||||
|
payload=init,
|
||||||
|
media_type="video/mp4",
|
||||||
|
artifact_id="camera-primary",
|
||||||
|
camera_epoch=1,
|
||||||
|
),
|
||||||
|
*[
|
||||||
|
_worker_source_member(
|
||||||
|
job,
|
||||||
|
kind="camera-segment",
|
||||||
|
payload=payload,
|
||||||
|
media_type="video/iso.segment",
|
||||||
|
artifact_id="camera-primary",
|
||||||
|
camera_epoch=1,
|
||||||
|
camera_sequence=index,
|
||||||
|
)
|
||||||
|
for index, payload in enumerate(segments, start=1)
|
||||||
|
],
|
||||||
|
]
|
||||||
|
members.sort(key=lambda row: cast(str, row["member_id"]))
|
||||||
|
materialization = {
|
||||||
|
"schema_version": "missioncore.observatory-portable-source-materialization/v1",
|
||||||
|
"job_id": job.job_id,
|
||||||
|
"job_identity_sha256": job.identity_sha256,
|
||||||
|
"claim_generation": job.claim_generation,
|
||||||
|
"source": {
|
||||||
|
"session_id": job.source_session_id,
|
||||||
|
"bundle_sha256": job.source_bundle_sha256,
|
||||||
|
"capability_manifest_sha256": job.source_capability_manifest_sha256,
|
||||||
|
},
|
||||||
|
"members": members,
|
||||||
|
"authority": AUTHORITY,
|
||||||
|
}
|
||||||
|
(worker_root / "materialization-manifest.json").write_bytes(canonical_json(materialization))
|
||||||
|
stage = PortableWorkerSourceStage(
|
||||||
|
root=worker_root,
|
||||||
|
source_bundle_sha256=job.source_bundle_sha256,
|
||||||
|
source_capability_manifest_sha256=job.source_capability_manifest_sha256,
|
||||||
|
source_adapter_sha256=job.source_adapter_sha256,
|
||||||
|
)
|
||||||
|
built_from: list[Path] = []
|
||||||
|
|
||||||
|
def fake_build(capture: Path, output: Path, *, session_id: str) -> Path:
|
||||||
|
assert session_id == SESSION_ID
|
||||||
|
assert capture == worker_root / "mqtt.raw.k1mqtt"
|
||||||
|
assert output.name == "lidar-replay-packs"
|
||||||
|
built_from.append(capture)
|
||||||
|
return worker_root
|
||||||
|
|
||||||
|
monkeypatch.setattr(source_module, "build_lidar_replay_pack_v2", fake_build)
|
||||||
|
|
||||||
|
class _DeliveredSource:
|
||||||
|
def materialize(self, requested: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage:
|
||||||
|
assert requested == job
|
||||||
|
return stage
|
||||||
|
|
||||||
|
materializer = M49PortableSourceMaterializerAdapter(
|
||||||
|
upstream=_DeliveredSource(),
|
||||||
|
profile_path=PROFILE_PATH,
|
||||||
|
output_parent=tmp_path / "worker-output",
|
||||||
|
)
|
||||||
|
materialized = materializer.materialize(job)
|
||||||
|
assert built_from == [worker_root / "mqtt.raw.k1mqtt"]
|
||||||
|
assert isinstance(materialized, M49PortableBoundSourceStage)
|
||||||
|
assert materialized.m49_stage.timeline_frame_count == 3
|
||||||
|
assert materialized.m49_stage.available_lidar_frame_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_result_v2_assembler_and_exact_validator_round_trip(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
stage_root, expected = _materialized_source(tmp_path, monkeypatch)
|
||||||
|
registry = _ready_m49_registry(tmp_path)
|
||||||
|
definition = registry.resolve_setup("m49-tgs-portable-v2")
|
||||||
|
queue, running, claim_token = _running_job(tmp_path, definition, expected)
|
||||||
|
sealed = _seal_running_job(running)
|
||||||
|
prepared = tmp_path / "prepared-runner"
|
||||||
|
prepared.mkdir()
|
||||||
|
output_root, timing_path = _runner_outputs(stage_root, prepared)
|
||||||
|
binary = tmp_path / "run_m49_tgs_portable"
|
||||||
|
binary.write_bytes(b"\x7fELFtest-only")
|
||||||
|
binary.chmod(0o755)
|
||||||
|
build_seal = builder.seal_m49_compiled_runner_build(
|
||||||
|
source_root=REPOSITORY_ROOT,
|
||||||
|
source_revision="f" * 40,
|
||||||
|
binary_path=binary,
|
||||||
|
manifest_path=tmp_path / "runner-build-seal.json",
|
||||||
|
)
|
||||||
|
installation = M49PortableRunnerInstallation(
|
||||||
|
profile_path=PROFILE_PATH,
|
||||||
|
runner_binary_path=binary,
|
||||||
|
runner_build_seal_path=build_seal.manifest_path,
|
||||||
|
runner_build_seal_sha256=build_seal.manifest_sha256,
|
||||||
|
output_parent=tmp_path / "executor-output",
|
||||||
|
)
|
||||||
|
|
||||||
|
def fake_invoke(
|
||||||
|
*,
|
||||||
|
binary: Path,
|
||||||
|
sequence: Path,
|
||||||
|
schedule: Path,
|
||||||
|
output: Path,
|
||||||
|
timing: Path,
|
||||||
|
workspace: Path,
|
||||||
|
timeout_seconds: int,
|
||||||
|
) -> None:
|
||||||
|
assert binary == installation.runner_binary_path
|
||||||
|
assert sequence == Path(stage_root) / "tgs/sequence/velodyne"
|
||||||
|
assert schedule == Path(stage_root) / "schedule.tsv"
|
||||||
|
assert workspace.parent == installation.output_parent
|
||||||
|
assert timeout_seconds == installation.timeout_seconds
|
||||||
|
shutil.copytree(output_root, output)
|
||||||
|
shutil.copyfile(timing_path, timing)
|
||||||
|
|
||||||
|
source_stage = validate_m49_portable_source_stage(stage_root)
|
||||||
|
bound_source = M49PortableBoundSourceStage(
|
||||||
|
root=source_stage.root,
|
||||||
|
source_bundle_sha256=sealed.source_bundle_sha256,
|
||||||
|
source_capability_manifest_sha256=sealed.source_capability_manifest_sha256,
|
||||||
|
source_adapter_sha256=sealed.source_adapter_sha256,
|
||||||
|
job=sealed,
|
||||||
|
m49_stage=source_stage,
|
||||||
|
)
|
||||||
|
runner = M49PortableProfileRunnerAdapter(
|
||||||
|
definition=definition,
|
||||||
|
installation=installation,
|
||||||
|
created_at_utc=lambda: NOW,
|
||||||
|
invoker=fake_invoke,
|
||||||
|
)
|
||||||
|
draft = runner.run(
|
||||||
|
PortableWorkerRuntimePlan(
|
||||||
|
job_id=sealed.job_id,
|
||||||
|
adapter_id="m49-tgs-worker006-portable-v2",
|
||||||
|
candidate_sha256="f" * 64,
|
||||||
|
setup_id=sealed.setup_id,
|
||||||
|
definition_sha256=sealed.definition_sha256,
|
||||||
|
source_bundle_sha256=sealed.source_bundle_sha256,
|
||||||
|
source_capability_manifest_sha256=sealed.source_capability_manifest_sha256,
|
||||||
|
result_contract_sha256=definition.result_contract.contract_sha256,
|
||||||
|
phases=M49_PORTABLE_RUNTIME_PHASES,
|
||||||
|
),
|
||||||
|
bound_source,
|
||||||
|
)
|
||||||
|
package = PortableResultPackageManifest.from_bytes((draft.root / "manifest.json").read_bytes())
|
||||||
|
assert draft.root.name == package.manifest_sha256
|
||||||
|
assert draft.result_id.startswith("m49-tgs-portable-review-")
|
||||||
|
succeeded = queue.succeed(
|
||||||
|
running.job_id,
|
||||||
|
claim_token=claim_token,
|
||||||
|
result_id=draft.result_id,
|
||||||
|
result_sha256=draft.result_sha256,
|
||||||
|
)
|
||||||
|
artifact_paths = {
|
||||||
|
artifact.role: draft.root.joinpath(*Path(artifact.relative_path).parts)
|
||||||
|
for artifact in package.artifacts
|
||||||
|
}
|
||||||
|
result_document = cast(
|
||||||
|
dict[str, object],
|
||||||
|
json.loads(artifact_paths["result-document"].read_bytes()),
|
||||||
|
)
|
||||||
|
context = PortableResultValidationContext(
|
||||||
|
manifest=package,
|
||||||
|
job=succeeded,
|
||||||
|
definition=definition,
|
||||||
|
result_document=result_document,
|
||||||
|
artifact_paths=artifact_paths,
|
||||||
|
)
|
||||||
|
validate_m49_portable_result(context)
|
||||||
|
|
||||||
|
states = np.load(artifact_paths["costmap-states"], mmap_mode="r", allow_pickle=False)
|
||||||
|
assert np.all(states[0] == 0)
|
||||||
|
assert set(np.unique(states[1])).issubset({0, 1, 2, 3})
|
||||||
|
assert 3 in states[1]
|
||||||
|
point_accounting = cast(dict[str, object], result_document["point_accounting"])
|
||||||
|
assert point_accounting["unaccounted"] == 0
|
||||||
|
|
||||||
|
drifted = copy.deepcopy(result_document)
|
||||||
|
cast(dict[str, object], drifted["point_accounting"])["eligible"] = 999
|
||||||
|
with pytest.raises(M49PortableResultError):
|
||||||
|
validate_m49_portable_result(
|
||||||
|
PortableResultValidationContext(
|
||||||
|
manifest=package,
|
||||||
|
job=succeeded,
|
||||||
|
definition=definition,
|
||||||
|
result_document=drifted,
|
||||||
|
artifact_paths=artifact_paths,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_executor_release_candidate_is_deterministic_blocked_and_tamper_evident(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
source_root = tmp_path / "source"
|
||||||
|
for relative in builder.M49_RELEASE_SOURCES:
|
||||||
|
target = source_root / relative
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copyfile(REPOSITORY_ROOT / relative, target)
|
||||||
|
first = builder.build_m49_executor_release_candidate(
|
||||||
|
source_root=source_root,
|
||||||
|
output_directory=tmp_path / "out-a",
|
||||||
|
source_revision="f" * 40,
|
||||||
|
source_state="uncommitted-candidate",
|
||||||
|
)
|
||||||
|
second = builder.build_m49_executor_release_candidate(
|
||||||
|
source_root=source_root,
|
||||||
|
output_directory=tmp_path / "out-b",
|
||||||
|
source_revision="f" * 40,
|
||||||
|
source_state="uncommitted-candidate",
|
||||||
|
)
|
||||||
|
assert first.archive.read_bytes() == second.archive.read_bytes()
|
||||||
|
assert first.archive_sha256 == second.archive_sha256
|
||||||
|
assert first.manifest["state"] == "blocked"
|
||||||
|
assert tuple(first.manifest["blockers"]) == builder.M49_RELEASE_BLOCKERS
|
||||||
|
assert first.manifest["executor_image_sha256"] is None
|
||||||
|
assert first.manifest["compiled_runner"] is None
|
||||||
|
assert builder.verify_m49_executor_release_candidate(first.archive) == first
|
||||||
|
|
||||||
|
dockerfile = DOCKERFILE_PATH.read_text(encoding="utf-8")
|
||||||
|
assert "FROM ndc/mission-core-m49-t3-travel:20260826" in dockerfile
|
||||||
|
assert builder.M49_TRAVEL_IMAGE_SHA256 in dockerfile
|
||||||
|
assert "--network" not in dockerfile
|
||||||
|
assert "com.nodedc.authority=\"observation-only\"" in dockerfile
|
||||||
|
installer = INSTALLER_PATH.read_text(encoding="utf-8")
|
||||||
|
assert "--pull=false --no-cache --network none" in installer
|
||||||
|
assert "--network none --read-only" in installer
|
||||||
|
assert "committed-source-snapshot-missing" in installer
|
||||||
|
assert "MissionCore-M49TgsFullShadow" in installer
|
||||||
|
|
||||||
|
changed = source_root / "src" / "k1link" / "observatory" / "m49_portable_result.py"
|
||||||
|
changed.write_bytes(changed.read_bytes() + b"\n")
|
||||||
|
drifted = builder.build_m49_executor_release_candidate(
|
||||||
|
source_root=source_root,
|
||||||
|
output_directory=tmp_path / "out-c",
|
||||||
|
source_revision="f" * 40,
|
||||||
|
source_state="uncommitted-candidate",
|
||||||
|
)
|
||||||
|
assert drifted.candidate_sha256 != first.candidate_sha256
|
||||||
@@ -0,0 +1,600 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import cast
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from k1link.observatory.portable_artifact_transport import (
|
||||||
|
PortableArtifactTransportIntegrityError,
|
||||||
|
PortableArtifactTransportUnavailableError,
|
||||||
|
PortableObservatoryArtifactTransport,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_result_contract import (
|
||||||
|
OBSERVATION_ONLY_AUTHORITY,
|
||||||
|
RESULT_DOCUMENT_ROLE,
|
||||||
|
PortableResultArtifact,
|
||||||
|
PortableResultPackageManifest,
|
||||||
|
canonical_json,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_run_definitions import (
|
||||||
|
PortableRunDefinition,
|
||||||
|
PortableRunDefinitionRegistry,
|
||||||
|
canonical_sha256,
|
||||||
|
)
|
||||||
|
from k1link.observatory.recorded_jobs import (
|
||||||
|
ObservatoryRecordedJob,
|
||||||
|
ObservatoryRecordedJobIntent,
|
||||||
|
ObservatoryRecordedJobQueue,
|
||||||
|
ObservatoryRecordedQueueStaleClaimError,
|
||||||
|
RecordedRunDefinitionRegistry,
|
||||||
|
)
|
||||||
|
from k1link.observatory.source_admission import (
|
||||||
|
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||||
|
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||||
|
PORTABLE_SOURCE_DOCUMENT_DIRECTORY,
|
||||||
|
)
|
||||||
|
from k1link.sessions.media import (
|
||||||
|
RecordedMediaEpoch,
|
||||||
|
RecordedMediaManifest,
|
||||||
|
RecordedMediaSegment,
|
||||||
|
)
|
||||||
|
from k1link.sessions.models import (
|
||||||
|
RecordedMediaArtifact,
|
||||||
|
ReplayArtifact,
|
||||||
|
ReplayCommand,
|
||||||
|
SessionDetail,
|
||||||
|
SessionSummary,
|
||||||
|
)
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||||
|
NOW = "2026-08-31T10:00:00.000Z"
|
||||||
|
SOURCE_SESSION_ID = "20260831T095500Z_viewer_live"
|
||||||
|
RESULT_ID = "portable-result-transport-001"
|
||||||
|
|
||||||
|
|
||||||
|
def _ready_registry(tmp_path: Path) -> PortableRunDefinitionRegistry:
|
||||||
|
registry = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||||
|
base = registry.definitions[0]
|
||||||
|
document = cast(
|
||||||
|
dict[str, object], json.loads(REGISTRY_PATH.read_text(encoding="utf-8"))
|
||||||
|
)
|
||||||
|
rows = cast(list[object], document["definitions"])
|
||||||
|
selected = copy.deepcopy(cast(dict[str, object], rows[0]))
|
||||||
|
selected["executor"] = {
|
||||||
|
"contour_id": "worker-006",
|
||||||
|
"state": "ready",
|
||||||
|
"release_id": "portable-transport-test-executor",
|
||||||
|
"release_sha256": "1" * 64,
|
||||||
|
"image_sha256": "2" * 64,
|
||||||
|
"reason_code": None,
|
||||||
|
"reason": None,
|
||||||
|
}
|
||||||
|
identity = copy.deepcopy(base.identity_document())
|
||||||
|
identity["executor"] = {
|
||||||
|
"contour_id": "worker-006",
|
||||||
|
"state": "ready",
|
||||||
|
"release_id": "portable-transport-test-executor",
|
||||||
|
"release_sha256": "1" * 64,
|
||||||
|
"image_sha256": "2" * 64,
|
||||||
|
}
|
||||||
|
selected["definition_sha256"] = canonical_sha256(identity)
|
||||||
|
path = tmp_path / "definitions.json"
|
||||||
|
path.write_bytes(
|
||||||
|
canonical_json(
|
||||||
|
{
|
||||||
|
"schema_version": document["schema_version"],
|
||||||
|
"definitions": [selected],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return PortableRunDefinitionRegistry.from_file(path)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _Store:
|
||||||
|
data_dir: Path
|
||||||
|
detail: SessionDetail
|
||||||
|
catalog_sha256: str
|
||||||
|
replay: ReplayCommand
|
||||||
|
recorded_media: tuple[RecordedMediaArtifact, ...]
|
||||||
|
|
||||||
|
def get_session_with_catalog_snapshot(
|
||||||
|
self, session_id: str
|
||||||
|
) -> tuple[SessionDetail, str]:
|
||||||
|
assert session_id == SOURCE_SESSION_ID
|
||||||
|
return self.detail, self.catalog_sha256
|
||||||
|
|
||||||
|
def prepare_replay(self, session_id: str) -> ReplayCommand:
|
||||||
|
assert session_id == SOURCE_SESSION_ID
|
||||||
|
return self.replay
|
||||||
|
|
||||||
|
def list_recorded_media(
|
||||||
|
self, session_id: str
|
||||||
|
) -> tuple[RecordedMediaArtifact, ...]:
|
||||||
|
assert session_id == SOURCE_SESSION_ID
|
||||||
|
return self.recorded_media
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _Inspector:
|
||||||
|
manifest: RecordedMediaManifest
|
||||||
|
|
||||||
|
def restore_prepared(
|
||||||
|
self,
|
||||||
|
artifact: RecordedMediaArtifact,
|
||||||
|
replay: ReplayCommand,
|
||||||
|
) -> RecordedMediaManifest:
|
||||||
|
assert artifact.artifact_id == self.manifest.artifact_id
|
||||||
|
assert replay.session_id == self.manifest.session_id
|
||||||
|
return self.manifest
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _Fixture:
|
||||||
|
service: PortableObservatoryArtifactTransport
|
||||||
|
queue: ObservatoryRecordedJobQueue
|
||||||
|
definition: PortableRunDefinition
|
||||||
|
job: ObservatoryRecordedJob
|
||||||
|
claim_token: str
|
||||||
|
raw_path: Path
|
||||||
|
|
||||||
|
|
||||||
|
def _fixture(tmp_path: Path) -> _Fixture:
|
||||||
|
registry = _ready_registry(tmp_path)
|
||||||
|
definition = registry.definitions[0]
|
||||||
|
data_dir = tmp_path / "data"
|
||||||
|
source_root = tmp_path / "source"
|
||||||
|
raw_path = source_root / "mqtt.raw.k1mqtt"
|
||||||
|
metadata_path = source_root / "mqtt.metadata.jsonl"
|
||||||
|
init_path = source_root / "camera" / "epoch-1" / "init.mp4"
|
||||||
|
segment_path = source_root / "camera" / "epoch-1" / "segments" / "1.m4s"
|
||||||
|
segment_path.parent.mkdir(parents=True)
|
||||||
|
raw_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
raw_path.write_bytes(b"sealed-raw-replay")
|
||||||
|
metadata_path.write_bytes(b'{"offset":0,"topic":"/points"}\n')
|
||||||
|
init_path.write_bytes(b"sealed-init")
|
||||||
|
segment_path.write_bytes(b"sealed-segment")
|
||||||
|
raw_sha = hashlib.sha256(raw_path.read_bytes()).hexdigest()
|
||||||
|
metadata_sha = hashlib.sha256(metadata_path.read_bytes()).hexdigest()
|
||||||
|
init_sha = hashlib.sha256(init_path.read_bytes()).hexdigest()
|
||||||
|
segment_sha = hashlib.sha256(segment_path.read_bytes()).hexdigest()
|
||||||
|
catalog_sha = "3" * 64
|
||||||
|
generation_sha = "4" * 64
|
||||||
|
replay = ReplayCommand(
|
||||||
|
session_id=SOURCE_SESSION_ID,
|
||||||
|
plugin_id=definition.source_requirements.plugin_id,
|
||||||
|
allowed_root=source_root,
|
||||||
|
session_root=source_root,
|
||||||
|
primary_artifact_id="raw-transport-primary",
|
||||||
|
artifacts=(
|
||||||
|
ReplayArtifact(
|
||||||
|
artifact_id="raw-transport-primary",
|
||||||
|
path=raw_path,
|
||||||
|
media_type="application/x-nodedc-k1mqtt",
|
||||||
|
file_byte_length=raw_path.stat().st_size,
|
||||||
|
replay_byte_length=raw_path.stat().st_size,
|
||||||
|
expected_sha256=raw_sha,
|
||||||
|
),
|
||||||
|
ReplayArtifact(
|
||||||
|
artifact_id="raw-transport-index",
|
||||||
|
path=metadata_path,
|
||||||
|
media_type="application/x-ndjson",
|
||||||
|
file_byte_length=metadata_path.stat().st_size,
|
||||||
|
replay_byte_length=metadata_path.stat().st_size,
|
||||||
|
expected_sha256=None,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
timeline_origin_epoch_ns=1,
|
||||||
|
timeline_origin_monotonic_ns=2,
|
||||||
|
speed=1.0,
|
||||||
|
loop=False,
|
||||||
|
)
|
||||||
|
recorded_media = RecordedMediaArtifact(
|
||||||
|
session_id=SOURCE_SESSION_ID,
|
||||||
|
public_source_id="recorded.camera.right",
|
||||||
|
artifact_id="recorded-video-right",
|
||||||
|
source_path=source_root / "camera",
|
||||||
|
byte_length=init_path.stat().st_size + segment_path.stat().st_size,
|
||||||
|
)
|
||||||
|
epoch = RecordedMediaEpoch(
|
||||||
|
ordinal=1,
|
||||||
|
path=init_path.parent,
|
||||||
|
init_path=init_path,
|
||||||
|
init_byte_length=init_path.stat().st_size,
|
||||||
|
init_sha256=init_sha,
|
||||||
|
media_type='video/mp4; codecs="avc1.641028"',
|
||||||
|
timeline_start_seconds=0.0,
|
||||||
|
timeline_end_seconds=0.1,
|
||||||
|
segments=(
|
||||||
|
RecordedMediaSegment(
|
||||||
|
sequence=1,
|
||||||
|
path=segment_path,
|
||||||
|
byte_length=segment_path.stat().st_size,
|
||||||
|
sha256=segment_sha,
|
||||||
|
random_access=True,
|
||||||
|
end_time_seconds=0.1,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
manifest = RecordedMediaManifest(
|
||||||
|
session_id=SOURCE_SESSION_ID,
|
||||||
|
public_source_id=recorded_media.public_source_id,
|
||||||
|
artifact_id=recorded_media.artifact_id,
|
||||||
|
synchronization="host-arrival-best-effort",
|
||||||
|
generation_sha256=generation_sha,
|
||||||
|
timeline_start_seconds=0.0,
|
||||||
|
timeline_end_seconds=0.1,
|
||||||
|
byte_length=recorded_media.byte_length,
|
||||||
|
epochs=(epoch,),
|
||||||
|
)
|
||||||
|
detail = SessionDetail(
|
||||||
|
summary=SessionSummary(
|
||||||
|
session_id=SOURCE_SESSION_ID,
|
||||||
|
display_name="Portable source",
|
||||||
|
status="ready",
|
||||||
|
started_at_utc=NOW,
|
||||||
|
completed_at_utc=NOW,
|
||||||
|
duration_seconds=0.1,
|
||||||
|
modalities=("point-cloud", "trajectory", "video"),
|
||||||
|
source_count=3,
|
||||||
|
total_bytes=100,
|
||||||
|
replayable=True,
|
||||||
|
origin=definition.source_requirements.archive_id,
|
||||||
|
),
|
||||||
|
sources=(),
|
||||||
|
artifacts=(),
|
||||||
|
plugin_id=definition.source_requirements.plugin_id,
|
||||||
|
archive_id=definition.source_requirements.archive_id,
|
||||||
|
)
|
||||||
|
source_adapter = {
|
||||||
|
"id": definition.source_adapter.adapter_id,
|
||||||
|
"version": definition.source_adapter.version,
|
||||||
|
"sha256": definition.source_adapter.contract_sha256,
|
||||||
|
}
|
||||||
|
bundle = {
|
||||||
|
"schema_version": PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||||
|
"source_session_id": SOURCE_SESSION_ID,
|
||||||
|
"source_catalog_sha256": catalog_sha,
|
||||||
|
"plugin_id": definition.source_requirements.plugin_id,
|
||||||
|
"archive_id": definition.source_requirements.archive_id,
|
||||||
|
"source_adapter": source_adapter,
|
||||||
|
"sources": [],
|
||||||
|
"spatial_replay": {
|
||||||
|
"primary_artifact_id": replay.primary_artifact_id,
|
||||||
|
"members": [
|
||||||
|
{
|
||||||
|
"artifact_id": "raw-transport-primary",
|
||||||
|
"media_type": "application/x-nodedc-k1mqtt",
|
||||||
|
"byte_length": raw_path.stat().st_size,
|
||||||
|
"replay_byte_length": raw_path.stat().st_size,
|
||||||
|
"sha256": raw_sha,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"artifact_id": "raw-transport-index",
|
||||||
|
"media_type": "application/x-ndjson",
|
||||||
|
"byte_length": metadata_path.stat().st_size,
|
||||||
|
"replay_byte_length": metadata_path.stat().st_size,
|
||||||
|
"sha256": metadata_sha,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"timeline_origin_epoch_ns": 1,
|
||||||
|
"timeline_origin_monotonic_ns": 2,
|
||||||
|
},
|
||||||
|
"camera": {
|
||||||
|
"artifact_id": recorded_media.artifact_id,
|
||||||
|
"public_source_id": recorded_media.public_source_id,
|
||||||
|
"generation_sha256": generation_sha,
|
||||||
|
"synchronization": "host-arrival-best-effort",
|
||||||
|
"epoch": {
|
||||||
|
"ordinal": 1,
|
||||||
|
"media_type": epoch.media_type,
|
||||||
|
"init": {
|
||||||
|
"byte_length": epoch.init_byte_length,
|
||||||
|
"sha256": init_sha,
|
||||||
|
},
|
||||||
|
"timeline_start_seconds": 0.0,
|
||||||
|
"timeline_end_seconds": 0.1,
|
||||||
|
"segments": [
|
||||||
|
{
|
||||||
|
"sequence": 1,
|
||||||
|
"byte_length": segment_path.stat().st_size,
|
||||||
|
"sha256": segment_sha,
|
||||||
|
"random_access": True,
|
||||||
|
"end_time_seconds": 0.1,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"authority": OBSERVATION_ONLY_AUTHORITY,
|
||||||
|
}
|
||||||
|
bundle_payload = canonical_json(bundle)
|
||||||
|
bundle_sha = hashlib.sha256(bundle_payload).hexdigest()
|
||||||
|
capability = {
|
||||||
|
"schema_version": PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||||
|
"source_session_id": SOURCE_SESSION_ID,
|
||||||
|
"source_catalog_sha256": catalog_sha,
|
||||||
|
"source_bundle_sha256": bundle_sha,
|
||||||
|
"source_adapter_sha256": definition.source_adapter.contract_sha256,
|
||||||
|
"modalities": [],
|
||||||
|
"camera_profile": {},
|
||||||
|
"calibration": {},
|
||||||
|
"authority": OBSERVATION_ONLY_AUTHORITY,
|
||||||
|
}
|
||||||
|
capability_payload = canonical_json(capability)
|
||||||
|
capability_sha = hashlib.sha256(capability_payload).hexdigest()
|
||||||
|
documents = data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY
|
||||||
|
documents.mkdir(parents=True)
|
||||||
|
(documents / f"{bundle_sha}.json").write_bytes(bundle_payload)
|
||||||
|
(documents / f"{capability_sha}.json").write_bytes(capability_payload)
|
||||||
|
queue = ObservatoryRecordedJobQueue(
|
||||||
|
data_dir,
|
||||||
|
definitions=RecordedRunDefinitionRegistry(
|
||||||
|
(definition.to_recorded_run_definition(),)
|
||||||
|
),
|
||||||
|
clock=lambda: NOW,
|
||||||
|
)
|
||||||
|
job, created = queue.submit(
|
||||||
|
ObservatoryRecordedJobIntent(
|
||||||
|
idempotency_key="portable-artifact-transport-001",
|
||||||
|
source_session_id=SOURCE_SESSION_ID,
|
||||||
|
source_catalog_sha256=catalog_sha,
|
||||||
|
source_bundle_sha256=bundle_sha,
|
||||||
|
source_capability_manifest_sha256=capability_sha,
|
||||||
|
setup_id=definition.setup_id,
|
||||||
|
definition_sha256=definition.definition_sha256,
|
||||||
|
),
|
||||||
|
enqueue=True,
|
||||||
|
)
|
||||||
|
assert created
|
||||||
|
claim = queue.claim_next(
|
||||||
|
claimant_id="worker-006",
|
||||||
|
claim_request_id="portable-artifact-claim-001",
|
||||||
|
)
|
||||||
|
assert claim is not None
|
||||||
|
running = queue.start(job.job_id, claim_token=claim.claim_token)
|
||||||
|
store = _Store(
|
||||||
|
data_dir=data_dir,
|
||||||
|
detail=detail,
|
||||||
|
catalog_sha256=catalog_sha,
|
||||||
|
replay=replay,
|
||||||
|
recorded_media=(recorded_media,),
|
||||||
|
)
|
||||||
|
service = PortableObservatoryArtifactTransport(
|
||||||
|
queue=queue,
|
||||||
|
session_store=store, # type: ignore[arg-type]
|
||||||
|
media_inspector=_Inspector(manifest), # type: ignore[arg-type]
|
||||||
|
definitions=registry,
|
||||||
|
)
|
||||||
|
return _Fixture(service, queue, definition, running, claim.claim_token, raw_path)
|
||||||
|
|
||||||
|
|
||||||
|
def _result_package(
|
||||||
|
tmp_path: Path,
|
||||||
|
fixture: _Fixture,
|
||||||
|
) -> tuple[PortableResultPackageManifest, bytes]:
|
||||||
|
result_document = {
|
||||||
|
"schema_version": fixture.definition.result_contract.result_schema,
|
||||||
|
"result_id": RESULT_ID,
|
||||||
|
"result_kind": fixture.definition.result_contract.result_kind,
|
||||||
|
"authority": OBSERVATION_ONLY_AUTHORITY,
|
||||||
|
}
|
||||||
|
payload = canonical_json(result_document)
|
||||||
|
artifact = PortableResultArtifact(
|
||||||
|
role=RESULT_DOCUMENT_ROLE,
|
||||||
|
relative_path="artifacts/result.json",
|
||||||
|
media_type="application/json",
|
||||||
|
byte_length=len(payload),
|
||||||
|
sha256=hashlib.sha256(payload).hexdigest(),
|
||||||
|
)
|
||||||
|
package = PortableResultPackageManifest.create(
|
||||||
|
job=fixture.job,
|
||||||
|
definition=fixture.definition,
|
||||||
|
result_id=RESULT_ID,
|
||||||
|
created_at_utc=NOW,
|
||||||
|
artifacts=(artifact,),
|
||||||
|
)
|
||||||
|
return package, payload
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_members_are_claim_bound_and_materialized_through_cas(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
fixture = _fixture(tmp_path)
|
||||||
|
manifest = fixture.service.source_manifest(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert {member.kind for member in manifest.members} == {
|
||||||
|
"source-bundle",
|
||||||
|
"source-capability",
|
||||||
|
"spatial-replay",
|
||||||
|
"spatial-replay-metadata",
|
||||||
|
"camera-init",
|
||||||
|
"camera-segment",
|
||||||
|
}
|
||||||
|
assert "path" not in json.dumps(manifest.as_dict(), sort_keys=True)
|
||||||
|
raw = next(member for member in manifest.members if member.kind == "spatial-replay")
|
||||||
|
admitted, cas_path = fixture.service.materialize_source_member(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
member_id=raw.member_id,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert admitted.sha256 == raw.sha256
|
||||||
|
assert cas_path.read_bytes() == b"sealed-raw-replay"
|
||||||
|
assert cas_path != fixture.raw_path
|
||||||
|
assert cas_path.name == raw.sha256
|
||||||
|
metadata = next(
|
||||||
|
member
|
||||||
|
for member in manifest.members
|
||||||
|
if member.kind == "spatial-replay-metadata"
|
||||||
|
)
|
||||||
|
admitted_metadata, metadata_cas_path = fixture.service.materialize_source_member(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
member_id=metadata.member_id,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
assert admitted_metadata.artifact_id == "raw-transport-index"
|
||||||
|
assert metadata_cas_path.read_bytes() == b'{"offset":0,"topic":"/points"}\n'
|
||||||
|
assert metadata_cas_path.name == metadata.sha256
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_member_rejects_tampering_and_stale_generation(tmp_path: Path) -> None:
|
||||||
|
fixture = _fixture(tmp_path)
|
||||||
|
manifest = fixture.service.source_manifest(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
raw = next(member for member in manifest.members if member.kind == "spatial-replay")
|
||||||
|
fixture.raw_path.write_bytes(b"changed")
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
PortableArtifactTransportIntegrityError,
|
||||||
|
match="admitted regular file|changed",
|
||||||
|
):
|
||||||
|
fixture.service.materialize_source_member(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
member_id=raw.member_id,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueStaleClaimError):
|
||||||
|
fixture.service.source_manifest(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation + 1,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_result_upload_is_atomic_resumable_and_required_before_success(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
fixture = _fixture(tmp_path)
|
||||||
|
package, result_payload = _result_package(tmp_path, fixture)
|
||||||
|
plan = fixture.service.stage_result_manifest(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
result_sha256=package.manifest_sha256,
|
||||||
|
manifest_payload=package.canonical_bytes,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
assert plan.complete is False
|
||||||
|
with pytest.raises(PortableArtifactTransportUnavailableError, match="incomplete"):
|
||||||
|
fixture.service.complete_result_upload(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
result_sha256=package.manifest_sha256,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
|
||||||
|
async def chunks() -> object:
|
||||||
|
yield result_payload[:7]
|
||||||
|
yield result_payload[7:]
|
||||||
|
|
||||||
|
uploaded = asyncio.run(
|
||||||
|
fixture.service.upload_result_member(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
result_sha256=package.manifest_sha256,
|
||||||
|
member_id=plan.members[0].member_id,
|
||||||
|
chunks=chunks(), # type: ignore[arg-type]
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert uploaded.complete is True
|
||||||
|
repeated = fixture.service.stage_result_manifest(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
result_sha256=package.manifest_sha256,
|
||||||
|
manifest_payload=package.canonical_bytes,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
assert repeated.members[0].uploaded is True
|
||||||
|
receipt = fixture.service.complete_result_upload(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
result_sha256=package.manifest_sha256,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
assert receipt.result_id == RESULT_ID
|
||||||
|
package_root = fixture.service.require_completed_for_success(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
result_id=RESULT_ID,
|
||||||
|
result_sha256=package.manifest_sha256,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
succeeded = fixture.queue.succeed(
|
||||||
|
fixture.job.job_id,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
result_id=RESULT_ID,
|
||||||
|
result_sha256=package.manifest_sha256,
|
||||||
|
)
|
||||||
|
assert fixture.service.package_root_for_terminal(succeeded) == package_root
|
||||||
|
assert (package_root / "artifacts" / "result.json").read_bytes() == result_payload
|
||||||
|
|
||||||
|
|
||||||
|
def test_result_upload_rejects_wrong_digest_and_never_publishes_partial_member(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
fixture = _fixture(tmp_path)
|
||||||
|
package, result_payload = _result_package(tmp_path, fixture)
|
||||||
|
plan = fixture.service.stage_result_manifest(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
result_sha256=package.manifest_sha256,
|
||||||
|
manifest_payload=package.canonical_bytes,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
|
||||||
|
async def bad_chunks() -> object:
|
||||||
|
yield b"x" * len(result_payload)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
PortableArtifactTransportIntegrityError,
|
||||||
|
match="differs from its manifest",
|
||||||
|
):
|
||||||
|
asyncio.run(
|
||||||
|
fixture.service.upload_result_member(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
result_sha256=package.manifest_sha256,
|
||||||
|
member_id=plan.members[0].member_id,
|
||||||
|
chunks=bad_chunks(), # type: ignore[arg-type]
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
repeated = fixture.service.stage_result_manifest(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
result_sha256=package.manifest_sha256,
|
||||||
|
manifest_payload=package.canonical_bytes,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
assert repeated.members[0].uploaded is False
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -367,6 +367,34 @@ def test_not_installed_definition_fails_before_source_or_queue_writes(
|
|||||||
assert not (tmp_path / "observatory-recorded-jobs.sqlite3").exists()
|
assert not (tmp_path / "observatory-recorded-jobs.sqlite3").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_not_installed_model_free_definition_remains_capability_probeable(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
registry = _blocked_registry()
|
||||||
|
_write_probe_summary(tmp_path, segment_count=17)
|
||||||
|
service, store, queue, inspector = _service(
|
||||||
|
tmp_path,
|
||||||
|
registry=registry,
|
||||||
|
with_queue=False,
|
||||||
|
)
|
||||||
|
definition = registry.resolve_setup("m49-tgs-portable-v2")
|
||||||
|
|
||||||
|
capability = service.probe(
|
||||||
|
source_session_id=SESSION_ID,
|
||||||
|
setup_id=definition.setup_id,
|
||||||
|
definition_sha256=definition.definition_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert capability.source_session_id == SESSION_ID
|
||||||
|
assert capability.camera_segment_count == 17
|
||||||
|
assert capability.source_adapter_sha256 == definition.source_adapter.contract_sha256
|
||||||
|
assert store.catalog_reads == 1
|
||||||
|
assert store.prepare_replay_calls == 0
|
||||||
|
assert inspector.restore_calls == 0
|
||||||
|
assert queue is None
|
||||||
|
assert not (tmp_path / PORTABLE_SOURCE_DOCUMENT_DIRECTORY).exists()
|
||||||
|
|
||||||
|
|
||||||
def test_probe_is_bounded_and_does_not_enter_replay_or_media_inspector(
|
def test_probe_is_bounded_and_does_not_enter_replay_or_media_inspector(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -0,0 +1,595 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from collections.abc import Callable
|
||||||
|
from dataclasses import replace
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import cast
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from k1link.artifact_gateway import CentralArtifactStore
|
||||||
|
from k1link.observatory.portable_result_contract import (
|
||||||
|
OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||||
|
PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
|
||||||
|
RESULT_DOCUMENT_ROLE,
|
||||||
|
PortableCalculationProfilePolicy,
|
||||||
|
PortableCalculationProfileRegistry,
|
||||||
|
PortableResultArtifact,
|
||||||
|
PortableResultContractValidatorRegistration,
|
||||||
|
PortableResultContractValidatorRegistry,
|
||||||
|
PortableResultPackageIntegrityError,
|
||||||
|
PortableResultPackageManifest,
|
||||||
|
PortableResultPublicationBlockedError,
|
||||||
|
PortableResultValidationContext,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_result_publisher import (
|
||||||
|
PortableObservatoryResultPublisher,
|
||||||
|
resolve_published_portable_calculation_profile,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_run_definitions import (
|
||||||
|
PortableRunDefinition,
|
||||||
|
PortableRunDefinitionRegistry,
|
||||||
|
canonical_sha256,
|
||||||
|
)
|
||||||
|
from k1link.observatory.recorded_jobs import (
|
||||||
|
ObservatoryRecordedJob,
|
||||||
|
ObservatoryRecordedJobIntent,
|
||||||
|
ObservatoryRecordedJobQueue,
|
||||||
|
RecordedRunDefinitionRegistry,
|
||||||
|
)
|
||||||
|
from k1link.observatory.source_admission import (
|
||||||
|
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||||
|
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||||
|
PORTABLE_SOURCE_DOCUMENT_DIRECTORY,
|
||||||
|
)
|
||||||
|
from k1link.sessions import (
|
||||||
|
ObservationArchiveSource,
|
||||||
|
ObservationSessionCandidate,
|
||||||
|
SessionStore,
|
||||||
|
)
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||||
|
NOW = "2026-08-31T08:00:00.000Z"
|
||||||
|
SOURCE_SESSION_ID = "20260831T075500Z_viewer_live"
|
||||||
|
RESULT_ID = "portable-lab-result-001"
|
||||||
|
AUTHORITY = {
|
||||||
|
"commands_enabled": False,
|
||||||
|
"actuation_allowed": False,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
"production_accepted": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_json(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
allow_nan=False,
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _ready_registry(tmp_path: Path) -> PortableRunDefinitionRegistry:
|
||||||
|
base_definition = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH).definitions[0]
|
||||||
|
document = cast(
|
||||||
|
dict[str, object],
|
||||||
|
json.loads(REGISTRY_PATH.read_text(encoding="utf-8")),
|
||||||
|
)
|
||||||
|
rows = cast(list[object], document["definitions"])
|
||||||
|
selected = copy.deepcopy(cast(dict[str, object], rows[0]))
|
||||||
|
selected["executor"] = {
|
||||||
|
"contour_id": "worker-006",
|
||||||
|
"state": "ready",
|
||||||
|
"release_id": "lab-v1-portable-executor-v1",
|
||||||
|
"release_sha256": "1" * 64,
|
||||||
|
"image_sha256": "2" * 64,
|
||||||
|
"reason_code": None,
|
||||||
|
"reason": None,
|
||||||
|
}
|
||||||
|
identity = copy.deepcopy(base_definition.identity_document())
|
||||||
|
identity["executor"] = {
|
||||||
|
"contour_id": "worker-006",
|
||||||
|
"state": "ready",
|
||||||
|
"release_id": "lab-v1-portable-executor-v1",
|
||||||
|
"release_sha256": "1" * 64,
|
||||||
|
"image_sha256": "2" * 64,
|
||||||
|
}
|
||||||
|
selected["definition_sha256"] = canonical_sha256(identity)
|
||||||
|
path = tmp_path / "portable-definitions.json"
|
||||||
|
path.write_bytes(
|
||||||
|
_canonical_json(
|
||||||
|
{
|
||||||
|
"schema_version": document["schema_version"],
|
||||||
|
"definitions": [selected],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return PortableRunDefinitionRegistry.from_file(path)
|
||||||
|
|
||||||
|
|
||||||
|
def _source_store(
|
||||||
|
tmp_path: Path,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
) -> tuple[SessionStore, str, str, str]:
|
||||||
|
repository = tmp_path / "repository"
|
||||||
|
archive_root = tmp_path / "source-archive"
|
||||||
|
session_root = archive_root / SOURCE_SESSION_ID
|
||||||
|
session_root.mkdir(parents=True)
|
||||||
|
candidate = ObservationSessionCandidate(
|
||||||
|
session_id=SOURCE_SESSION_ID,
|
||||||
|
display_name="Portable result source",
|
||||||
|
status="ready",
|
||||||
|
started_at_utc="2026-08-31T07:55:00.000Z",
|
||||||
|
completed_at_utc="2026-08-31T07:59:00.000Z",
|
||||||
|
duration_seconds=240.0,
|
||||||
|
modalities=(),
|
||||||
|
replayable=False,
|
||||||
|
total_bytes=0,
|
||||||
|
allowed_root=archive_root,
|
||||||
|
session_root=session_root,
|
||||||
|
primary_replay_artifact_id=None,
|
||||||
|
timeline_origin_epoch_ns=None,
|
||||||
|
timeline_origin_monotonic_ns=None,
|
||||||
|
sources=(),
|
||||||
|
artifacts=(),
|
||||||
|
)
|
||||||
|
archive = ObservationArchiveSource(
|
||||||
|
plugin_id=definition.source_requirements.plugin_id,
|
||||||
|
archive_id=definition.source_requirements.archive_id,
|
||||||
|
root=archive_root,
|
||||||
|
discover=lambda _root: (candidate,),
|
||||||
|
)
|
||||||
|
store = SessionStore(repository, data_dir=tmp_path / "mission-core-data")
|
||||||
|
assert store.reconcile_archive(archive) == (SOURCE_SESSION_ID,)
|
||||||
|
_detail, catalog_sha256 = store.get_session_with_catalog_snapshot(SOURCE_SESSION_ID)
|
||||||
|
|
||||||
|
source_adapter = {
|
||||||
|
"id": definition.source_adapter.adapter_id,
|
||||||
|
"version": definition.source_adapter.version,
|
||||||
|
"sha256": definition.source_adapter.contract_sha256,
|
||||||
|
}
|
||||||
|
bundle = {
|
||||||
|
"schema_version": PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||||
|
"source_session_id": SOURCE_SESSION_ID,
|
||||||
|
"source_catalog_sha256": catalog_sha256,
|
||||||
|
"plugin_id": definition.source_requirements.plugin_id,
|
||||||
|
"archive_id": definition.source_requirements.archive_id,
|
||||||
|
"source_adapter": source_adapter,
|
||||||
|
"sources": [],
|
||||||
|
"spatial_replay": {},
|
||||||
|
"camera": {},
|
||||||
|
"authority": AUTHORITY,
|
||||||
|
}
|
||||||
|
bundle_bytes = _canonical_json(bundle)
|
||||||
|
bundle_sha256 = hashlib.sha256(bundle_bytes).hexdigest()
|
||||||
|
capability = {
|
||||||
|
"schema_version": PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||||
|
"source_session_id": SOURCE_SESSION_ID,
|
||||||
|
"source_catalog_sha256": catalog_sha256,
|
||||||
|
"source_bundle_sha256": bundle_sha256,
|
||||||
|
"source_adapter_sha256": definition.source_adapter.contract_sha256,
|
||||||
|
"modalities": [],
|
||||||
|
"camera_profile": {},
|
||||||
|
"calibration": {},
|
||||||
|
"authority": AUTHORITY,
|
||||||
|
}
|
||||||
|
capability_bytes = _canonical_json(capability)
|
||||||
|
capability_sha256 = hashlib.sha256(capability_bytes).hexdigest()
|
||||||
|
source_documents = store.data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY
|
||||||
|
source_documents.mkdir()
|
||||||
|
(source_documents / f"{bundle_sha256}.json").write_bytes(bundle_bytes)
|
||||||
|
(source_documents / f"{capability_sha256}.json").write_bytes(capability_bytes)
|
||||||
|
return store, catalog_sha256, bundle_sha256, capability_sha256
|
||||||
|
|
||||||
|
|
||||||
|
def _running_job(
|
||||||
|
tmp_path: Path,
|
||||||
|
*,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
catalog_sha256: str,
|
||||||
|
bundle_sha256: str,
|
||||||
|
capability_sha256: str,
|
||||||
|
) -> tuple[ObservatoryRecordedJobQueue, ObservatoryRecordedJob, str]:
|
||||||
|
recorded = definition.to_recorded_run_definition()
|
||||||
|
queue = ObservatoryRecordedJobQueue(
|
||||||
|
tmp_path / "mission-core-data",
|
||||||
|
definitions=RecordedRunDefinitionRegistry((recorded,)),
|
||||||
|
clock=lambda: NOW,
|
||||||
|
)
|
||||||
|
job, created = queue.submit(
|
||||||
|
ObservatoryRecordedJobIntent(
|
||||||
|
idempotency_key="portable-result-publication-001",
|
||||||
|
source_session_id=SOURCE_SESSION_ID,
|
||||||
|
source_catalog_sha256=catalog_sha256,
|
||||||
|
source_bundle_sha256=bundle_sha256,
|
||||||
|
source_capability_manifest_sha256=capability_sha256,
|
||||||
|
setup_id=definition.setup_id,
|
||||||
|
definition_sha256=definition.definition_sha256,
|
||||||
|
),
|
||||||
|
enqueue=True,
|
||||||
|
)
|
||||||
|
assert created is True
|
||||||
|
claim = queue.claim_next(
|
||||||
|
claimant_id="worker-006",
|
||||||
|
claim_request_id="portable-result-claim-001",
|
||||||
|
)
|
||||||
|
assert claim is not None
|
||||||
|
running = queue.start(job.job_id, claim_token=claim.claim_token)
|
||||||
|
assert running.state == "running"
|
||||||
|
return queue, running, claim.claim_token
|
||||||
|
|
||||||
|
|
||||||
|
def _package(
|
||||||
|
tmp_path: Path,
|
||||||
|
*,
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
result_id: str = RESULT_ID,
|
||||||
|
accepted: bool = True,
|
||||||
|
) -> tuple[Path, PortableResultPackageManifest]:
|
||||||
|
result_document = {
|
||||||
|
"schema_version": definition.result_contract.result_schema,
|
||||||
|
"result_id": result_id,
|
||||||
|
"result_kind": definition.result_contract.result_kind,
|
||||||
|
"accepted": accepted,
|
||||||
|
"authority": AUTHORITY,
|
||||||
|
}
|
||||||
|
result_bytes = _canonical_json(result_document)
|
||||||
|
artifact = PortableResultArtifact(
|
||||||
|
role=RESULT_DOCUMENT_ROLE,
|
||||||
|
relative_path="artifacts/result.json",
|
||||||
|
media_type="application/json",
|
||||||
|
byte_length=len(result_bytes),
|
||||||
|
sha256=hashlib.sha256(result_bytes).hexdigest(),
|
||||||
|
)
|
||||||
|
package = PortableResultPackageManifest.create(
|
||||||
|
job=job,
|
||||||
|
definition=definition,
|
||||||
|
result_id=result_id,
|
||||||
|
created_at_utc=NOW,
|
||||||
|
artifacts=(artifact,),
|
||||||
|
)
|
||||||
|
root = tmp_path / "packages" / package.manifest_sha256
|
||||||
|
(root / "artifacts").mkdir(parents=True)
|
||||||
|
(root / "manifest.json").write_bytes(package.canonical_bytes)
|
||||||
|
(root / "artifacts" / "result.json").write_bytes(result_bytes)
|
||||||
|
return root, package
|
||||||
|
|
||||||
|
|
||||||
|
def _profile(definition: PortableRunDefinition) -> PortableCalculationProfilePolicy:
|
||||||
|
return PortableCalculationProfilePolicy(
|
||||||
|
setup_id=definition.setup_id,
|
||||||
|
definition_id=definition.definition_id,
|
||||||
|
definition_version=definition.version,
|
||||||
|
definition_sha256=definition.definition_sha256,
|
||||||
|
lab_id="LAB V1",
|
||||||
|
display_name="LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validator(context: PortableResultValidationContext) -> None:
|
||||||
|
expected = {
|
||||||
|
"schema_version": context.definition.result_contract.result_schema,
|
||||||
|
"result_id": context.job.result_id,
|
||||||
|
"result_kind": context.definition.result_contract.result_kind,
|
||||||
|
"accepted": True,
|
||||||
|
"authority": AUTHORITY,
|
||||||
|
}
|
||||||
|
if dict(context.result_document) != expected:
|
||||||
|
raise ValueError("result contract payload was not accepted")
|
||||||
|
|
||||||
|
|
||||||
|
def _publisher(
|
||||||
|
tmp_path: Path,
|
||||||
|
*,
|
||||||
|
store: SessionStore,
|
||||||
|
registry: PortableRunDefinitionRegistry,
|
||||||
|
profile: PortableCalculationProfilePolicy | None,
|
||||||
|
validator: Callable[[PortableResultValidationContext], None] | None,
|
||||||
|
) -> PortableObservatoryResultPublisher:
|
||||||
|
definition = registry.definitions[0]
|
||||||
|
registrations = (
|
||||||
|
()
|
||||||
|
if validator is None
|
||||||
|
else (
|
||||||
|
PortableResultContractValidatorRegistration(
|
||||||
|
contract_sha256=definition.result_contract.contract_sha256,
|
||||||
|
validator=validator,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return PortableObservatoryResultPublisher(
|
||||||
|
session_store=store,
|
||||||
|
artifact_store=CentralArtifactStore(tmp_path / "central-artifacts", create=True),
|
||||||
|
definitions=registry,
|
||||||
|
calculation_profiles=PortableCalculationProfileRegistry(
|
||||||
|
() if profile is None else (profile,)
|
||||||
|
),
|
||||||
|
validators=PortableResultContractValidatorRegistry(registrations),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _fixture(
|
||||||
|
tmp_path: Path,
|
||||||
|
*,
|
||||||
|
result_id: str = RESULT_ID,
|
||||||
|
accepted: bool = True,
|
||||||
|
) -> tuple[
|
||||||
|
PortableRunDefinitionRegistry,
|
||||||
|
PortableRunDefinition,
|
||||||
|
SessionStore,
|
||||||
|
ObservatoryRecordedJob,
|
||||||
|
Path,
|
||||||
|
]:
|
||||||
|
registry = _ready_registry(tmp_path)
|
||||||
|
definition = registry.definitions[0]
|
||||||
|
store, catalog_sha, bundle_sha, capability_sha = _source_store(tmp_path, definition)
|
||||||
|
queue, running, claim_token = _running_job(
|
||||||
|
tmp_path,
|
||||||
|
definition=definition,
|
||||||
|
catalog_sha256=catalog_sha,
|
||||||
|
bundle_sha256=bundle_sha,
|
||||||
|
capability_sha256=capability_sha,
|
||||||
|
)
|
||||||
|
package_root, package = _package(
|
||||||
|
tmp_path,
|
||||||
|
job=running,
|
||||||
|
definition=definition,
|
||||||
|
result_id=result_id,
|
||||||
|
accepted=accepted,
|
||||||
|
)
|
||||||
|
succeeded = queue.succeed(
|
||||||
|
running.job_id,
|
||||||
|
claim_token=claim_token,
|
||||||
|
result_id=result_id,
|
||||||
|
result_sha256=package.manifest_sha256,
|
||||||
|
)
|
||||||
|
return registry, definition, store, succeeded, package_root
|
||||||
|
|
||||||
|
|
||||||
|
def test_verified_package_publishes_immutable_binding_and_profile_provenance(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
registry, definition, store, job, package_root = _fixture(tmp_path)
|
||||||
|
publisher = _publisher(
|
||||||
|
tmp_path,
|
||||||
|
store=store,
|
||||||
|
registry=registry,
|
||||||
|
profile=_profile(definition),
|
||||||
|
validator=_validator,
|
||||||
|
)
|
||||||
|
|
||||||
|
first = publisher.publish(job=job, package_root=package_root)
|
||||||
|
second = publisher.publish(job=job, package_root=package_root)
|
||||||
|
|
||||||
|
assert second.binding == first.binding
|
||||||
|
assert second.artifact_manifest == first.artifact_manifest
|
||||||
|
assert first.binding.session_id == RESULT_ID
|
||||||
|
assert first.binding.source_session_id == SOURCE_SESSION_ID
|
||||||
|
assert first.binding.config_sha256 == definition.definition_sha256
|
||||||
|
assert first.binding.replay_capability is None
|
||||||
|
assert first.binding.provenance["calculation_profile"] == {
|
||||||
|
"schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||||
|
"setup_id": definition.setup_id,
|
||||||
|
"display_name": "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||||
|
"origin": "archived-definition",
|
||||||
|
"definition_id": definition.definition_id,
|
||||||
|
"definition_version": definition.version,
|
||||||
|
"definition_sha256": definition.definition_sha256,
|
||||||
|
}
|
||||||
|
package_provenance = cast(dict[str, object], first.binding.provenance["result_package"])
|
||||||
|
assert package_provenance["manifest_sha256"] == job.result_sha256
|
||||||
|
assert package_provenance["artifact_manifest_id"] == first.artifact_manifest.manifest_id
|
||||||
|
assert store.get_lab_instance(RESULT_ID) == first.binding
|
||||||
|
assert store.get_session(SOURCE_SESSION_ID).summary.lab is None
|
||||||
|
|
||||||
|
summary = store.get_session(RESULT_ID).summary
|
||||||
|
assert summary.display_name == (
|
||||||
|
"Portable result source · полный маршрут и воспроизведение"
|
||||||
|
)
|
||||||
|
profiles = PortableCalculationProfileRegistry((_profile(definition),))
|
||||||
|
assert resolve_published_portable_calculation_profile(
|
||||||
|
summary,
|
||||||
|
definitions=registry,
|
||||||
|
calculation_profiles=profiles,
|
||||||
|
) == _profile(definition).as_dict()
|
||||||
|
|
||||||
|
assert summary.lab is not None
|
||||||
|
drifted_provenance = copy.deepcopy(summary.lab.provenance)
|
||||||
|
drifted_provenance["calculation_profile_sha256"] = "0" * 64
|
||||||
|
drifted = replace(
|
||||||
|
summary,
|
||||||
|
lab=replace(summary.lab, provenance=drifted_provenance),
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
resolve_published_portable_calculation_profile(
|
||||||
|
drifted,
|
||||||
|
definitions=registry,
|
||||||
|
calculation_profiles=profiles,
|
||||||
|
)
|
||||||
|
is None
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_result_contract_fails_before_artifacts_or_catalog_are_published(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
registry, definition, store, job, package_root = _fixture(tmp_path)
|
||||||
|
publisher = _publisher(
|
||||||
|
tmp_path,
|
||||||
|
store=store,
|
||||||
|
registry=registry,
|
||||||
|
profile=_profile(definition),
|
||||||
|
validator=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
PortableResultPublicationBlockedError,
|
||||||
|
match="validator is not installed",
|
||||||
|
):
|
||||||
|
publisher.publish(job=job, package_root=package_root)
|
||||||
|
|
||||||
|
assert store.get_lab_instance(RESULT_ID) is None
|
||||||
|
assert not tuple((tmp_path / "central-artifacts").glob("manifests/sha256/*/*"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_definition_bound_profile_is_not_inferred_from_result_or_ui(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
registry, _definition, store, job, package_root = _fixture(tmp_path)
|
||||||
|
publisher = _publisher(
|
||||||
|
tmp_path,
|
||||||
|
store=store,
|
||||||
|
registry=registry,
|
||||||
|
profile=None,
|
||||||
|
validator=_validator,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
PortableResultPublicationBlockedError,
|
||||||
|
match="profile policy is not registered",
|
||||||
|
):
|
||||||
|
publisher.publish(job=job, package_root=package_root)
|
||||||
|
|
||||||
|
assert store.get_lab_instance(RESULT_ID) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_changed_artifact_bytes_fail_closed_before_catalog_publication(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
registry, definition, store, job, package_root = _fixture(tmp_path)
|
||||||
|
(package_root / "artifacts" / "result.json").write_bytes(b"x" * 8)
|
||||||
|
publisher = _publisher(
|
||||||
|
tmp_path,
|
||||||
|
store=store,
|
||||||
|
registry=registry,
|
||||||
|
profile=_profile(definition),
|
||||||
|
validator=_validator,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
PortableResultPackageIntegrityError,
|
||||||
|
match="artifact content changed",
|
||||||
|
):
|
||||||
|
publisher.publish(job=job, package_root=package_root)
|
||||||
|
|
||||||
|
assert store.get_lab_instance(RESULT_ID) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_contract_validator_rejection_never_becomes_a_lab_result(tmp_path: Path) -> None:
|
||||||
|
registry, definition, store, job, package_root = _fixture(
|
||||||
|
tmp_path,
|
||||||
|
accepted=False,
|
||||||
|
)
|
||||||
|
publisher = _publisher(
|
||||||
|
tmp_path,
|
||||||
|
store=store,
|
||||||
|
registry=registry,
|
||||||
|
profile=_profile(definition),
|
||||||
|
validator=_validator,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
PortableResultPackageIntegrityError,
|
||||||
|
match="failed its exact contract validator",
|
||||||
|
):
|
||||||
|
publisher.publish(job=job, package_root=package_root)
|
||||||
|
|
||||||
|
assert store.get_lab_instance(RESULT_ID) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_persisted_source_contract_is_a_publication_blocker(tmp_path: Path) -> None:
|
||||||
|
registry, definition, store, job, package_root = _fixture(tmp_path)
|
||||||
|
source_documents = store.data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY
|
||||||
|
for path in source_documents.iterdir():
|
||||||
|
path.unlink()
|
||||||
|
publisher = _publisher(
|
||||||
|
tmp_path,
|
||||||
|
store=store,
|
||||||
|
registry=registry,
|
||||||
|
profile=_profile(definition),
|
||||||
|
validator=_validator,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
PortableResultPublicationBlockedError,
|
||||||
|
match="source document is unavailable",
|
||||||
|
):
|
||||||
|
publisher.publish(job=job, package_root=package_root)
|
||||||
|
|
||||||
|
assert store.get_lab_instance(RESULT_ID) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_package_manifest_rejects_noncanonical_or_authority_elevating_documents(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
registry = _ready_registry(tmp_path)
|
||||||
|
definition = registry.definitions[0]
|
||||||
|
store, catalog_sha, bundle_sha, capability_sha = _source_store(tmp_path, definition)
|
||||||
|
_queue, running, _claim_token = _running_job(
|
||||||
|
tmp_path,
|
||||||
|
definition=definition,
|
||||||
|
catalog_sha256=catalog_sha,
|
||||||
|
bundle_sha256=bundle_sha,
|
||||||
|
capability_sha256=capability_sha,
|
||||||
|
)
|
||||||
|
_root, manifest = _package(
|
||||||
|
tmp_path,
|
||||||
|
job=running,
|
||||||
|
definition=definition,
|
||||||
|
)
|
||||||
|
elevated = manifest.as_dict()
|
||||||
|
authority = cast(dict[str, object], elevated["authority"])
|
||||||
|
authority["commands_enabled"] = True
|
||||||
|
identity = {
|
||||||
|
"schema_version": PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
|
||||||
|
**{
|
||||||
|
key: value
|
||||||
|
for key, value in elevated.items()
|
||||||
|
if key not in {"schema_version", "identity_sha256"}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
elevated["identity_sha256"] = canonical_sha256(identity)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
PortableResultPackageIntegrityError,
|
||||||
|
match="not observation-only",
|
||||||
|
):
|
||||||
|
PortableResultPackageManifest.from_bytes(_canonical_json(elevated))
|
||||||
|
with pytest.raises(
|
||||||
|
PortableResultPackageIntegrityError,
|
||||||
|
match="not canonical JSON",
|
||||||
|
):
|
||||||
|
PortableResultPackageManifest.from_bytes(
|
||||||
|
json.dumps(manifest.as_dict(), indent=2).encode()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_legacy_canonical_result_namespace_cannot_be_republished(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
legacy_result_id = "lab-v1-vegetation-shadow-" + "a" * 64
|
||||||
|
registry, definition, store, job, package_root = _fixture(
|
||||||
|
tmp_path,
|
||||||
|
result_id=legacy_result_id,
|
||||||
|
)
|
||||||
|
publisher = _publisher(
|
||||||
|
tmp_path,
|
||||||
|
store=store,
|
||||||
|
registry=registry,
|
||||||
|
profile=_profile(definition),
|
||||||
|
validator=_validator,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
PortableResultPublicationBlockedError,
|
||||||
|
match="legacy canonical result namespace",
|
||||||
|
):
|
||||||
|
publisher.publish(job=job, package_root=package_root)
|
||||||
|
|
||||||
|
assert store.get_lab_instance(legacy_result_id) is None
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import copy
|
import copy
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
from dataclasses import FrozenInstanceError, replace
|
from dataclasses import FrozenInstanceError, replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -18,8 +19,10 @@ from k1link.observatory.portable_run_definitions import (
|
|||||||
|
|
||||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||||
DEFINITION_SHA256 = "57bf8f0859e10e54e30322c9a8aa28b427699f6fe6b5267e279ec3390fa78466"
|
DEFINITION_SHA256 = "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2"
|
||||||
MODEL_MANIFEST_SHA256 = "3fd2d43af73bd73f89d9ffae95d8770cfdeb46033ec967509124fac6ae4afe56"
|
MODEL_MANIFEST_SHA256 = "3fd2d43af73bd73f89d9ffae95d8770cfdeb46033ec967509124fac6ae4afe56"
|
||||||
|
M49_DEFINITION_SHA256 = "73611f24d70319ea1edca428726d6538a3cbad012a415cc0c1a7ecb7d9b4d910"
|
||||||
|
M49_MODEL_MANIFEST_SHA256 = "489a43448f720a9b5c7993dc8279d167b77191a586f0d87b6d38b81cf728e2f1"
|
||||||
|
|
||||||
|
|
||||||
def _registry() -> PortableRunDefinitionRegistry:
|
def _registry() -> PortableRunDefinitionRegistry:
|
||||||
@@ -96,6 +99,54 @@ def test_source_requirements_map_exactly_to_admission_contract() -> None:
|
|||||||
assert admission.adapter_sha256 == definition.source_adapter.contract_sha256
|
assert admission.adapter_sha256 == definition.source_adapter.contract_sha256
|
||||||
|
|
||||||
|
|
||||||
|
def test_m49_portable_v2_is_model_free_and_contains_no_exact_source_binding() -> None:
|
||||||
|
definition = _registry().resolve_setup("m49-tgs-portable-v2")
|
||||||
|
|
||||||
|
assert definition.definition_sha256 == M49_DEFINITION_SHA256
|
||||||
|
assert definition.models == ()
|
||||||
|
assert definition.learned_models == ()
|
||||||
|
assert definition.model_manifest_sha256 == M49_MODEL_MANIFEST_SHA256
|
||||||
|
assert definition.resource_profile.accelerator_id == "cpu-only"
|
||||||
|
assert definition.executor.state == "not-installed"
|
||||||
|
assert definition.executor.release_id is None
|
||||||
|
assert definition.executor.release_sha256 is None
|
||||||
|
assert definition.executor.image_sha256 is None
|
||||||
|
identity = json.dumps(definition.identity_document(), sort_keys=True)
|
||||||
|
assert "RAVNOVES00" not in identity
|
||||||
|
assert "20260720T065719Z_viewer_live" not in identity
|
||||||
|
assert "4489" not in identity
|
||||||
|
assert "3928" not in identity
|
||||||
|
|
||||||
|
profile_path = REPOSITORY_ROOT / "config" / "perception" / "m49-tgs-portable-v2.json"
|
||||||
|
profile = json.loads(profile_path.read_text(encoding="utf-8"))
|
||||||
|
components = {component.component_id: component for component in definition.components}
|
||||||
|
assert hashlib.sha256(profile_path.read_bytes()).hexdigest() == (
|
||||||
|
components["m49-tgs-portable-profile-v2"].sha256
|
||||||
|
)
|
||||||
|
assert profile["source_binding"] == {
|
||||||
|
"mode": "admitted-k1-recording",
|
||||||
|
"camera_timeline": "dynamic",
|
||||||
|
"lidar_replay": "dynamic",
|
||||||
|
"trajectory": "dynamic",
|
||||||
|
"frame_counts": "source-derived",
|
||||||
|
"filesystem_paths": "executor-resolved",
|
||||||
|
}
|
||||||
|
|
||||||
|
with pytest.raises(PortableRunDefinitionRegistryError, match="runner"):
|
||||||
|
replace(
|
||||||
|
definition,
|
||||||
|
executor=PortableExecutorAvailability(
|
||||||
|
contour_id="worker-006",
|
||||||
|
state="ready",
|
||||||
|
release_id="m49-tgs-portable-executor-v2",
|
||||||
|
release_sha256="1" * 64,
|
||||||
|
image_sha256="2" * 64,
|
||||||
|
reason_code=None,
|
||||||
|
reason=None,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_all_canonical_identities_are_recomputed_from_typed_content() -> None:
|
def test_all_canonical_identities_are_recomputed_from_typed_content() -> None:
|
||||||
definition = _registry().definitions[0]
|
definition = _registry().definitions[0]
|
||||||
|
|
||||||
@@ -278,6 +329,33 @@ def test_conversion_to_recorded_definition_requires_and_preserves_sealed_identit
|
|||||||
assert recorded.checkpoint_policy == "non-checkpointable"
|
assert recorded.checkpoint_policy == "non-checkpointable"
|
||||||
|
|
||||||
|
|
||||||
|
def test_blocked_definition_does_not_hide_an_unrelated_ready_definition() -> None:
|
||||||
|
registry = _registry()
|
||||||
|
blocked_lab = registry.resolve_setup("lab-v1-eomt-ddrnet-portable-v1")
|
||||||
|
blocked_m49 = registry.resolve_setup("m49-tgs-portable-v2")
|
||||||
|
ready_executor = PortableExecutorAvailability(
|
||||||
|
contour_id="worker-006",
|
||||||
|
state="ready",
|
||||||
|
release_id="lab-v1-eomt-ddrnet-executor-v1",
|
||||||
|
release_sha256="1" * 64,
|
||||||
|
image_sha256="2" * 64,
|
||||||
|
reason_code=None,
|
||||||
|
reason=None,
|
||||||
|
)
|
||||||
|
identity = blocked_lab.identity_document()
|
||||||
|
identity["executor"] = ready_executor.identity_document()
|
||||||
|
ready_lab = replace(
|
||||||
|
blocked_lab,
|
||||||
|
executor=ready_executor,
|
||||||
|
definition_sha256=canonical_sha256(identity),
|
||||||
|
)
|
||||||
|
mixed = PortableRunDefinitionRegistry((ready_lab, blocked_m49))
|
||||||
|
|
||||||
|
assert mixed.ready_recorded_definitions() == (ready_lab.to_recorded_run_definition(),)
|
||||||
|
assert mixed.to_recorded_registry().definitions == (ready_lab.to_recorded_run_definition(),)
|
||||||
|
assert mixed.resolve_setup("m49-tgs-portable-v2") is blocked_m49
|
||||||
|
|
||||||
|
|
||||||
def test_production_lab_v1_model_component_and_result_identities_are_exact() -> None:
|
def test_production_lab_v1_model_component_and_result_identities_are_exact() -> None:
|
||||||
definition = _registry().definitions[0]
|
definition = _registry().definitions[0]
|
||||||
models = {model.release_id: model for model in definition.models}
|
models = {model.release_id: model for model in definition.models}
|
||||||
|
|||||||
@@ -1,13 +1,26 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry
|
from k1link.observatory.portable_run_definitions import (
|
||||||
from k1link.observatory.portable_setup_projection import PortableLabV1SetupProjector
|
PortableExecutorAvailability,
|
||||||
|
PortableRunDefinitionRegistry,
|
||||||
|
canonical_sha256,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_setup_projection import (
|
||||||
|
PortableLabV1SetupProjector,
|
||||||
|
PortableSetupProjector,
|
||||||
|
)
|
||||||
|
from k1link.observatory.recorded_jobs import (
|
||||||
|
ObservatoryRecordedJobIntent,
|
||||||
|
ObservatoryRecordedJobQueue,
|
||||||
|
RecordedRunDefinitionRegistry,
|
||||||
|
)
|
||||||
from k1link.observatory.source_admission import PortableRecordedSourceCapability
|
from k1link.observatory.source_admission import PortableRecordedSourceCapability
|
||||||
from k1link.sessions import SessionNotFoundError
|
from k1link.sessions import SessionNotFoundError
|
||||||
from k1link.sessions.models import SessionSummary
|
from k1link.sessions.models import SessionSummary
|
||||||
@@ -111,4 +124,193 @@ def test_portable_setup_catalog_preserves_source_and_optional_slice_failures() -
|
|||||||
params={"source_session_id": SOURCE_SESSION_ID},
|
params={"source_session_id": SOURCE_SESSION_ID},
|
||||||
)
|
)
|
||||||
assert unavailable.status_code == 503
|
assert unavailable.status_code == 503
|
||||||
assert unavailable.json()["detail"] == "Portable-каталог LAB V1 недоступен."
|
assert unavailable.json()["detail"] == "Portable-каталог сетапов недоступен."
|
||||||
|
|
||||||
|
|
||||||
|
def _ready_lab_registry() -> PortableRunDefinitionRegistry:
|
||||||
|
blocked = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH).resolve_setup(
|
||||||
|
"lab-v1-eomt-ddrnet-portable-v1"
|
||||||
|
)
|
||||||
|
executor = PortableExecutorAvailability(
|
||||||
|
contour_id="worker-006",
|
||||||
|
state="ready",
|
||||||
|
release_id="lab-v1-eomt-ddrnet-executor-v1",
|
||||||
|
release_sha256="1" * 64,
|
||||||
|
image_sha256="2" * 64,
|
||||||
|
reason_code=None,
|
||||||
|
reason=None,
|
||||||
|
)
|
||||||
|
identity = blocked.identity_document()
|
||||||
|
identity["executor"] = executor.identity_document()
|
||||||
|
return PortableRunDefinitionRegistry(
|
||||||
|
(
|
||||||
|
replace(
|
||||||
|
blocked,
|
||||||
|
executor=executor,
|
||||||
|
definition_sha256=canonical_sha256(identity),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _PortableBinding:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
registry: PortableRunDefinitionRegistry,
|
||||||
|
queue: ObservatoryRecordedJobQueue,
|
||||||
|
) -> None:
|
||||||
|
self.registry = registry
|
||||||
|
self.queue = queue
|
||||||
|
self.check_sha256 = "9" * 64
|
||||||
|
self.check_count = 0
|
||||||
|
self.submit_count = 0
|
||||||
|
|
||||||
|
def probe(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
source_session_id: str,
|
||||||
|
setup_id: str,
|
||||||
|
definition_sha256: str,
|
||||||
|
) -> PortableRecordedSourceCapability:
|
||||||
|
definition = self.registry.resolve(setup_id, definition_sha256)
|
||||||
|
return PortableRecordedSourceCapability(
|
||||||
|
source_session_id=source_session_id,
|
||||||
|
source_catalog_sha256="a" * 64,
|
||||||
|
source_adapter_sha256=definition.source_adapter.contract_sha256,
|
||||||
|
camera_segment_count=600,
|
||||||
|
)
|
||||||
|
|
||||||
|
def check(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
source_session_id: str,
|
||||||
|
setup_id: str,
|
||||||
|
definition_sha256: str,
|
||||||
|
) -> SimpleNamespace:
|
||||||
|
self.registry.resolve(setup_id, definition_sha256)
|
||||||
|
assert source_session_id == SOURCE_SESSION_ID
|
||||||
|
self.check_count += 1
|
||||||
|
return SimpleNamespace(check_sha256=self.check_sha256)
|
||||||
|
|
||||||
|
def submit(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
source_session_id: str,
|
||||||
|
setup_id: str,
|
||||||
|
definition_sha256: str,
|
||||||
|
expected_check_sha256: str,
|
||||||
|
idempotency_key: str,
|
||||||
|
) -> tuple[object, bool]:
|
||||||
|
self.registry.resolve(setup_id, definition_sha256)
|
||||||
|
assert source_session_id == SOURCE_SESSION_ID
|
||||||
|
assert expected_check_sha256 == self.check_sha256
|
||||||
|
self.submit_count += 1
|
||||||
|
return self.queue.submit(
|
||||||
|
ObservatoryRecordedJobIntent(
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
source_session_id=source_session_id,
|
||||||
|
source_catalog_sha256="a" * 64,
|
||||||
|
source_bundle_sha256="b" * 64,
|
||||||
|
source_capability_manifest_sha256="c" * 64,
|
||||||
|
setup_id=setup_id,
|
||||||
|
definition_sha256=definition_sha256,
|
||||||
|
),
|
||||||
|
enqueue=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_portable_api_check_sha_fences_ready_submission(tmp_path: Path) -> None:
|
||||||
|
registry = _ready_lab_registry()
|
||||||
|
queue = ObservatoryRecordedJobQueue(
|
||||||
|
tmp_path,
|
||||||
|
definitions=RecordedRunDefinitionRegistry(registry.ready_recorded_definitions()),
|
||||||
|
)
|
||||||
|
binding = _PortableBinding(registry, queue)
|
||||||
|
projector = PortableSetupProjector(
|
||||||
|
registry=registry,
|
||||||
|
capability_probe=binding, # type: ignore[arg-type]
|
||||||
|
dispatch_available=True,
|
||||||
|
)
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(
|
||||||
|
build_observatory_router(
|
||||||
|
_Store(), # type: ignore[arg-type]
|
||||||
|
portable_setup_projector=projector,
|
||||||
|
portable_binding_service=binding, # type: ignore[arg-type]
|
||||||
|
recorded_job_queue=queue,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
client = TestClient(app)
|
||||||
|
definition = registry.definitions[0]
|
||||||
|
preflight = client.post(
|
||||||
|
"/api/v1/observatory/run-preflights",
|
||||||
|
json={
|
||||||
|
"schema_version": "missioncore.observatory-run-preflight-request/v1",
|
||||||
|
"source_session_id": SOURCE_SESSION_ID,
|
||||||
|
"setup_id": definition.setup_id,
|
||||||
|
"definition_sha256": definition.definition_sha256,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert preflight.status_code == 200
|
||||||
|
assert preflight.json()["outcome"] == "queueable"
|
||||||
|
assert preflight.json()["check_sha256"] == binding.check_sha256
|
||||||
|
assert binding.check_count == 1
|
||||||
|
|
||||||
|
submitted = client.post(
|
||||||
|
"/api/v1/observatory/runs",
|
||||||
|
json={
|
||||||
|
"schema_version": "missioncore.observatory-recorded-run-submit/v1",
|
||||||
|
"idempotency_key": "portable:lab-v1:source-005",
|
||||||
|
"source_session_id": SOURCE_SESSION_ID,
|
||||||
|
"setup_id": definition.setup_id,
|
||||||
|
"definition_sha256": definition.definition_sha256,
|
||||||
|
"check_sha256": binding.check_sha256,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert submitted.status_code == 202
|
||||||
|
assert submitted.json()["state"] == "queued"
|
||||||
|
assert submitted.json()["setup"]["setup_id"] == definition.setup_id
|
||||||
|
assert binding.submit_count == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_portable_api_rejects_blocked_executor_before_binding_submit(tmp_path: Path) -> None:
|
||||||
|
full_registry = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||||
|
ready_registry = _ready_lab_registry()
|
||||||
|
queue = ObservatoryRecordedJobQueue(
|
||||||
|
tmp_path,
|
||||||
|
definitions=RecordedRunDefinitionRegistry(ready_registry.ready_recorded_definitions()),
|
||||||
|
)
|
||||||
|
binding = _PortableBinding(full_registry, queue)
|
||||||
|
projector = PortableSetupProjector(
|
||||||
|
registry=full_registry,
|
||||||
|
capability_probe=binding, # type: ignore[arg-type]
|
||||||
|
dispatch_available=True,
|
||||||
|
)
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(
|
||||||
|
build_observatory_router(
|
||||||
|
_Store(), # type: ignore[arg-type]
|
||||||
|
portable_setup_projector=projector,
|
||||||
|
portable_binding_service=binding, # type: ignore[arg-type]
|
||||||
|
recorded_job_queue=queue,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
definition = full_registry.resolve_setup("m49-tgs-portable-v2")
|
||||||
|
response = TestClient(app).post(
|
||||||
|
"/api/v1/observatory/runs",
|
||||||
|
json={
|
||||||
|
"schema_version": "missioncore.observatory-recorded-run-submit/v1",
|
||||||
|
"idempotency_key": "portable:m49:blocked",
|
||||||
|
"source_session_id": SOURCE_SESSION_ID,
|
||||||
|
"setup_id": definition.setup_id,
|
||||||
|
"definition_sha256": definition.definition_sha256,
|
||||||
|
"check_sha256": "9" * 64,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 409
|
||||||
|
assert response.json()["detail"] == "Portable executor-релиз не установлен."
|
||||||
|
assert binding.submit_count == 0
|
||||||
|
assert queue.list_jobs() == ()
|
||||||
|
|||||||
@@ -13,9 +13,12 @@ from k1link.observatory.portable_run_definitions import (
|
|||||||
from k1link.observatory.portable_setup_projection import (
|
from k1link.observatory.portable_setup_projection import (
|
||||||
PORTABLE_LAB_V1_DISPLAY_NAME,
|
PORTABLE_LAB_V1_DISPLAY_NAME,
|
||||||
PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA,
|
PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA,
|
||||||
|
PORTABLE_M49_DISPLAY_NAME,
|
||||||
PortableLabV1SetupProjector,
|
PortableLabV1SetupProjector,
|
||||||
PortableSetupProjectionError,
|
PortableSetupProjectionError,
|
||||||
|
PortableSetupProjector,
|
||||||
PortableSourceCapabilityProbe,
|
PortableSourceCapabilityProbe,
|
||||||
|
portable_calculation_profile_registry,
|
||||||
)
|
)
|
||||||
from k1link.observatory.source_admission import (
|
from k1link.observatory.source_admission import (
|
||||||
PortableRecordedSourceCapability,
|
PortableRecordedSourceCapability,
|
||||||
@@ -119,6 +122,61 @@ def test_projection_uses_portable_identity_and_exact_model_presentation() -> Non
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class _GenericProbe:
|
||||||
|
def __init__(self, registry: PortableRunDefinitionRegistry) -> None:
|
||||||
|
self.registry = registry
|
||||||
|
|
||||||
|
def probe(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
source_session_id: str,
|
||||||
|
setup_id: str,
|
||||||
|
definition_sha256: str,
|
||||||
|
) -> PortableRecordedSourceCapability:
|
||||||
|
definition = self.registry.resolve(setup_id, definition_sha256)
|
||||||
|
return _capability(
|
||||||
|
source_session_id,
|
||||||
|
adapter_sha256=definition.source_adapter.contract_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_generic_catalog_projects_lab_v1_and_model_free_m49_independently() -> None:
|
||||||
|
registry = _registry()
|
||||||
|
catalog = PortableSetupProjector(
|
||||||
|
registry=registry,
|
||||||
|
capability_probe=_GenericProbe(registry),
|
||||||
|
).catalog(_source(NEW_SESSION_ID))
|
||||||
|
|
||||||
|
setups = {setup["setup_id"]: setup for setup in catalog["setups"]}
|
||||||
|
assert set(setups) == {
|
||||||
|
"lab-v1-eomt-ddrnet-portable-v1",
|
||||||
|
"m49-tgs-portable-v2",
|
||||||
|
}
|
||||||
|
m49 = setups["m49-tgs-portable-v2"]
|
||||||
|
assert m49["display_name"] == PORTABLE_M49_DISPLAY_NAME
|
||||||
|
assert m49["run_definition"]["models"] == []
|
||||||
|
assert m49["source_compatibility"]["outcome"] == "pass"
|
||||||
|
assert m49["executor"]["state"] == "not-installed"
|
||||||
|
assert m49["preflight"]["submission_allowed"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_calculation_profile_policies_cover_both_exact_portable_definitions() -> None:
|
||||||
|
registry = _registry()
|
||||||
|
profiles = portable_calculation_profile_registry(registry)
|
||||||
|
|
||||||
|
resolved = {
|
||||||
|
definition.setup_id: profiles.resolve(definition)
|
||||||
|
for definition in registry.definitions
|
||||||
|
}
|
||||||
|
assert resolved["lab-v1-eomt-ddrnet-portable-v1"].lab_id == "LAB V1"
|
||||||
|
assert (
|
||||||
|
resolved["lab-v1-eomt-ddrnet-portable-v1"].display_name
|
||||||
|
== PORTABLE_LAB_V1_DISPLAY_NAME
|
||||||
|
)
|
||||||
|
assert resolved["m49-tgs-portable-v2"].lab_id == "LAB M4.9T5"
|
||||||
|
assert resolved["m49-tgs-portable-v2"].display_name == PORTABLE_M49_DISPLAY_NAME
|
||||||
|
|
||||||
|
|
||||||
def test_new_compatible_source_passes_capability_but_uninstalled_executor_blocks() -> None:
|
def test_new_compatible_source_passes_capability_but_uninstalled_executor_blocks() -> None:
|
||||||
source = _source(NEW_SESSION_ID)
|
source = _source(NEW_SESSION_ID)
|
||||||
setup = _projector(lambda session_id: _capability(session_id)).project(source)
|
setup = _projector(lambda session_id: _capability(session_id)).project(source)
|
||||||
@@ -235,10 +293,7 @@ def test_ready_executor_still_blocks_without_a_dispatch_boundary() -> None:
|
|||||||
assert setup["preflight"] == {
|
assert setup["preflight"] == {
|
||||||
"outcome": "blocked",
|
"outcome": "blocked",
|
||||||
"action": "blocked",
|
"action": "blocked",
|
||||||
"reason": (
|
"reason": ("Server-side проверка и постановка portable-сетапа в очередь недоступны."),
|
||||||
"Server-side проверка definition/check SHA и постановка portable "
|
|
||||||
"LAB V1 в очередь пока недоступны."
|
|
||||||
),
|
|
||||||
"submission_allowed": False,
|
"submission_allowed": False,
|
||||||
"existing_result_ids": [],
|
"existing_result_ids": [],
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,253 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import cast
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from k1link.artifact_gateway import CentralArtifactStore
|
||||||
|
from k1link.observatory.m49_portable_result import (
|
||||||
|
M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||||
|
validate_m49_portable_result,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_artifact_transport import (
|
||||||
|
PortableObservatoryArtifactTransport,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_lab_v1_executor import validate_lab_v1_result_v2
|
||||||
|
from k1link.observatory.portable_result_contract import (
|
||||||
|
PortableResultContractValidatorRegistration,
|
||||||
|
PortableResultContractValidatorRegistry,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_result_publisher import (
|
||||||
|
PortableObservatoryResultPublisher,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_run_definitions import (
|
||||||
|
PortableRunDefinitionRegistry,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_setup_projection import (
|
||||||
|
PORTABLE_LAB_V1_DISPLAY_NAME,
|
||||||
|
PORTABLE_LAB_V1_SETUP_ID,
|
||||||
|
PORTABLE_M49_DISPLAY_NAME,
|
||||||
|
PORTABLE_M49_SETUP_ID,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_worker_integration import (
|
||||||
|
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV,
|
||||||
|
OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV,
|
||||||
|
PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256,
|
||||||
|
PortableWorkerIntegrationError,
|
||||||
|
PortableWorkerStorageRoots,
|
||||||
|
build_portable_observatory_worker_integration,
|
||||||
|
portable_result_validator_registry,
|
||||||
|
)
|
||||||
|
from k1link.observatory.recorded_jobs import ObservatoryRecordedJobQueue
|
||||||
|
from k1link.sessions import RecordedMediaInspector, SessionStore
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||||
|
|
||||||
|
|
||||||
|
def _definitions() -> PortableRunDefinitionRegistry:
|
||||||
|
return PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||||
|
|
||||||
|
|
||||||
|
def test_exact_validator_registry_covers_both_portable_profiles() -> None:
|
||||||
|
definitions = _definitions()
|
||||||
|
|
||||||
|
validators = portable_result_validator_registry(definitions)
|
||||||
|
|
||||||
|
assert validators.resolve(PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256) is validate_lab_v1_result_v2
|
||||||
|
assert validators.resolve(M49_PORTABLE_RESULT_CONTRACT_SHA256) is validate_m49_portable_result
|
||||||
|
|
||||||
|
|
||||||
|
def test_validator_registry_fails_when_one_required_profile_is_absent() -> None:
|
||||||
|
definitions = _definitions()
|
||||||
|
only_lab_v1 = PortableRunDefinitionRegistry((definitions.definitions[0],))
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
PortableWorkerIntegrationError,
|
||||||
|
match="required portable setup is unavailable",
|
||||||
|
):
|
||||||
|
portable_result_validator_registry(only_lab_v1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_server_integration_constructs_dormant_transport_and_publisher(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
definitions = _definitions()
|
||||||
|
store = SessionStore(tmp_path / "repository")
|
||||||
|
inspector = RecordedMediaInspector(tmp_path / "media-inspections")
|
||||||
|
source_cas_root = tmp_path / "source-cas"
|
||||||
|
result_staging_root = tmp_path / "result-staging"
|
||||||
|
source_cas_root.mkdir()
|
||||||
|
result_staging_root.mkdir()
|
||||||
|
|
||||||
|
integration = build_portable_observatory_worker_integration(
|
||||||
|
queue=cast(ObservatoryRecordedJobQueue, object()),
|
||||||
|
session_store=store,
|
||||||
|
media_inspector=inspector,
|
||||||
|
definitions=definitions,
|
||||||
|
artifact_store=CentralArtifactStore(tmp_path / "artifacts", create=True),
|
||||||
|
source_cas_root=source_cas_root,
|
||||||
|
result_staging_root=result_staging_root,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert integration.supported_setup_ids == (
|
||||||
|
PORTABLE_LAB_V1_SETUP_ID,
|
||||||
|
PORTABLE_M49_SETUP_ID,
|
||||||
|
)
|
||||||
|
assert isinstance(
|
||||||
|
integration.artifact_transport,
|
||||||
|
PortableObservatoryArtifactTransport,
|
||||||
|
)
|
||||||
|
assert isinstance(
|
||||||
|
integration.result_publisher,
|
||||||
|
PortableObservatoryResultPublisher,
|
||||||
|
)
|
||||||
|
profiles = {
|
||||||
|
policy.setup_id: policy.display_name for policy in integration.calculation_profiles.policies
|
||||||
|
}
|
||||||
|
assert profiles == {
|
||||||
|
PORTABLE_LAB_V1_SETUP_ID: PORTABLE_LAB_V1_DISPLAY_NAME,
|
||||||
|
PORTABLE_M49_SETUP_ID: PORTABLE_M49_DISPLAY_NAME,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_server_integration_rejects_swapped_validator_identities(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
definitions = _definitions()
|
||||||
|
swapped = PortableResultContractValidatorRegistry(
|
||||||
|
(
|
||||||
|
PortableResultContractValidatorRegistration(
|
||||||
|
PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256,
|
||||||
|
validate_m49_portable_result,
|
||||||
|
),
|
||||||
|
PortableResultContractValidatorRegistration(
|
||||||
|
M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||||
|
validate_lab_v1_result_v2,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
source_cas_root = tmp_path / "source-cas"
|
||||||
|
result_staging_root = tmp_path / "result-staging"
|
||||||
|
source_cas_root.mkdir()
|
||||||
|
result_staging_root.mkdir()
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
PortableWorkerIntegrationError,
|
||||||
|
match="validator registration changed identity",
|
||||||
|
):
|
||||||
|
build_portable_observatory_worker_integration(
|
||||||
|
queue=cast(ObservatoryRecordedJobQueue, object()),
|
||||||
|
session_store=SessionStore(tmp_path / "repository"),
|
||||||
|
media_inspector=RecordedMediaInspector(tmp_path / "media-inspections"),
|
||||||
|
definitions=definitions,
|
||||||
|
artifact_store=CentralArtifactStore(tmp_path / "artifacts", create=True),
|
||||||
|
validators=swapped,
|
||||||
|
source_cas_root=source_cas_root,
|
||||||
|
result_staging_root=result_staging_root,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_storage_roots_load_only_from_existing_disjoint_central_directories(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
boundary = tmp_path / "nodedc-mission-core"
|
||||||
|
artifact_store = boundary / "artifact-store"
|
||||||
|
source_cas = boundary / "observatory-worker" / "source-cas"
|
||||||
|
result_staging = boundary / "observatory-worker" / "result-staging"
|
||||||
|
for path in (artifact_store, source_cas, result_staging):
|
||||||
|
path.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
roots = PortableWorkerStorageRoots.from_environment(
|
||||||
|
artifact_store_root=artifact_store,
|
||||||
|
environment={
|
||||||
|
OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV: str(source_cas),
|
||||||
|
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV: str(result_staging),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert roots == PortableWorkerStorageRoots(
|
||||||
|
source_cas_root=source_cas,
|
||||||
|
result_staging_root=result_staging,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_server_integration_has_no_data_directory_storage_fallback(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
store = SessionStore(tmp_path / "repository")
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
PortableWorkerIntegrationError,
|
||||||
|
match="storage roots must be explicitly configured",
|
||||||
|
):
|
||||||
|
build_portable_observatory_worker_integration(
|
||||||
|
queue=cast(ObservatoryRecordedJobQueue, object()),
|
||||||
|
session_store=store,
|
||||||
|
media_inspector=RecordedMediaInspector(tmp_path / "media-inspections"),
|
||||||
|
definitions=_definitions(),
|
||||||
|
artifact_store=CentralArtifactStore(tmp_path / "artifacts", create=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not (store.data_dir / "observatory-worker-source-cas").exists()
|
||||||
|
assert not (store.data_dir / "observatory-worker-result-staging").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_storage_root_loading_does_not_create_an_unavailable_mount_path(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
boundary = tmp_path / "nodedc-mission-core"
|
||||||
|
artifact_store = boundary / "artifact-store"
|
||||||
|
artifact_store.mkdir(parents=True)
|
||||||
|
missing_source = boundary / "observatory-worker" / "source-cas"
|
||||||
|
missing_result = boundary / "observatory-worker" / "result-staging"
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
PortableWorkerIntegrationError,
|
||||||
|
match="portable source CAS is unavailable",
|
||||||
|
):
|
||||||
|
PortableWorkerStorageRoots.from_environment(
|
||||||
|
artifact_store_root=artifact_store,
|
||||||
|
environment={
|
||||||
|
OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV: str(missing_source),
|
||||||
|
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV: str(missing_result),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not missing_source.exists()
|
||||||
|
assert not missing_result.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_storage_roots_reject_escape_overlap_and_symlink(tmp_path: Path) -> None:
|
||||||
|
boundary = tmp_path / "nodedc-mission-core"
|
||||||
|
artifact_store = boundary / "artifact-store"
|
||||||
|
result_staging = boundary / "observatory-worker" / "result-staging"
|
||||||
|
outside = tmp_path / "outside"
|
||||||
|
for path in (artifact_store, result_staging, outside):
|
||||||
|
path.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
with pytest.raises(PortableWorkerIntegrationError, match="must be inside"):
|
||||||
|
PortableWorkerStorageRoots.from_paths(
|
||||||
|
artifact_store_root=artifact_store,
|
||||||
|
source_cas_root=outside,
|
||||||
|
result_staging_root=result_staging,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(PortableWorkerIntegrationError, match="disjoint from"):
|
||||||
|
PortableWorkerStorageRoots.from_paths(
|
||||||
|
artifact_store_root=artifact_store,
|
||||||
|
source_cas_root=artifact_store,
|
||||||
|
result_staging_root=result_staging,
|
||||||
|
)
|
||||||
|
|
||||||
|
source_target = boundary / "observatory-worker" / "source-target"
|
||||||
|
source_target.mkdir(parents=True)
|
||||||
|
source_link = boundary / "observatory-worker" / "source-link"
|
||||||
|
source_link.symlink_to(source_target, target_is_directory=True)
|
||||||
|
with pytest.raises(PortableWorkerIntegrationError, match="canonical directory"):
|
||||||
|
PortableWorkerStorageRoots.from_paths(
|
||||||
|
artifact_store_root=artifact_store,
|
||||||
|
source_cas_root=source_link,
|
||||||
|
result_staging_root=result_staging,
|
||||||
|
)
|
||||||
@@ -0,0 +1,574 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from dataclasses import replace
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from k1link.observatory.portable_run_definitions import (
|
||||||
|
PortableExecutorAvailability,
|
||||||
|
PortableRunDefinitionRegistry,
|
||||||
|
canonical_sha256,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_worker_runtime import (
|
||||||
|
PortableWorkerAssetVerification,
|
||||||
|
PortableWorkerExecutorAdapter,
|
||||||
|
PortableWorkerExecutorSeal,
|
||||||
|
PortableWorkerLocalAssetBinding,
|
||||||
|
PortableWorkerResultDraft,
|
||||||
|
PortableWorkerRuntimeAdmission,
|
||||||
|
PortableWorkerRuntimePhase,
|
||||||
|
PortableWorkerRuntimePlan,
|
||||||
|
PortableWorkerRuntimeRegistry,
|
||||||
|
PortableWorkerRuntimeRegistryError,
|
||||||
|
PortableWorkerRuntimeUnavailableError,
|
||||||
|
PortableWorkerSourceStage,
|
||||||
|
inspect_runtime_candidate,
|
||||||
|
)
|
||||||
|
from k1link.observatory.worker_agent import (
|
||||||
|
ObservatoryWorkerExecutionResult,
|
||||||
|
ObservatoryWorkerExecutorIdentity,
|
||||||
|
SealedObservatoryRecordedJob,
|
||||||
|
)
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
DEFINITION_REGISTRY = (
|
||||||
|
REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||||
|
)
|
||||||
|
RUNTIME_REGISTRY = (
|
||||||
|
REPOSITORY_ROOT / "config" / "observatory-worker-runtime-candidates.json"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _definitions() -> PortableRunDefinitionRegistry:
|
||||||
|
return PortableRunDefinitionRegistry.from_file(DEFINITION_REGISTRY)
|
||||||
|
|
||||||
|
|
||||||
|
def _runtime() -> PortableWorkerRuntimeRegistry:
|
||||||
|
return PortableWorkerRuntimeRegistry.from_file(
|
||||||
|
RUNTIME_REGISTRY,
|
||||||
|
definitions=_definitions(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _all_keys(value: object) -> set[str]:
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return set(value) | {
|
||||||
|
nested
|
||||||
|
for child in value.values()
|
||||||
|
for nested in _all_keys(child)
|
||||||
|
}
|
||||||
|
if isinstance(value, list):
|
||||||
|
return {nested for child in value for nested in _all_keys(child)}
|
||||||
|
return set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_production_candidates_bind_exact_definitions_but_remain_blocked() -> None:
|
||||||
|
registry = _runtime()
|
||||||
|
|
||||||
|
assert {candidate.setup_id for candidate in registry.candidates} == {
|
||||||
|
"lab-v1-eomt-ddrnet-portable-v1",
|
||||||
|
"m49-tgs-portable-v2",
|
||||||
|
}
|
||||||
|
for candidate in registry.candidates:
|
||||||
|
assert candidate.ready is False
|
||||||
|
assert candidate.executor is None
|
||||||
|
assert "executor-release-unsealed" in candidate.blockers
|
||||||
|
assert any(phase.state == "missing" for phase in candidate.phases)
|
||||||
|
with pytest.raises(
|
||||||
|
PortableWorkerRuntimeUnavailableError,
|
||||||
|
match="no executor identity",
|
||||||
|
):
|
||||||
|
candidate.executor_identity()
|
||||||
|
|
||||||
|
|
||||||
|
def test_candidate_contract_is_source_independent_and_instruction_free() -> None:
|
||||||
|
document = json.loads(RUNTIME_REGISTRY.read_text(encoding="utf-8"))
|
||||||
|
serialized = json.dumps(document, sort_keys=True)
|
||||||
|
keys = _all_keys(document)
|
||||||
|
|
||||||
|
assert "RAVNOVES" not in serialized
|
||||||
|
assert "source_session_id" not in serialized
|
||||||
|
assert "filesystem" not in serialized
|
||||||
|
assert not {"command", "commands", "argv", "env", "environment"} & keys
|
||||||
|
assert not any("priority" in key for key in keys)
|
||||||
|
|
||||||
|
|
||||||
|
def test_m49_reuses_portable_profile_generic_runner_and_travel_image() -> None:
|
||||||
|
candidate = _runtime().resolve(
|
||||||
|
"m49-tgs-portable-v2",
|
||||||
|
"73611f24d70319ea1edca428726d6538a3cbad012a415cc0c1a7ecb7d9b4d910",
|
||||||
|
)
|
||||||
|
bindings = {
|
||||||
|
"m49-portable-profile": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="m49-portable-profile",
|
||||||
|
file_path=REPOSITORY_ROOT / "config" / "perception" / "m49-tgs-portable-v2.json",
|
||||||
|
),
|
||||||
|
"m49-portable-runner-source": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="m49-portable-runner-source",
|
||||||
|
file_path=(
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "observatory_portable"
|
||||||
|
/ "run_m49_tgs_portable.cpp"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
"m49-portable-runner-manifest": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="m49-portable-runner-manifest",
|
||||||
|
file_path=(
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "observatory_portable"
|
||||||
|
/ "m49-tgs-portable-runner-source.json"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
"m49-portable-runner-wrapper": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="m49-portable-runner-wrapper",
|
||||||
|
file_path=(
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "observatory_portable"
|
||||||
|
/ "run_m49_tgs_portable.sh"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
"m49-portable-smoke": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="m49-portable-smoke",
|
||||||
|
file_path=(
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "observatory_portable"
|
||||||
|
/ "smoke_m49_tgs_portable.sh"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
"travel-tgs-image": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="travel-tgs-image",
|
||||||
|
image_sha256=(
|
||||||
|
"7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
admission = inspect_runtime_candidate(candidate, bindings)
|
||||||
|
|
||||||
|
assert {item.state for item in admission.assets} == {"matched"}
|
||||||
|
assert admission.ready is False
|
||||||
|
assert "portable-tgs-runner-unsealed" in admission.blockers
|
||||||
|
assert "portable-camera-lidar-timeline-unimplemented" in admission.blockers
|
||||||
|
assert "portable-result-assembler-unimplemented" in admission.blockers
|
||||||
|
|
||||||
|
|
||||||
|
def test_m49_portable_runner_has_no_exact_source_or_frame_count_binding() -> None:
|
||||||
|
source = (
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "observatory_portable"
|
||||||
|
/ "run_m49_tgs_portable.cpp"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
wrapper = (
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "observatory_portable"
|
||||||
|
/ "run_m49_tgs_portable.sh"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "RAVNOVES" not in source + wrapper
|
||||||
|
assert "4489" not in source + wrapper
|
||||||
|
assert "3928" not in source + wrapper
|
||||||
|
assert "schedule.rows.size()" in source
|
||||||
|
assert "verifySequence(sequence_dir, schedule.available_count)" in source
|
||||||
|
assert "std::vector<float> values(point_count * 4)" in source
|
||||||
|
assert "--network" not in wrapper
|
||||||
|
|
||||||
|
|
||||||
|
def test_m49_portable_runner_source_release_is_content_addressed() -> None:
|
||||||
|
root = (
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "observatory_portable"
|
||||||
|
)
|
||||||
|
manifest = json.loads(
|
||||||
|
(root / "m49-tgs-portable-runner-source.json").read_text(encoding="utf-8")
|
||||||
|
)
|
||||||
|
expected_release_sha256 = manifest.pop("source_release_sha256")
|
||||||
|
actual_release_sha256 = hashlib.sha256(
|
||||||
|
json.dumps(
|
||||||
|
manifest,
|
||||||
|
ensure_ascii=False,
|
||||||
|
allow_nan=False,
|
||||||
|
separators=(",", ":"),
|
||||||
|
sort_keys=True,
|
||||||
|
).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
assert actual_release_sha256 == expected_release_sha256
|
||||||
|
assert manifest["input_contract"]["timeline_frame_count"] == "source-derived"
|
||||||
|
assert manifest["input_contract"]["available_lidar_frame_count"] == "source-derived"
|
||||||
|
for item in manifest["files"]:
|
||||||
|
path = root / item["name"]
|
||||||
|
assert path.stat().st_size == item["byte_length"]
|
||||||
|
assert hashlib.sha256(path.read_bytes()).hexdigest() == item["sha256"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_lab_candidate_verifies_reusable_repository_assets_without_claiming_executor() -> None:
|
||||||
|
candidate = _runtime().resolve(
|
||||||
|
"lab-v1-eomt-ddrnet-portable-v1",
|
||||||
|
"3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2",
|
||||||
|
)
|
||||||
|
bindings = {
|
||||||
|
"ddrnet-goose-image": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="ddrnet-goose-image",
|
||||||
|
image_sha256=(
|
||||||
|
"591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
"ddrnet-goose-runner": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="ddrnet-goose-runner",
|
||||||
|
file_path=(
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "lab_v1_vegetation_goose"
|
||||||
|
/ "run_goose_vegetation_benchmark.py"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
"eomt-image": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="eomt-image",
|
||||||
|
image_sha256=(
|
||||||
|
"58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
"eomt-orchestrator": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="eomt-orchestrator",
|
||||||
|
file_path=(
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "Invoke-E4FullSessionSegmentation.ps1"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
"eomt-profile": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="eomt-profile",
|
||||||
|
file_path=(
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "e3_k1_camera1_profile.json"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
"eomt-runner": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="eomt-runner",
|
||||||
|
file_path=(
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "run_e4_full_session_segmentation.py"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
"vegetation-policy": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="vegetation-policy",
|
||||||
|
file_path=(
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "config"
|
||||||
|
/ "perception"
|
||||||
|
/ "lab-v1-vegetation-mission-policy-v1.json"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
"vegetation-provider-map": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="vegetation-provider-map",
|
||||||
|
file_path=(
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "config"
|
||||||
|
/ "perception"
|
||||||
|
/ "lab-v1-vegetation-provider-label-map-v1.json"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
admission = inspect_runtime_candidate(candidate, bindings)
|
||||||
|
states = {item.asset_id: item.state for item in admission.assets}
|
||||||
|
|
||||||
|
assert states["ddrnet-goose-image"] == "matched"
|
||||||
|
assert states["ddrnet-goose-runner"] == "matched"
|
||||||
|
assert states["eomt-image"] == "matched"
|
||||||
|
assert states["eomt-orchestrator"] == "matched"
|
||||||
|
assert states["eomt-profile"] == "matched"
|
||||||
|
assert states["eomt-runner"] == "matched"
|
||||||
|
assert states["ddrnet-portable-config"] == "missing"
|
||||||
|
assert states["ddrnet-checkpoint"] == "missing"
|
||||||
|
assert states["eomt-model-weights"] == "missing"
|
||||||
|
assert admission.ready is False
|
||||||
|
assert "ddrnet-component-port-uninstalled" in admission.blockers
|
||||||
|
assert "worker-installation-receipt-unavailable" in admission.blockers
|
||||||
|
|
||||||
|
|
||||||
|
def test_local_asset_tampering_is_reported_without_execution(tmp_path: Path) -> None:
|
||||||
|
candidate = _runtime().resolve(
|
||||||
|
"m49-tgs-portable-v2",
|
||||||
|
"73611f24d70319ea1edca428726d6538a3cbad012a415cc0c1a7ecb7d9b4d910",
|
||||||
|
)
|
||||||
|
tampered = tmp_path / "m49-profile.json"
|
||||||
|
tampered.write_text("{}\n", encoding="utf-8")
|
||||||
|
|
||||||
|
admission = inspect_runtime_candidate(
|
||||||
|
candidate,
|
||||||
|
{
|
||||||
|
"m49-portable-profile": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="m49-portable-profile",
|
||||||
|
file_path=tampered,
|
||||||
|
),
|
||||||
|
"travel-tgs-image": PortableWorkerLocalAssetBinding(
|
||||||
|
asset_id="travel-tgs-image",
|
||||||
|
image_sha256="7" * 64,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
states = {item.asset_id: item.state for item in admission.assets}
|
||||||
|
assert states["m49-portable-profile"] == "mismatched"
|
||||||
|
assert states["travel-tgs-image"] == "mismatched"
|
||||||
|
assert states["m49-portable-runner-source"] == "missing"
|
||||||
|
assert states["m49-portable-runner-manifest"] == "missing"
|
||||||
|
assert states["m49-portable-runner-wrapper"] == "missing"
|
||||||
|
assert "asset-m49-portable-profile-mismatched" in admission.blockers
|
||||||
|
assert "asset-travel-tgs-image-mismatched" in admission.blockers
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_registry_rejects_digest_drift_and_executable_instructions(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
document = json.loads(RUNTIME_REGISTRY.read_text(encoding="utf-8"))
|
||||||
|
document["candidates"][0]["candidate_sha256"] = "f" * 64
|
||||||
|
drifted = tmp_path / "drifted.json"
|
||||||
|
drifted.write_text(json.dumps(document), encoding="utf-8")
|
||||||
|
|
||||||
|
with pytest.raises(PortableWorkerRuntimeRegistryError, match="digest changed"):
|
||||||
|
PortableWorkerRuntimeRegistry.from_file(drifted, definitions=_definitions())
|
||||||
|
|
||||||
|
document = json.loads(RUNTIME_REGISTRY.read_text(encoding="utf-8"))
|
||||||
|
document["candidates"][0]["command"] = "run-anything"
|
||||||
|
unsafe = tmp_path / "unsafe.json"
|
||||||
|
unsafe.write_text(json.dumps(document), encoding="utf-8")
|
||||||
|
|
||||||
|
with pytest.raises(PortableWorkerRuntimeRegistryError, match="forbids 'command'"):
|
||||||
|
PortableWorkerRuntimeRegistry.from_file(unsafe, definitions=_definitions())
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_plan_is_identity_only_and_observation_only() -> None:
|
||||||
|
plan = PortableWorkerRuntimePlan(
|
||||||
|
job_id="observatory-run-" + ("a" * 32),
|
||||||
|
adapter_id="m49-tgs-worker006-portable-v2",
|
||||||
|
candidate_sha256="1" * 64,
|
||||||
|
setup_id="m49-tgs-portable-v2",
|
||||||
|
definition_sha256="2" * 64,
|
||||||
|
source_bundle_sha256="3" * 64,
|
||||||
|
source_capability_manifest_sha256="4" * 64,
|
||||||
|
result_contract_sha256="5" * 64,
|
||||||
|
phases=("source-delivery", "portable-tgs-runner"),
|
||||||
|
).as_dict()
|
||||||
|
|
||||||
|
assert plan["authority"] == {
|
||||||
|
"commands_enabled": False,
|
||||||
|
"actuation_allowed": False,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
"production_accepted": False,
|
||||||
|
}
|
||||||
|
serialized = json.dumps(plan, sort_keys=True)
|
||||||
|
keys = _all_keys(plan)
|
||||||
|
assert not {"command", "commands", "argv", "env", "environment", "path"} & keys
|
||||||
|
assert not any("priority" in key for key in keys)
|
||||||
|
assert "D:\\" not in serialized
|
||||||
|
|
||||||
|
|
||||||
|
def test_ready_local_adapter_composes_only_local_ports_and_exact_job(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
definitions = _definitions()
|
||||||
|
base_definition = definitions.resolve_setup("lab-v1-eomt-ddrnet-portable-v1")
|
||||||
|
executor_availability = PortableExecutorAvailability(
|
||||||
|
contour_id="worker-006",
|
||||||
|
state="ready",
|
||||||
|
release_id="lab-v1-portable-executor-v1",
|
||||||
|
release_sha256="1" * 64,
|
||||||
|
image_sha256="2" * 64,
|
||||||
|
reason_code=None,
|
||||||
|
reason=None,
|
||||||
|
)
|
||||||
|
definition_identity = base_definition.identity_document()
|
||||||
|
definition_identity["executor"] = executor_availability.identity_document()
|
||||||
|
definition = replace(
|
||||||
|
base_definition,
|
||||||
|
executor=executor_availability,
|
||||||
|
definition_sha256=canonical_sha256(definition_identity),
|
||||||
|
)
|
||||||
|
|
||||||
|
blocked = _runtime().resolve(
|
||||||
|
"lab-v1-eomt-ddrnet-portable-v1",
|
||||||
|
base_definition.definition_sha256,
|
||||||
|
)
|
||||||
|
executor_seal = PortableWorkerExecutorSeal(
|
||||||
|
release_id="lab-v1-portable-executor-v1",
|
||||||
|
release_sha256="1" * 64,
|
||||||
|
image_sha256="2" * 64,
|
||||||
|
)
|
||||||
|
phases = tuple(
|
||||||
|
PortableWorkerRuntimePhase(phase.phase_id, "implemented")
|
||||||
|
for phase in blocked.phases
|
||||||
|
)
|
||||||
|
candidate_identity = blocked.identity_document()
|
||||||
|
candidate_identity["definition_sha256"] = definition.definition_sha256
|
||||||
|
candidate_identity["state"] = "ready"
|
||||||
|
candidate_identity["executor"] = executor_seal.as_dict()
|
||||||
|
candidate_identity["phases"] = [phase.as_dict() for phase in phases]
|
||||||
|
candidate_identity["blockers"] = []
|
||||||
|
candidate = replace(
|
||||||
|
blocked,
|
||||||
|
definition_sha256=definition.definition_sha256,
|
||||||
|
state="ready",
|
||||||
|
executor=executor_seal,
|
||||||
|
phases=phases,
|
||||||
|
blockers=(),
|
||||||
|
candidate_sha256=canonical_sha256(candidate_identity),
|
||||||
|
)
|
||||||
|
admission = PortableWorkerRuntimeAdmission(
|
||||||
|
candidate_sha256=candidate.candidate_sha256,
|
||||||
|
ready=True,
|
||||||
|
blockers=(),
|
||||||
|
assets=tuple(
|
||||||
|
PortableWorkerAssetVerification(asset.asset_id, "matched", None)
|
||||||
|
for asset in candidate.reusable_assets
|
||||||
|
),
|
||||||
|
)
|
||||||
|
source_root = tmp_path / "source"
|
||||||
|
result_root = tmp_path / "result"
|
||||||
|
source_root.mkdir()
|
||||||
|
result_root.mkdir()
|
||||||
|
result_id = "portable-result-001"
|
||||||
|
result_sha256 = "9" * 64
|
||||||
|
|
||||||
|
class SourceMaterializer:
|
||||||
|
def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage:
|
||||||
|
return PortableWorkerSourceStage(
|
||||||
|
root=source_root,
|
||||||
|
source_bundle_sha256=job.source_bundle_sha256,
|
||||||
|
source_capability_manifest_sha256=(
|
||||||
|
job.source_capability_manifest_sha256
|
||||||
|
),
|
||||||
|
source_adapter_sha256=job.source_adapter_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
class Runner:
|
||||||
|
def run(
|
||||||
|
self,
|
||||||
|
plan: PortableWorkerRuntimePlan,
|
||||||
|
source: PortableWorkerSourceStage,
|
||||||
|
) -> PortableWorkerResultDraft:
|
||||||
|
assert source.root == source_root
|
||||||
|
assert plan.definition_sha256 == definition.definition_sha256
|
||||||
|
return PortableWorkerResultDraft(
|
||||||
|
root=result_root,
|
||||||
|
result_id=result_id,
|
||||||
|
result_sha256=result_sha256,
|
||||||
|
result_contract_sha256=candidate.result_contract_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
class Publisher:
|
||||||
|
def publish(
|
||||||
|
self,
|
||||||
|
job: SealedObservatoryRecordedJob,
|
||||||
|
draft: PortableWorkerResultDraft,
|
||||||
|
) -> ObservatoryWorkerExecutionResult:
|
||||||
|
assert job.source_session_id == "20260831T120000Z_viewer_live"
|
||||||
|
return ObservatoryWorkerExecutionResult(
|
||||||
|
result_id=draft.result_id,
|
||||||
|
result_sha256=draft.result_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
adapter = PortableWorkerExecutorAdapter(
|
||||||
|
candidate=candidate,
|
||||||
|
definition=definition,
|
||||||
|
admission=admission,
|
||||||
|
source_materializer=SourceMaterializer(),
|
||||||
|
runner=Runner(),
|
||||||
|
publisher=Publisher(),
|
||||||
|
)
|
||||||
|
with pytest.raises(
|
||||||
|
PortableWorkerRuntimeUnavailableError,
|
||||||
|
match="does not prove every candidate asset",
|
||||||
|
):
|
||||||
|
PortableWorkerExecutorAdapter(
|
||||||
|
candidate=candidate,
|
||||||
|
definition=definition,
|
||||||
|
admission=replace(admission, assets=()),
|
||||||
|
source_materializer=SourceMaterializer(),
|
||||||
|
runner=Runner(),
|
||||||
|
publisher=Publisher(),
|
||||||
|
)
|
||||||
|
job = SealedObservatoryRecordedJob(
|
||||||
|
job_id="observatory-run-" + ("a" * 32),
|
||||||
|
request_sha256="3" * 64,
|
||||||
|
identity_sha256="4" * 64,
|
||||||
|
submission_receipt_sha256="c" * 64,
|
||||||
|
source_session_id="20260831T120000Z_viewer_live",
|
||||||
|
source_catalog_sha256="5" * 64,
|
||||||
|
source_bundle_sha256="6" * 64,
|
||||||
|
source_capability_manifest_sha256="7" * 64,
|
||||||
|
source_adapter_id=definition.source_adapter.adapter_id,
|
||||||
|
source_adapter_version=definition.source_adapter.version,
|
||||||
|
source_adapter_sha256=definition.source_adapter.contract_sha256,
|
||||||
|
setup_id=definition.setup_id,
|
||||||
|
definition_id=definition.definition_id,
|
||||||
|
definition_version=definition.version,
|
||||||
|
definition_sha256=definition.definition_sha256,
|
||||||
|
executor_release_id=executor_seal.release_id,
|
||||||
|
executor_identity=ObservatoryWorkerExecutorIdentity(
|
||||||
|
release_sha256=executor_seal.release_sha256,
|
||||||
|
image_sha256=executor_seal.image_sha256,
|
||||||
|
model_manifest_sha256=definition.model_manifest_sha256,
|
||||||
|
resource_profile_sha256=definition.resource_profile.profile_sha256,
|
||||||
|
),
|
||||||
|
model_release_ids=definition.learned_models,
|
||||||
|
resource_profile_id=definition.resource_profile.profile_id,
|
||||||
|
checkpoint_policy=definition.resource_profile.checkpoint_policy,
|
||||||
|
allowed_checkpoints=definition.resource_profile.allowed_checkpoints,
|
||||||
|
claim_generation=1,
|
||||||
|
claim_claimed_at_utc="2026-08-31T09:00:00.000Z",
|
||||||
|
claim_expires_at_utc="2026-08-31T09:05:00.000Z",
|
||||||
|
claim_heartbeat_at_utc="2026-08-31T09:00:00.000Z",
|
||||||
|
claim_renewal_count=0,
|
||||||
|
restart_from_zero=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert adapter.execute(job) == ObservatoryWorkerExecutionResult(
|
||||||
|
result_id=result_id,
|
||||||
|
result_sha256=result_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(PortableWorkerRuntimeUnavailableError):
|
||||||
|
PortableWorkerExecutorAdapter(
|
||||||
|
candidate=blocked,
|
||||||
|
definition=base_definition,
|
||||||
|
admission=replace(admission, candidate_sha256=blocked.candidate_sha256),
|
||||||
|
source_materializer=SourceMaterializer(),
|
||||||
|
runner=Runner(),
|
||||||
|
publisher=Publisher(),
|
||||||
|
)
|
||||||
@@ -350,6 +350,210 @@ def test_claim_is_exactly_idempotent_including_empty_result(tmp_path: Path) -> N
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_claim_lease_renews_idempotently_and_requeues_expired_unstarted_job(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
clock_value = [NOW]
|
||||||
|
queue = ObservatoryRecordedJobQueue(
|
||||||
|
tmp_path,
|
||||||
|
definitions=_definitions(),
|
||||||
|
clock=lambda: clock_value[0],
|
||||||
|
claim_lease_seconds=10,
|
||||||
|
)
|
||||||
|
job, _ = queue.submit(_intent(), enqueue=True)
|
||||||
|
claim = queue.claim_next(
|
||||||
|
claimant_id="recorded-worker",
|
||||||
|
claim_request_id="lease-poll-001",
|
||||||
|
)
|
||||||
|
assert claim is not None
|
||||||
|
assert claim.job.claimed_at_utc == NOW
|
||||||
|
assert claim.job.claim_heartbeat_at_utc == NOW
|
||||||
|
assert claim.job.claim_expires_at_utc == "2026-08-30T21:00:10.000Z"
|
||||||
|
assert claim.job.claim_renewal_count == 0
|
||||||
|
assert claim.job.as_dict()["claim_lease"] == {
|
||||||
|
"claimed_at_utc": NOW,
|
||||||
|
"expires_at_utc": "2026-08-30T21:00:10.000Z",
|
||||||
|
"heartbeat_at_utc": NOW,
|
||||||
|
"renewal_count": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
clock_value[0] = "2026-08-30T21:00:04.000Z"
|
||||||
|
renewed = queue.renew_claim(
|
||||||
|
job.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
claim_generation=claim.job.claim_generation,
|
||||||
|
heartbeat_sequence=1,
|
||||||
|
)
|
||||||
|
repeated = queue.renew_claim(
|
||||||
|
job.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
claim_generation=claim.job.claim_generation,
|
||||||
|
heartbeat_sequence=1,
|
||||||
|
)
|
||||||
|
assert renewed.claim_heartbeat_at_utc == clock_value[0]
|
||||||
|
assert renewed.claim_expires_at_utc == "2026-08-30T21:00:14.000Z"
|
||||||
|
assert renewed.claim_renewal_count == 1
|
||||||
|
assert repeated == renewed
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueConflictError, match="contiguous"):
|
||||||
|
queue.renew_claim(
|
||||||
|
job.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
claim_generation=claim.job.claim_generation,
|
||||||
|
heartbeat_sequence=3,
|
||||||
|
)
|
||||||
|
|
||||||
|
clock_value[0] = "2026-08-30T21:00:14.000Z"
|
||||||
|
recovered = queue.recover_stale_claims()
|
||||||
|
assert [item.job_id for item in recovered] == [job.job_id]
|
||||||
|
assert recovered[0].state == "queued"
|
||||||
|
assert recovered[0].active_claim_token is None
|
||||||
|
assert recovered[0].claim_expires_at_utc is None
|
||||||
|
assert recovered[0].claim_renewal_count == 0
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"):
|
||||||
|
queue.start(job.job_id, claim_token=claim.claim_token)
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"):
|
||||||
|
queue.succeed(
|
||||||
|
job.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
result_id="expired-worker-result",
|
||||||
|
result_sha256=RESULT_SHA,
|
||||||
|
)
|
||||||
|
|
||||||
|
replacement = queue.claim_next(
|
||||||
|
claimant_id="recorded-worker",
|
||||||
|
claim_request_id="lease-poll-002",
|
||||||
|
)
|
||||||
|
assert replacement is not None
|
||||||
|
assert replacement.job.job_id == job.job_id
|
||||||
|
assert replacement.job.claim_generation == claim.job.claim_generation + 1
|
||||||
|
assert replacement.claim_token != claim.claim_token
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="receipt"):
|
||||||
|
queue.claim_next(
|
||||||
|
claimant_id="recorded-worker",
|
||||||
|
claim_request_id="lease-poll-001",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_expired_running_claim_is_quarantined_and_stale_terminal_is_fenced(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
clock_value = [NOW]
|
||||||
|
queue = ObservatoryRecordedJobQueue(
|
||||||
|
tmp_path,
|
||||||
|
definitions=_definitions(),
|
||||||
|
clock=lambda: clock_value[0],
|
||||||
|
claim_lease_seconds=10,
|
||||||
|
)
|
||||||
|
running, claim = _running_job(queue)
|
||||||
|
|
||||||
|
clock_value[0] = "2026-08-30T21:00:10.000Z"
|
||||||
|
recovered = queue.recover_stale_claims()
|
||||||
|
assert [item.job_id for item in recovered] == [running.job_id]
|
||||||
|
quarantined = recovered[0]
|
||||||
|
assert quarantined.state == "reconciliation-required"
|
||||||
|
assert quarantined.terminal_code == "claim-lease-expired"
|
||||||
|
assert quarantined.active_claim_token is None
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"):
|
||||||
|
queue.succeed(
|
||||||
|
running.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
result_id="late-worker-result",
|
||||||
|
result_sha256=RESULT_SHA,
|
||||||
|
)
|
||||||
|
assert queue.get(running.job_id).result_id is None
|
||||||
|
assert (
|
||||||
|
queue.claim_next(
|
||||||
|
claimant_id="recorded-worker",
|
||||||
|
claim_request_id="blocked-after-expired-running",
|
||||||
|
)
|
||||||
|
is None
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_legacy_sqlite_claim_schema_migrates_without_reusing_old_token(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
job, _ = queue.submit(_intent(), enqueue=True)
|
||||||
|
claim = queue.claim_next(
|
||||||
|
claimant_id="recorded-worker",
|
||||||
|
claim_request_id="legacy-claim-001",
|
||||||
|
)
|
||||||
|
assert claim is not None
|
||||||
|
with sqlite3.connect(queue.database_path) as connection:
|
||||||
|
for column in (
|
||||||
|
"claimed_at_utc",
|
||||||
|
"claim_expires_at_utc",
|
||||||
|
"claim_heartbeat_at_utc",
|
||||||
|
"claim_renewal_count",
|
||||||
|
):
|
||||||
|
connection.execute(
|
||||||
|
f"ALTER TABLE observatory_recorded_jobs DROP COLUMN {column}"
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
|
||||||
|
migrated_queue = _queue(tmp_path)
|
||||||
|
migrated = migrated_queue.get(job.job_id)
|
||||||
|
assert migrated.state == "queued"
|
||||||
|
assert migrated.claim_generation == 1
|
||||||
|
assert migrated.active_claim_token is None
|
||||||
|
assert migrated.claimed_at_utc is None
|
||||||
|
assert migrated.claim_expires_at_utc is None
|
||||||
|
assert migrated.claim_heartbeat_at_utc is None
|
||||||
|
assert migrated.claim_renewal_count == 0
|
||||||
|
with sqlite3.connect(migrated_queue.database_path) as connection:
|
||||||
|
columns = {
|
||||||
|
row[1]
|
||||||
|
for row in connection.execute(
|
||||||
|
"PRAGMA table_info(observatory_recorded_jobs)"
|
||||||
|
).fetchall()
|
||||||
|
}
|
||||||
|
assert {
|
||||||
|
"claimed_at_utc",
|
||||||
|
"claim_expires_at_utc",
|
||||||
|
"claim_heartbeat_at_utc",
|
||||||
|
"claim_renewal_count",
|
||||||
|
}.issubset(columns)
|
||||||
|
replacement = migrated_queue.claim_next(
|
||||||
|
claimant_id="recorded-worker",
|
||||||
|
claim_request_id="legacy-claim-002",
|
||||||
|
)
|
||||||
|
assert replacement is not None
|
||||||
|
assert replacement.job.claim_generation == 2
|
||||||
|
assert replacement.claim_token != claim.claim_token
|
||||||
|
|
||||||
|
|
||||||
|
def test_legacy_sqlite_running_owner_migrates_to_reconciliation(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
running, claim = _running_job(queue)
|
||||||
|
with sqlite3.connect(queue.database_path) as connection:
|
||||||
|
for column in (
|
||||||
|
"claimed_at_utc",
|
||||||
|
"claim_expires_at_utc",
|
||||||
|
"claim_heartbeat_at_utc",
|
||||||
|
"claim_renewal_count",
|
||||||
|
):
|
||||||
|
connection.execute(
|
||||||
|
f"ALTER TABLE observatory_recorded_jobs DROP COLUMN {column}"
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
|
||||||
|
migrated_queue = _queue(tmp_path)
|
||||||
|
migrated = migrated_queue.get(running.job_id)
|
||||||
|
assert migrated.state == "reconciliation-required"
|
||||||
|
assert migrated.terminal_code == "claim-lease-migration"
|
||||||
|
assert migrated.active_claim_token is None
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"):
|
||||||
|
migrated_queue.succeed(
|
||||||
|
running.job_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
result_id="late-legacy-result",
|
||||||
|
result_sha256=RESULT_SHA,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_single_worker_resource_has_only_one_recorded_owner(tmp_path: Path) -> None:
|
def test_single_worker_resource_has_only_one_recorded_owner(tmp_path: Path) -> None:
|
||||||
queue = _queue(tmp_path)
|
queue = _queue(tmp_path)
|
||||||
first, _ = queue.submit(_intent())
|
first, _ = queue.submit(_intent())
|
||||||
|
|||||||
@@ -9,10 +9,15 @@ import pytest
|
|||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
from k1link.observatory import LaboratorySetupRegistry, LaboratorySetupRegistryError
|
from k1link.observatory import (
|
||||||
|
OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||||
|
LaboratorySetupRegistry,
|
||||||
|
LaboratorySetupRegistryError,
|
||||||
|
)
|
||||||
from k1link.sessions import LabReplayCapability, LabSessionBinding, SessionNotFoundError
|
from k1link.sessions import LabReplayCapability, LabSessionBinding, SessionNotFoundError
|
||||||
from k1link.sessions.models import SessionSummary
|
from k1link.sessions.models import SessionSummary
|
||||||
from k1link.web.observatory_api import build_observatory_router
|
from k1link.web.observatory_api import build_observatory_router
|
||||||
|
from k1link.web.session_api import build_session_router
|
||||||
|
|
||||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-laboratory-setups.json"
|
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-laboratory-setups.json"
|
||||||
@@ -163,6 +168,34 @@ def test_repository_setup_registry_keeps_real_definition_and_pre_definition_resu
|
|||||||
assert existing["preflight"]["submission_allowed"] is False
|
assert existing["preflight"]["submission_allowed"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_registry_attributes_only_the_exact_admitted_preserved_result() -> None:
|
||||||
|
registry = _registry()
|
||||||
|
projection = _rav004_projection()
|
||||||
|
|
||||||
|
assert registry.observatory_calculation_profile(projection) == {
|
||||||
|
"schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||||
|
"setup_id": "lab-v1-ravnoves004tree-final",
|
||||||
|
"display_name": "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||||
|
"origin": "existing-result",
|
||||||
|
"definition_id": None,
|
||||||
|
"definition_version": None,
|
||||||
|
"definition_sha256": None,
|
||||||
|
}
|
||||||
|
assert registry.observatory_calculation_profile(
|
||||||
|
replace(
|
||||||
|
projection,
|
||||||
|
session_id="lab-v1-vegetation-shadow-" + "0" * 64,
|
||||||
|
)
|
||||||
|
) is None
|
||||||
|
assert projection.lab is not None
|
||||||
|
assert registry.observatory_calculation_profile(
|
||||||
|
replace(
|
||||||
|
projection,
|
||||||
|
lab=replace(projection.lab, provenance={}),
|
||||||
|
)
|
||||||
|
) is None
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("source", "reason_code"),
|
("source", "reason_code"),
|
||||||
[
|
[
|
||||||
@@ -238,6 +271,54 @@ class _Store:
|
|||||||
raise SessionNotFoundError(session_id) from exc
|
raise SessionNotFoundError(session_id) from exc
|
||||||
return SimpleNamespace(summary=summary)
|
return SimpleNamespace(summary=summary)
|
||||||
|
|
||||||
|
def list_recent(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
limit: int,
|
||||||
|
cursor: str | None,
|
||||||
|
scope: str,
|
||||||
|
include_capability_projections: bool = False,
|
||||||
|
):
|
||||||
|
del limit, cursor
|
||||||
|
items = (
|
||||||
|
(_rav004_projection(),)
|
||||||
|
if scope == "laboratory" and include_capability_projections
|
||||||
|
else ()
|
||||||
|
)
|
||||||
|
return SimpleNamespace(items=items, next_cursor=None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_session_catalog_v3_projects_the_exact_preserved_profile() -> None:
|
||||||
|
registry = _registry()
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(
|
||||||
|
build_session_router(
|
||||||
|
_Store(), # type: ignore[arg-type]
|
||||||
|
lab_calculation_profile_resolver=registry.observatory_calculation_profile,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
v2_lab = client.get(
|
||||||
|
"/api/v1/observation-sessions",
|
||||||
|
params={"scope": "laboratory", "lab_contract": "v2"},
|
||||||
|
).json()["items"][0]["lab"]
|
||||||
|
v3_lab = client.get(
|
||||||
|
"/api/v1/observation-sessions",
|
||||||
|
params={"scope": "laboratory", "lab_contract": "v3"},
|
||||||
|
).json()["items"][0]["lab"]
|
||||||
|
|
||||||
|
assert "calculation_profile" not in v2_lab
|
||||||
|
assert v3_lab["calculation_profile"] == {
|
||||||
|
"schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||||
|
"setup_id": "lab-v1-ravnoves004tree-final",
|
||||||
|
"display_name": "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||||
|
"origin": "existing-result",
|
||||||
|
"definition_id": None,
|
||||||
|
"definition_version": None,
|
||||||
|
"definition_sha256": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_observatory_setup_catalog_and_preflight_are_read_only_and_fail_closed() -> None:
|
def test_observatory_setup_catalog_and_preflight_are_read_only_and_fail_closed() -> None:
|
||||||
app = FastAPI()
|
app = FastAPI()
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
from dataclasses import replace
|
from dataclasses import replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -334,6 +335,121 @@ def test_portable_source_admission_is_independent_from_session_label(
|
|||||||
assert capability["camera_profile"]["height"] == 600
|
assert capability["camera_profile"]["height"] == 600
|
||||||
|
|
||||||
|
|
||||||
|
def test_admission_seals_exact_catalogued_spatial_replay_metadata(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
session_id = "20260831T000000Z_viewer_live"
|
||||||
|
detail = _detail(session_id, "ignored-label")
|
||||||
|
metadata_path = tmp_path / session_id / "captures/mqtt_live/mqtt.metadata.jsonl"
|
||||||
|
metadata_path.parent.mkdir(parents=True)
|
||||||
|
metadata_payload = b'{"offset":0,"topic":"/points"}\n'
|
||||||
|
metadata_path.write_bytes(metadata_payload)
|
||||||
|
detail = replace(
|
||||||
|
detail,
|
||||||
|
artifacts=(
|
||||||
|
*detail.artifacts,
|
||||||
|
SessionArtifact(
|
||||||
|
artifact_id="raw-transport-index",
|
||||||
|
kind="raw-transport-index",
|
||||||
|
media_type="application/x-ndjson",
|
||||||
|
byte_length=len(metadata_payload),
|
||||||
|
sha256=None,
|
||||||
|
integrity_status="verified",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
store = _Store(tmp_path, detail)
|
||||||
|
store.replay = replace(
|
||||||
|
store.replay,
|
||||||
|
artifacts=(
|
||||||
|
*store.replay.artifacts,
|
||||||
|
ReplayArtifact(
|
||||||
|
artifact_id="raw-transport-index",
|
||||||
|
path=metadata_path,
|
||||||
|
media_type="application/x-ndjson",
|
||||||
|
file_byte_length=len(metadata_payload),
|
||||||
|
replay_byte_length=len(metadata_payload),
|
||||||
|
expected_sha256=None,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
service = RecordedK1SourceAdmissionService(
|
||||||
|
data_dir=tmp_path,
|
||||||
|
session_store=store, # type: ignore[arg-type]
|
||||||
|
media_inspector=_Inspector(_manifest(session_id, tmp_path)), # type: ignore[arg-type]
|
||||||
|
requirements=_requirements(),
|
||||||
|
)
|
||||||
|
|
||||||
|
admission = service.check(session_id)
|
||||||
|
|
||||||
|
bundle = json.loads(admission.source_bundle)
|
||||||
|
metadata = next(
|
||||||
|
member
|
||||||
|
for member in bundle["spatial_replay"]["members"]
|
||||||
|
if member["artifact_id"] == "raw-transport-index"
|
||||||
|
)
|
||||||
|
assert metadata == {
|
||||||
|
"artifact_id": "raw-transport-index",
|
||||||
|
"media_type": "application/x-ndjson",
|
||||||
|
"byte_length": len(metadata_payload),
|
||||||
|
"replay_byte_length": len(metadata_payload),
|
||||||
|
"sha256": hashlib.sha256(metadata_payload).hexdigest(),
|
||||||
|
}
|
||||||
|
assert "path" not in json.dumps(metadata, sort_keys=True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_admission_rejects_spatial_metadata_outside_session_root(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
session_id = "20260831T000000Z_viewer_live"
|
||||||
|
detail = _detail(session_id, "ignored-label")
|
||||||
|
(tmp_path / session_id).mkdir()
|
||||||
|
metadata_path = tmp_path / "outside" / "mqtt.metadata.jsonl"
|
||||||
|
metadata_path.parent.mkdir(parents=True)
|
||||||
|
metadata_path.write_bytes(b"{}\n")
|
||||||
|
detail = replace(
|
||||||
|
detail,
|
||||||
|
artifacts=(
|
||||||
|
*detail.artifacts,
|
||||||
|
SessionArtifact(
|
||||||
|
artifact_id="raw-transport-index",
|
||||||
|
kind="raw-transport-index",
|
||||||
|
media_type="application/x-ndjson",
|
||||||
|
byte_length=3,
|
||||||
|
sha256=None,
|
||||||
|
integrity_status="verified",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
store = _Store(tmp_path, detail)
|
||||||
|
store.replay = replace(
|
||||||
|
store.replay,
|
||||||
|
artifacts=(
|
||||||
|
*store.replay.artifacts,
|
||||||
|
ReplayArtifact(
|
||||||
|
artifact_id="raw-transport-index",
|
||||||
|
path=metadata_path,
|
||||||
|
media_type="application/x-ndjson",
|
||||||
|
file_byte_length=3,
|
||||||
|
replay_byte_length=3,
|
||||||
|
expected_sha256=None,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
service = RecordedK1SourceAdmissionService(
|
||||||
|
data_dir=tmp_path,
|
||||||
|
session_store=store, # type: ignore[arg-type]
|
||||||
|
media_inspector=_Inspector(_manifest(session_id, tmp_path)), # type: ignore[arg-type]
|
||||||
|
requirements=_requirements(),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
PortableSourceAdmissionIntegrityError,
|
||||||
|
match="escapes its admitted session root",
|
||||||
|
):
|
||||||
|
service.check(session_id)
|
||||||
|
|
||||||
|
|
||||||
def test_catalog_capability_check_restores_media_without_preparing_it(
|
def test_catalog_capability_check_restores_media_without_preparing_it(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ from k1link.observatory.worker_agent import (
|
|||||||
WORKER_006_CONTOUR_ID,
|
WORKER_006_CONTOUR_ID,
|
||||||
ObservatoryWorkerAgent,
|
ObservatoryWorkerAgent,
|
||||||
ObservatoryWorkerAgentBusyError,
|
ObservatoryWorkerAgentBusyError,
|
||||||
|
ObservatoryWorkerCycleReport,
|
||||||
ObservatoryWorkerExecutionResult,
|
ObservatoryWorkerExecutionResult,
|
||||||
ObservatoryWorkerExecutorIdentity,
|
ObservatoryWorkerExecutorIdentity,
|
||||||
ObservatoryWorkerExecutorRegistration,
|
ObservatoryWorkerExecutorRegistration,
|
||||||
@@ -105,6 +106,8 @@ class FakeTransport:
|
|||||||
starts: list[str] = field(default_factory=list)
|
starts: list[str] = field(default_factory=list)
|
||||||
successes: list[tuple[str, str, str]] = field(default_factory=list)
|
successes: list[tuple[str, str, str]] = field(default_factory=list)
|
||||||
failures: list[tuple[str, str, str]] = field(default_factory=list)
|
failures: list[tuple[str, str, str]] = field(default_factory=list)
|
||||||
|
renewals: list[int] = field(default_factory=list)
|
||||||
|
renewed: Event | None = None
|
||||||
|
|
||||||
def claim_next(
|
def claim_next(
|
||||||
self,
|
self,
|
||||||
@@ -132,6 +135,27 @@ class FakeTransport:
|
|||||||
self.starts.append(job_id)
|
self.starts.append(job_id)
|
||||||
return self.queue.start(job_id, claim_token=claim_token).as_dict()
|
return self.queue.start(job_id, claim_token=claim_token).as_dict()
|
||||||
|
|
||||||
|
def renew_claim(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
claimant_id: str,
|
||||||
|
job_id: str,
|
||||||
|
claim_token: str,
|
||||||
|
claim_generation: int,
|
||||||
|
heartbeat_sequence: int,
|
||||||
|
) -> Mapping[str, object]:
|
||||||
|
assert claimant_id == WORKER_006_CONTOUR_ID
|
||||||
|
renewed = self.queue.renew_claim(
|
||||||
|
job_id,
|
||||||
|
claim_token=claim_token,
|
||||||
|
claim_generation=claim_generation,
|
||||||
|
heartbeat_sequence=heartbeat_sequence,
|
||||||
|
).as_dict()
|
||||||
|
self.renewals.append(heartbeat_sequence)
|
||||||
|
if self.renewed is not None:
|
||||||
|
self.renewed.set()
|
||||||
|
return renewed
|
||||||
|
|
||||||
def succeed(
|
def succeed(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -228,12 +252,137 @@ def test_worker_agent_executes_one_sealed_allowlisted_job(tmp_path: Path) -> Non
|
|||||||
sealed_job = executor.jobs[0]
|
sealed_job = executor.jobs[0]
|
||||||
assert sealed_job.executor_identity == _identity()
|
assert sealed_job.executor_identity == _identity()
|
||||||
assert sealed_job.source_bundle_sha256 == SOURCE_BUNDLE_SHA
|
assert sealed_job.source_bundle_sha256 == SOURCE_BUNDLE_SHA
|
||||||
|
assert (
|
||||||
|
sealed_job.submission_receipt_sha256
|
||||||
|
== queue.get(job_id).submission_receipt_sha256
|
||||||
|
)
|
||||||
|
assert sealed_job.claim_expires_at_utc is not None
|
||||||
|
assert sealed_job.claim_heartbeat_at_utc is not None
|
||||||
|
assert sealed_job.claim_renewal_count == 0
|
||||||
assert not hasattr(sealed_job, "command")
|
assert not hasattr(sealed_job, "command")
|
||||||
assert not hasattr(sealed_job, "path")
|
assert not hasattr(sealed_job, "path")
|
||||||
assert not hasattr(sealed_job, "environment")
|
assert not hasattr(sealed_job, "environment")
|
||||||
assert queue.get(job_id).state == "succeeded"
|
assert queue.get(job_id).state == "succeeded"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class BlockingExecutor:
|
||||||
|
entered: Event
|
||||||
|
release: Event
|
||||||
|
|
||||||
|
def execute(
|
||||||
|
self,
|
||||||
|
_job: SealedObservatoryRecordedJob,
|
||||||
|
) -> ObservatoryWorkerExecutionResult:
|
||||||
|
self.entered.set()
|
||||||
|
assert self.release.wait(timeout=2)
|
||||||
|
return ObservatoryWorkerExecutionResult(
|
||||||
|
result_id="lab-v1-result",
|
||||||
|
result_sha256=RESULT_SHA,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _heartbeat_agent(
|
||||||
|
transport: FakeTransport,
|
||||||
|
executor: BlockingExecutor,
|
||||||
|
) -> ObservatoryWorkerAgent:
|
||||||
|
return ObservatoryWorkerAgent(
|
||||||
|
transport=transport,
|
||||||
|
executors=ObservatoryWorkerExecutorRegistry(
|
||||||
|
(
|
||||||
|
ObservatoryWorkerExecutorRegistration(
|
||||||
|
identity=_identity(),
|
||||||
|
adapter=executor,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
),
|
||||||
|
claim_request_id_factory=lambda: "worker-006:heartbeat-cycle",
|
||||||
|
heartbeat_interval_seconds=0.01,
|
||||||
|
heartbeat_stop_timeout_seconds=1.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_agent_renews_claim_through_executor_and_upload_window(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
job_id = _enqueue(queue)
|
||||||
|
entered = Event()
|
||||||
|
release = Event()
|
||||||
|
renewed = Event()
|
||||||
|
transport = FakeTransport(queue, renewed=renewed)
|
||||||
|
agent = _heartbeat_agent(
|
||||||
|
transport,
|
||||||
|
BlockingExecutor(entered=entered, release=release),
|
||||||
|
)
|
||||||
|
reports: list[ObservatoryWorkerCycleReport] = []
|
||||||
|
|
||||||
|
thread = Thread(target=lambda: reports.append(agent.run_once()))
|
||||||
|
thread.start()
|
||||||
|
assert entered.wait(timeout=2)
|
||||||
|
assert renewed.wait(timeout=2)
|
||||||
|
release.set()
|
||||||
|
thread.join(timeout=2)
|
||||||
|
|
||||||
|
assert not thread.is_alive()
|
||||||
|
assert len(reports) == 1
|
||||||
|
assert reports[0].state == "succeeded"
|
||||||
|
assert transport.renewals
|
||||||
|
assert transport.renewals == list(range(1, len(transport.renewals) + 1))
|
||||||
|
assert queue.get(job_id).state == "succeeded"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class FailingRenewTransport(FakeTransport):
|
||||||
|
renewal_attempted: Event = field(default_factory=Event)
|
||||||
|
|
||||||
|
def renew_claim(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
claimant_id: str,
|
||||||
|
job_id: str,
|
||||||
|
claim_token: str,
|
||||||
|
claim_generation: int,
|
||||||
|
heartbeat_sequence: int,
|
||||||
|
) -> Mapping[str, object]:
|
||||||
|
assert claimant_id == WORKER_006_CONTOUR_ID
|
||||||
|
assert job_id
|
||||||
|
assert claim_token
|
||||||
|
assert claim_generation == 1
|
||||||
|
assert heartbeat_sequence == 1
|
||||||
|
self.renewal_attempted.set()
|
||||||
|
raise RuntimeError("synthetic heartbeat transport loss")
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_agent_never_seals_result_after_heartbeat_loss(tmp_path: Path) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
job_id = _enqueue(queue)
|
||||||
|
entered = Event()
|
||||||
|
release = Event()
|
||||||
|
transport = FailingRenewTransport(queue)
|
||||||
|
agent = _heartbeat_agent(
|
||||||
|
transport,
|
||||||
|
BlockingExecutor(entered=entered, release=release),
|
||||||
|
)
|
||||||
|
reports: list[ObservatoryWorkerCycleReport] = []
|
||||||
|
|
||||||
|
thread = Thread(target=lambda: reports.append(agent.run_once()))
|
||||||
|
thread.start()
|
||||||
|
assert entered.wait(timeout=2)
|
||||||
|
assert transport.renewal_attempted.wait(timeout=2)
|
||||||
|
release.set()
|
||||||
|
thread.join(timeout=2)
|
||||||
|
|
||||||
|
assert not thread.is_alive()
|
||||||
|
assert len(reports) == 1
|
||||||
|
report = reports[0]
|
||||||
|
assert report.state == "lease-lost"
|
||||||
|
assert report.failure_code == "claim-heartbeat-lost"
|
||||||
|
assert transport.successes == []
|
||||||
|
assert transport.failures == []
|
||||||
|
assert queue.get(job_id).state == "running"
|
||||||
|
|
||||||
|
|
||||||
def test_worker_agent_leaves_empty_queue_untouched(tmp_path: Path) -> None:
|
def test_worker_agent_leaves_empty_queue_untouched(tmp_path: Path) -> None:
|
||||||
queue = _queue(tmp_path)
|
queue = _queue(tmp_path)
|
||||||
transport = FakeTransport(queue)
|
transport = FakeTransport(queue)
|
||||||
@@ -299,9 +448,31 @@ def _corrupt_job_identity(payload: dict[str, object]) -> dict[str, object]:
|
|||||||
return changed
|
return changed
|
||||||
|
|
||||||
|
|
||||||
|
def _remove_claim_lease(payload: dict[str, object]) -> dict[str, object]:
|
||||||
|
changed = deepcopy(payload)
|
||||||
|
job = changed["job"]
|
||||||
|
assert isinstance(job, dict)
|
||||||
|
job["claim_lease"] = None
|
||||||
|
return changed
|
||||||
|
|
||||||
|
|
||||||
|
def _corrupt_submission_receipt(payload: dict[str, object]) -> dict[str, object]:
|
||||||
|
changed = deepcopy(payload)
|
||||||
|
job = changed["job"]
|
||||||
|
assert isinstance(job, dict)
|
||||||
|
job["submission_receipt_sha256"] = "c" * 64
|
||||||
|
return changed
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"mutator",
|
"mutator",
|
||||||
[_spoof_claimant, _inject_unknown_execution_payload, _corrupt_job_identity],
|
[
|
||||||
|
_spoof_claimant,
|
||||||
|
_inject_unknown_execution_payload,
|
||||||
|
_corrupt_job_identity,
|
||||||
|
_corrupt_submission_receipt,
|
||||||
|
_remove_claim_lease,
|
||||||
|
],
|
||||||
)
|
)
|
||||||
def test_spoofed_unknown_or_corrupted_claim_is_rejected_without_execution(
|
def test_spoofed_unknown_or_corrupted_claim_is_rejected_without_execution(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
@@ -370,6 +541,17 @@ class BlockingEmptyTransport:
|
|||||||
) -> Mapping[str, object]:
|
) -> Mapping[str, object]:
|
||||||
raise AssertionError("an empty transport cannot start a job")
|
raise AssertionError("an empty transport cannot start a job")
|
||||||
|
|
||||||
|
def renew_claim(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
claimant_id: str,
|
||||||
|
job_id: str,
|
||||||
|
claim_token: str,
|
||||||
|
claim_generation: int,
|
||||||
|
heartbeat_sequence: int,
|
||||||
|
) -> Mapping[str, object]:
|
||||||
|
raise AssertionError("an empty transport cannot renew a job")
|
||||||
|
|
||||||
def succeed(
|
def succeed(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
|
|||||||
@@ -2,12 +2,15 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any, cast
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
from k1link.observatory.portable_artifact_transport import (
|
||||||
|
PortableArtifactTransportUnavailableError,
|
||||||
|
)
|
||||||
from k1link.observatory.recorded_jobs import (
|
from k1link.observatory.recorded_jobs import (
|
||||||
ObservatoryRecordedJobIntent,
|
ObservatoryRecordedJobIntent,
|
||||||
ObservatoryRecordedJobQueue,
|
ObservatoryRecordedJobQueue,
|
||||||
@@ -15,6 +18,8 @@ from k1link.observatory.recorded_jobs import (
|
|||||||
RecordedRunDefinitionRegistry,
|
RecordedRunDefinitionRegistry,
|
||||||
)
|
)
|
||||||
from k1link.web.observatory_worker_api import (
|
from k1link.web.observatory_worker_api import (
|
||||||
|
OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER,
|
||||||
|
OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
|
||||||
OBSERVATORY_WORKER_CONTOUR_HEADER,
|
OBSERVATORY_WORKER_CONTOUR_HEADER,
|
||||||
ObservatoryWorkerAuthentication,
|
ObservatoryWorkerAuthentication,
|
||||||
build_observatory_worker_router,
|
build_observatory_worker_router,
|
||||||
@@ -29,6 +34,7 @@ WORKER_HEADERS = {
|
|||||||
}
|
}
|
||||||
CLAIM_SCHEMA = "missioncore.observatory-worker-claim-request/v1"
|
CLAIM_SCHEMA = "missioncore.observatory-worker-claim-request/v1"
|
||||||
START_SCHEMA = "missioncore.observatory-worker-start-request/v1"
|
START_SCHEMA = "missioncore.observatory-worker-start-request/v1"
|
||||||
|
RENEW_SCHEMA = "missioncore.observatory-worker-renew-request/v1"
|
||||||
CHECKPOINT_SCHEMA = "missioncore.observatory-worker-checkpoint-request/v1"
|
CHECKPOINT_SCHEMA = "missioncore.observatory-worker-checkpoint-request/v1"
|
||||||
SUCCEED_SCHEMA = "missioncore.observatory-worker-succeed-request/v1"
|
SUCCEED_SCHEMA = "missioncore.observatory-worker-succeed-request/v1"
|
||||||
FAIL_SCHEMA = "missioncore.observatory-worker-fail-request/v1"
|
FAIL_SCHEMA = "missioncore.observatory-worker-fail-request/v1"
|
||||||
@@ -74,6 +80,33 @@ def _services(tmp_path: Path) -> tuple[TestClient, ObservatoryRecordedJobQueue]:
|
|||||||
return TestClient(app), queue
|
return TestClient(app), queue
|
||||||
|
|
||||||
|
|
||||||
|
class _ArtifactTransportWithoutCompletedPackage:
|
||||||
|
def require_completed_for_success(self, **_values: object) -> Path:
|
||||||
|
raise PortableArtifactTransportUnavailableError("package is incomplete")
|
||||||
|
|
||||||
|
|
||||||
|
def _services_with_artifact_transport(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> tuple[TestClient, ObservatoryRecordedJobQueue]:
|
||||||
|
definition = _definition()
|
||||||
|
queue = ObservatoryRecordedJobQueue(
|
||||||
|
tmp_path,
|
||||||
|
definitions=RecordedRunDefinitionRegistry((definition,)),
|
||||||
|
)
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(
|
||||||
|
build_observatory_worker_router(
|
||||||
|
queue,
|
||||||
|
authentication=ObservatoryWorkerAuthentication(
|
||||||
|
bearer_token_sha256=WORKER_TOKEN_SHA256,
|
||||||
|
contour_id="worker-006",
|
||||||
|
),
|
||||||
|
artifact_transport=_ArtifactTransportWithoutCompletedPackage(), # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return TestClient(app), queue
|
||||||
|
|
||||||
|
|
||||||
def _enqueue(
|
def _enqueue(
|
||||||
queue: ObservatoryRecordedJobQueue,
|
queue: ObservatoryRecordedJobQueue,
|
||||||
*,
|
*,
|
||||||
@@ -110,7 +143,7 @@ def _claim(
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
return response.json()
|
return cast(dict[str, Any], response.json())
|
||||||
|
|
||||||
|
|
||||||
def test_worker_authentication_requires_digest_and_configured_contour(
|
def test_worker_authentication_requires_digest_and_configured_contour(
|
||||||
@@ -280,6 +313,48 @@ def test_worker_can_start_checkpoint_and_read_job_but_stale_token_fails_closed(
|
|||||||
assert "active_claim_token" not in fetched.json()
|
assert "active_claim_token" not in fetched.json()
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_can_renew_exact_claim_lease_idempotently(tmp_path: Path) -> None:
|
||||||
|
client, queue = _services(tmp_path)
|
||||||
|
job_id = _enqueue(queue)
|
||||||
|
claim = _claim(client)
|
||||||
|
initial_lease = claim["job"]["claim_lease"]
|
||||||
|
assert initial_lease["renewal_count"] == 0
|
||||||
|
request = {
|
||||||
|
"schema_version": RENEW_SCHEMA,
|
||||||
|
"claim_token": claim["claim_token"],
|
||||||
|
"claim_generation": claim["job"]["claim_generation"],
|
||||||
|
"heartbeat_sequence": 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
renewed = client.post(
|
||||||
|
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/lease/renew",
|
||||||
|
headers=WORKER_HEADERS,
|
||||||
|
json=request,
|
||||||
|
)
|
||||||
|
repeated = client.post(
|
||||||
|
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/lease/renew",
|
||||||
|
headers=WORKER_HEADERS,
|
||||||
|
json=request,
|
||||||
|
)
|
||||||
|
stale_generation = client.post(
|
||||||
|
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/lease/renew",
|
||||||
|
headers=WORKER_HEADERS,
|
||||||
|
json={**request, "claim_generation": request["claim_generation"] + 1},
|
||||||
|
)
|
||||||
|
injected = client.post(
|
||||||
|
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/lease/renew",
|
||||||
|
headers=WORKER_HEADERS,
|
||||||
|
json={**request, "command": ["python", "untrusted.py"]},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert renewed.status_code == 200
|
||||||
|
assert renewed.json()["claim_lease"]["renewal_count"] == 1
|
||||||
|
assert repeated.status_code == 200
|
||||||
|
assert repeated.json() == renewed.json()
|
||||||
|
assert stale_generation.status_code == 409
|
||||||
|
assert injected.status_code == 422
|
||||||
|
|
||||||
|
|
||||||
def test_worker_can_publish_success_idempotently(tmp_path: Path) -> None:
|
def test_worker_can_publish_success_idempotently(tmp_path: Path) -> None:
|
||||||
client, queue = _services(tmp_path)
|
client, queue = _services(tmp_path)
|
||||||
job_id = _enqueue(queue)
|
job_id = _enqueue(queue)
|
||||||
@@ -328,6 +403,59 @@ def test_worker_can_publish_success_idempotently(tmp_path: Path) -> None:
|
|||||||
assert conflicting_failure.status_code == 409
|
assert conflicting_failure.status_code == 409
|
||||||
|
|
||||||
|
|
||||||
|
def test_artifact_transport_blocks_success_without_completed_package(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
client, queue = _services_with_artifact_transport(tmp_path)
|
||||||
|
job_id = _enqueue(queue)
|
||||||
|
claim = _claim(client)
|
||||||
|
claim_token = claim["claim_token"]
|
||||||
|
generation = claim["job"]["claim_generation"]
|
||||||
|
client.post(
|
||||||
|
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/start",
|
||||||
|
headers=WORKER_HEADERS,
|
||||||
|
json={"schema_version": START_SCHEMA, "claim_token": claim_token},
|
||||||
|
)
|
||||||
|
|
||||||
|
missing_claim_headers = client.get(
|
||||||
|
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/source-materialization",
|
||||||
|
headers=WORKER_HEADERS,
|
||||||
|
)
|
||||||
|
malformed_generation = client.get(
|
||||||
|
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/source-materialization",
|
||||||
|
headers={
|
||||||
|
**WORKER_HEADERS,
|
||||||
|
OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER: claim_token,
|
||||||
|
OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER: "0",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
malformed_claim_token = client.get(
|
||||||
|
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/source-materialization",
|
||||||
|
headers={
|
||||||
|
**WORKER_HEADERS,
|
||||||
|
OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER: "../../not-a-claim-token",
|
||||||
|
OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER: str(generation),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
blocked = client.post(
|
||||||
|
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/succeed",
|
||||||
|
headers=WORKER_HEADERS,
|
||||||
|
json={
|
||||||
|
"schema_version": SUCCEED_SCHEMA,
|
||||||
|
"claim_token": claim_token,
|
||||||
|
"result_id": "portable-result-without-upload",
|
||||||
|
"result_sha256": "b" * 64,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert generation == 1
|
||||||
|
assert missing_claim_headers.status_code == 422
|
||||||
|
assert malformed_generation.status_code == 422
|
||||||
|
assert malformed_claim_token.status_code == 422
|
||||||
|
assert blocked.status_code == 409
|
||||||
|
assert queue.get(job_id).state == "running"
|
||||||
|
|
||||||
|
|
||||||
def test_worker_can_fail_claimed_job_idempotently(tmp_path: Path) -> None:
|
def test_worker_can_fail_claimed_job_idempotently(tmp_path: Path) -> None:
|
||||||
client, queue = _services(tmp_path)
|
client, queue = _services(tmp_path)
|
||||||
job_id = _enqueue(queue)
|
job_id = _enqueue(queue)
|
||||||
|
|||||||
@@ -2,6 +2,14 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from k1link.observatory.m49_portable_result import (
|
||||||
|
M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||||
|
validate_m49_portable_result,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_lab_v1_executor import validate_lab_v1_result_v2
|
||||||
|
from k1link.observatory.portable_worker_integration import (
|
||||||
|
PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256,
|
||||||
|
)
|
||||||
from k1link.web import app as app_module
|
from k1link.web import app as app_module
|
||||||
|
|
||||||
WORKER_ROUTE_PREFIX = "/api/v1/worker/observatory"
|
WORKER_ROUTE_PREFIX = "/api/v1/worker/observatory"
|
||||||
@@ -9,12 +17,34 @@ WORKER_ROUTE_PREFIX = "/api/v1/worker/observatory"
|
|||||||
|
|
||||||
def test_worker_router_is_hard_disabled_until_lease_and_publisher_exist() -> None:
|
def test_worker_router_is_hard_disabled_until_lease_and_publisher_exist() -> None:
|
||||||
assert app_module.OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
assert app_module.OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||||
|
assert app_module.OBSERVATORY_PORTABLE_RESULT_VALIDATORS is not None
|
||||||
|
assert (
|
||||||
|
app_module.OBSERVATORY_PORTABLE_RESULT_VALIDATORS.resolve(
|
||||||
|
PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256
|
||||||
|
)
|
||||||
|
is validate_lab_v1_result_v2
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
app_module.OBSERVATORY_PORTABLE_RESULT_VALIDATORS.resolve(
|
||||||
|
M49_PORTABLE_RESULT_CONTRACT_SHA256
|
||||||
|
)
|
||||||
|
is validate_m49_portable_result
|
||||||
|
)
|
||||||
assert app_module.OBSERVATORY_WORKER_CLAIM_LEASE_READY is False
|
assert app_module.OBSERVATORY_WORKER_CLAIM_LEASE_READY is False
|
||||||
assert app_module.OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY is False
|
assert app_module.OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY is False
|
||||||
assert app_module.OBSERVATORY_WORKER_PRODUCTION_API_ENABLED is False
|
assert app_module.OBSERVATORY_WORKER_PRODUCTION_API_ENABLED is False
|
||||||
|
assert app_module.OBSERVATORY_WORKER_DISPATCH_READY is False
|
||||||
assert app_module.OBSERVATORY_WORKER_AUTHENTICATION is None
|
assert app_module.OBSERVATORY_WORKER_AUTHENTICATION is None
|
||||||
|
assert app_module.OBSERVATORY_WORKER_AUTHENTICATION_ERROR is not None
|
||||||
assert app_module.OBSERVATORY_WORKER_API_ERROR is not None
|
assert app_module.OBSERVATORY_WORKER_API_ERROR is not None
|
||||||
assert "hard-disabled" in app_module.OBSERVATORY_WORKER_API_ERROR
|
assert "hard-disabled" in app_module.OBSERVATORY_WORKER_API_ERROR
|
||||||
|
if app_module.session_artifact_gateway is None:
|
||||||
|
assert app_module.OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None
|
||||||
|
assert app_module.OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR is not None
|
||||||
|
assert (
|
||||||
|
"central artifact store"
|
||||||
|
in app_module.OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR
|
||||||
|
)
|
||||||
assert not any(
|
assert not any(
|
||||||
getattr(route, "path", "").startswith(WORKER_ROUTE_PREFIX)
|
getattr(route, "path", "").startswith(WORKER_ROUTE_PREFIX)
|
||||||
for route in app_module.app.routes
|
for route in app_module.app.routes
|
||||||
|
|||||||
@@ -0,0 +1,515 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from collections.abc import Callable
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from k1link.observatory.portable_artifact_transport import (
|
||||||
|
PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA,
|
||||||
|
PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA,
|
||||||
|
PORTABLE_SOURCE_MATERIALIZATION_SCHEMA,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_result_contract import (
|
||||||
|
OBSERVATION_ONLY_AUTHORITY,
|
||||||
|
PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
|
||||||
|
RESULT_DOCUMENT_ROLE,
|
||||||
|
PortableResultArtifact,
|
||||||
|
PortableResultPackageManifest,
|
||||||
|
canonical_json,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_run_definitions import canonical_sha256
|
||||||
|
from k1link.observatory.portable_worker_runtime import PortableWorkerResultDraft
|
||||||
|
from k1link.observatory.worker_agent import (
|
||||||
|
WORKER_006_CONTOUR_ID,
|
||||||
|
ObservatoryWorkerExecutorIdentity,
|
||||||
|
SealedObservatoryRecordedJob,
|
||||||
|
)
|
||||||
|
from k1link.observatory.worker_http_transport import (
|
||||||
|
ObservatoryWorkerHttpError,
|
||||||
|
ObservatoryWorkerHttpGateway,
|
||||||
|
)
|
||||||
|
|
||||||
|
JOB_ID = f"observatory-run-{'1' * 32}"
|
||||||
|
CLAIM_TOKEN = "2" * 64
|
||||||
|
BEARER_TOKEN = "worker-006-test-bearer-token-000001"
|
||||||
|
NOW = "2026-08-31T11:00:00.000Z"
|
||||||
|
|
||||||
|
|
||||||
|
def _job(*, bundle_sha256: str, capability_sha256: str) -> SealedObservatoryRecordedJob:
|
||||||
|
return SealedObservatoryRecordedJob(
|
||||||
|
job_id=JOB_ID,
|
||||||
|
request_sha256="3" * 64,
|
||||||
|
identity_sha256="4" * 64,
|
||||||
|
submission_receipt_sha256="c" * 64,
|
||||||
|
source_session_id="20260831T105500Z_viewer_live",
|
||||||
|
source_catalog_sha256="5" * 64,
|
||||||
|
source_bundle_sha256=bundle_sha256,
|
||||||
|
source_capability_manifest_sha256=capability_sha256,
|
||||||
|
source_adapter_id="sealed-session-source",
|
||||||
|
source_adapter_version=1,
|
||||||
|
source_adapter_sha256="6" * 64,
|
||||||
|
setup_id="portable-lab-v1",
|
||||||
|
definition_id="portable-lab-v1-definition",
|
||||||
|
definition_version=1,
|
||||||
|
definition_sha256="7" * 64,
|
||||||
|
executor_release_id="portable-lab-v1-worker",
|
||||||
|
executor_identity=ObservatoryWorkerExecutorIdentity(
|
||||||
|
release_sha256="8" * 64,
|
||||||
|
image_sha256="9" * 64,
|
||||||
|
model_manifest_sha256="a" * 64,
|
||||||
|
resource_profile_sha256="b" * 64,
|
||||||
|
),
|
||||||
|
model_release_ids=("eomt-cityscapes-large", "ddrnet-39"),
|
||||||
|
resource_profile_id="worker006-single-gpu",
|
||||||
|
checkpoint_policy="cooperative",
|
||||||
|
allowed_checkpoints=("semantic-pass",),
|
||||||
|
claim_generation=1,
|
||||||
|
claim_claimed_at_utc=NOW,
|
||||||
|
claim_expires_at_utc="2026-08-31T11:05:00.000Z",
|
||||||
|
claim_heartbeat_at_utc=NOW,
|
||||||
|
claim_renewal_count=0,
|
||||||
|
restart_from_zero=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _claim_response() -> httpx.Response:
|
||||||
|
return httpx.Response(
|
||||||
|
200,
|
||||||
|
json={
|
||||||
|
"claim_token": CLAIM_TOKEN,
|
||||||
|
"job": {"job_id": JOB_ID, "claim_generation": 1},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _cache_claim(gateway: ObservatoryWorkerHttpGateway) -> None:
|
||||||
|
payload = gateway.claim_next(
|
||||||
|
claimant_id=WORKER_006_CONTOUR_ID,
|
||||||
|
claim_request_id="worker-006:http-transport-test",
|
||||||
|
)
|
||||||
|
assert payload is not None
|
||||||
|
|
||||||
|
|
||||||
|
def _source_member(
|
||||||
|
job: SealedObservatoryRecordedJob,
|
||||||
|
*,
|
||||||
|
kind: str,
|
||||||
|
payload: bytes,
|
||||||
|
artifact_id: str | None = None,
|
||||||
|
primary: bool = False,
|
||||||
|
camera_epoch: int | None = None,
|
||||||
|
camera_sequence: int | None = None,
|
||||||
|
media_type: str,
|
||||||
|
) -> tuple[dict[str, object], bytes]:
|
||||||
|
sha256 = hashlib.sha256(payload).hexdigest()
|
||||||
|
identity = {
|
||||||
|
"job_identity_sha256": job.identity_sha256,
|
||||||
|
"source_bundle_sha256": job.source_bundle_sha256,
|
||||||
|
"kind": kind,
|
||||||
|
"artifact_id": artifact_id,
|
||||||
|
"primary": primary,
|
||||||
|
"camera_epoch": camera_epoch,
|
||||||
|
"camera_sequence": camera_sequence,
|
||||||
|
"media_type": media_type,
|
||||||
|
"byte_length": len(payload),
|
||||||
|
"sha256": sha256,
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
{
|
||||||
|
"member_id": hashlib.sha256(canonical_json(identity)).hexdigest(),
|
||||||
|
"kind": kind,
|
||||||
|
"media_type": media_type,
|
||||||
|
"byte_length": len(payload),
|
||||||
|
"sha256": sha256,
|
||||||
|
"artifact_id": artifact_id,
|
||||||
|
"primary": primary,
|
||||||
|
"camera_epoch": camera_epoch,
|
||||||
|
"camera_sequence": camera_sequence,
|
||||||
|
},
|
||||||
|
payload,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _source_contract(
|
||||||
|
job: SealedObservatoryRecordedJob,
|
||||||
|
*,
|
||||||
|
inject_path: bool = False,
|
||||||
|
) -> tuple[dict[str, object], dict[str, bytes]]:
|
||||||
|
rows = [
|
||||||
|
_source_member(
|
||||||
|
job,
|
||||||
|
kind="source-bundle",
|
||||||
|
payload=b"source-bundle",
|
||||||
|
media_type="application/json",
|
||||||
|
),
|
||||||
|
_source_member(
|
||||||
|
job,
|
||||||
|
kind="source-capability",
|
||||||
|
payload=b"source-capability",
|
||||||
|
media_type="application/json",
|
||||||
|
),
|
||||||
|
_source_member(
|
||||||
|
job,
|
||||||
|
kind="spatial-replay",
|
||||||
|
payload=b"sealed-raw-replay",
|
||||||
|
artifact_id="raw-primary",
|
||||||
|
primary=True,
|
||||||
|
media_type="application/x-nodedc-k1mqtt",
|
||||||
|
),
|
||||||
|
_source_member(
|
||||||
|
job,
|
||||||
|
kind="spatial-replay-metadata",
|
||||||
|
payload=b'{"offset":0,"topic":"/points"}\n',
|
||||||
|
artifact_id="raw-transport-index",
|
||||||
|
media_type="application/x-ndjson",
|
||||||
|
),
|
||||||
|
_source_member(
|
||||||
|
job,
|
||||||
|
kind="camera-init",
|
||||||
|
payload=b"sealed-camera-init",
|
||||||
|
artifact_id="recorded-camera-right",
|
||||||
|
camera_epoch=1,
|
||||||
|
media_type='video/mp4; codecs="avc1.641028"',
|
||||||
|
),
|
||||||
|
_source_member(
|
||||||
|
job,
|
||||||
|
kind="camera-segment",
|
||||||
|
payload=b"sealed-camera-segment",
|
||||||
|
artifact_id="recorded-camera-right",
|
||||||
|
camera_epoch=1,
|
||||||
|
camera_sequence=1,
|
||||||
|
media_type="video/iso.segment",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
members = [row for row, _payload in rows]
|
||||||
|
members.sort(key=lambda row: str(row["member_id"]))
|
||||||
|
if inject_path:
|
||||||
|
members[0]["path"] = "../../operator-secret"
|
||||||
|
payloads = {str(row["member_id"]): payload for row, payload in rows}
|
||||||
|
return (
|
||||||
|
{
|
||||||
|
"schema_version": PORTABLE_SOURCE_MATERIALIZATION_SCHEMA,
|
||||||
|
"job_id": job.job_id,
|
||||||
|
"job_identity_sha256": job.identity_sha256,
|
||||||
|
"claim_generation": job.claim_generation,
|
||||||
|
"source": {
|
||||||
|
"session_id": job.source_session_id,
|
||||||
|
"bundle_sha256": job.source_bundle_sha256,
|
||||||
|
"capability_manifest_sha256": (
|
||||||
|
job.source_capability_manifest_sha256
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"members": members,
|
||||||
|
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||||
|
},
|
||||||
|
payloads,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_http_gateway_materializes_only_exact_claim_bound_members(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
bundle_payload = b"source-bundle"
|
||||||
|
capability_payload = b"source-capability"
|
||||||
|
job = _job(
|
||||||
|
bundle_sha256=hashlib.sha256(bundle_payload).hexdigest(),
|
||||||
|
capability_sha256=hashlib.sha256(capability_payload).hexdigest(),
|
||||||
|
)
|
||||||
|
manifest, payloads = _source_contract(job)
|
||||||
|
artifact_requests: list[httpx.Request] = []
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
if request.url.path.endswith("/claims"):
|
||||||
|
assert request.headers["authorization"] == f"Bearer {BEARER_TOKEN}"
|
||||||
|
assert request.headers["x-mission-core-contour-id"] == "worker-006"
|
||||||
|
return _claim_response()
|
||||||
|
assert request.headers["x-mission-core-claim-token"] == CLAIM_TOKEN
|
||||||
|
assert request.headers["x-mission-core-claim-generation"] == "1"
|
||||||
|
artifact_requests.append(request)
|
||||||
|
if request.url.path.endswith("/source-materialization"):
|
||||||
|
return httpx.Response(200, json=manifest)
|
||||||
|
member_id = request.url.path.rsplit("/", 1)[-1]
|
||||||
|
payload = payloads[member_id]
|
||||||
|
return httpx.Response(
|
||||||
|
200,
|
||||||
|
content=payload,
|
||||||
|
headers={
|
||||||
|
"X-Mission-Core-Content-Sha256": hashlib.sha256(payload).hexdigest()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
with ObservatoryWorkerHttpGateway(
|
||||||
|
base_url="http://127.0.0.1:18080",
|
||||||
|
bearer_token=BEARER_TOKEN,
|
||||||
|
work_root=tmp_path / "worker",
|
||||||
|
transport=httpx.MockTransport(handler),
|
||||||
|
) as gateway:
|
||||||
|
_cache_claim(gateway)
|
||||||
|
stage = gateway.materialize(job)
|
||||||
|
|
||||||
|
assert (stage.root / "source-bundle.json").read_bytes() == bundle_payload
|
||||||
|
assert (stage.root / "source-capability.json").read_bytes() == capability_payload
|
||||||
|
assert (stage.root / "mqtt.raw.k1mqtt").read_bytes() == b"sealed-raw-replay"
|
||||||
|
assert (stage.root / "mqtt.metadata.jsonl").read_bytes() == (
|
||||||
|
b'{"offset":0,"topic":"/points"}\n'
|
||||||
|
)
|
||||||
|
assert (stage.root / "camera/epoch-1/init.mp4").read_bytes() == b"sealed-camera-init"
|
||||||
|
assert (
|
||||||
|
stage.root / "camera/epoch-1/segments/1.m4s"
|
||||||
|
).read_bytes() == b"sealed-camera-segment"
|
||||||
|
persisted = json.loads(
|
||||||
|
(stage.root / "materialization-manifest.json").read_text(encoding="utf-8")
|
||||||
|
)
|
||||||
|
assert persisted == manifest
|
||||||
|
assert "path" not in json.dumps(persisted, sort_keys=True)
|
||||||
|
assert len(artifact_requests) == 1 + len(payloads)
|
||||||
|
|
||||||
|
|
||||||
|
def test_http_gateway_rejects_server_selected_source_path(tmp_path: Path) -> None:
|
||||||
|
job = _job(
|
||||||
|
bundle_sha256=hashlib.sha256(b"source-bundle").hexdigest(),
|
||||||
|
capability_sha256=hashlib.sha256(b"source-capability").hexdigest(),
|
||||||
|
)
|
||||||
|
manifest, _payloads = _source_contract(job, inject_path=True)
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
if request.url.path.endswith("/claims"):
|
||||||
|
return _claim_response()
|
||||||
|
return httpx.Response(200, json=manifest)
|
||||||
|
|
||||||
|
with ObservatoryWorkerHttpGateway(
|
||||||
|
base_url="http://localhost:18080",
|
||||||
|
bearer_token=BEARER_TOKEN,
|
||||||
|
work_root=tmp_path / "worker",
|
||||||
|
transport=httpx.MockTransport(handler),
|
||||||
|
) as gateway:
|
||||||
|
_cache_claim(gateway)
|
||||||
|
with pytest.raises(
|
||||||
|
ObservatoryWorkerHttpError,
|
||||||
|
match="member fields changed",
|
||||||
|
):
|
||||||
|
gateway.materialize(job)
|
||||||
|
|
||||||
|
assert not list((tmp_path / "worker").rglob("operator-secret"))
|
||||||
|
|
||||||
|
|
||||||
|
def _result_package(
|
||||||
|
tmp_path: Path,
|
||||||
|
job: SealedObservatoryRecordedJob,
|
||||||
|
) -> tuple[PortableWorkerResultDraft, PortableResultPackageManifest, bytes]:
|
||||||
|
result_id = "portable-http-result-001"
|
||||||
|
result_payload = b'{"accepted":true}'
|
||||||
|
artifact = PortableResultArtifact(
|
||||||
|
role=RESULT_DOCUMENT_ROLE,
|
||||||
|
relative_path="artifacts/result.json",
|
||||||
|
media_type="application/json",
|
||||||
|
byte_length=len(result_payload),
|
||||||
|
sha256=hashlib.sha256(result_payload).hexdigest(),
|
||||||
|
)
|
||||||
|
created_at = "2026-08-31T11:01:00.000Z"
|
||||||
|
job_document: dict[str, object] = {
|
||||||
|
"job_id": job.job_id,
|
||||||
|
"request_sha256": job.request_sha256,
|
||||||
|
"identity_sha256": job.identity_sha256,
|
||||||
|
"submission_receipt_sha256": job.submission_receipt_sha256,
|
||||||
|
"claim_generation": job.claim_generation,
|
||||||
|
}
|
||||||
|
source_document: dict[str, object] = {}
|
||||||
|
definition_document: dict[str, object] = {}
|
||||||
|
result_document: dict[str, object] = {"result_id": result_id}
|
||||||
|
identity = {
|
||||||
|
"schema_version": PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
|
||||||
|
"created_at_utc": created_at,
|
||||||
|
"job": job_document,
|
||||||
|
"source": source_document,
|
||||||
|
"run_definition": definition_document,
|
||||||
|
"result": result_document,
|
||||||
|
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||||
|
"artifacts": [artifact.as_dict()],
|
||||||
|
}
|
||||||
|
package = PortableResultPackageManifest(
|
||||||
|
identity_sha256=canonical_sha256(identity),
|
||||||
|
created_at_utc=created_at,
|
||||||
|
job=job_document,
|
||||||
|
source=source_document,
|
||||||
|
run_definition=definition_document,
|
||||||
|
result=result_document,
|
||||||
|
authority=dict(OBSERVATION_ONLY_AUTHORITY),
|
||||||
|
artifacts=(artifact,),
|
||||||
|
)
|
||||||
|
root = tmp_path / "draft"
|
||||||
|
(root / "artifacts").mkdir(parents=True)
|
||||||
|
(root / "manifest.json").write_bytes(package.canonical_bytes)
|
||||||
|
(root / "artifacts/result.json").write_bytes(result_payload)
|
||||||
|
return (
|
||||||
|
PortableWorkerResultDraft(
|
||||||
|
root=root,
|
||||||
|
result_id=result_id,
|
||||||
|
result_sha256=package.manifest_sha256,
|
||||||
|
result_contract_sha256="c" * 64,
|
||||||
|
),
|
||||||
|
package,
|
||||||
|
result_payload,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _result_handler(
|
||||||
|
*,
|
||||||
|
job: SealedObservatoryRecordedJob,
|
||||||
|
package: PortableResultPackageManifest,
|
||||||
|
result_payload: bytes,
|
||||||
|
receipt_mutator: Callable[[dict[str, object]], None] | None = None,
|
||||||
|
) -> tuple[httpx.MockTransport, list[str]]:
|
||||||
|
artifact = package.artifacts[0]
|
||||||
|
member_id = hashlib.sha256(
|
||||||
|
canonical_json(
|
||||||
|
{
|
||||||
|
"package_identity_sha256": package.identity_sha256,
|
||||||
|
"artifact": artifact.as_dict(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
).hexdigest()
|
||||||
|
requests: list[str] = []
|
||||||
|
|
||||||
|
def plan(uploaded: bool) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"schema_version": PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA,
|
||||||
|
"job_id": job.job_id,
|
||||||
|
"claim_generation": job.claim_generation,
|
||||||
|
"result_id": str(package.result["result_id"]),
|
||||||
|
"result_sha256": package.manifest_sha256,
|
||||||
|
"package_identity_sha256": package.identity_sha256,
|
||||||
|
"members": [
|
||||||
|
{
|
||||||
|
"member_id": member_id,
|
||||||
|
"role": artifact.role,
|
||||||
|
"media_type": artifact.media_type,
|
||||||
|
"byte_length": artifact.byte_length,
|
||||||
|
"sha256": artifact.sha256,
|
||||||
|
"uploaded": uploaded,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"complete": uploaded,
|
||||||
|
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||||
|
}
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
if request.url.path.endswith("/claims"):
|
||||||
|
return _claim_response()
|
||||||
|
assert request.headers["x-mission-core-claim-token"] == CLAIM_TOKEN
|
||||||
|
assert request.headers["x-mission-core-claim-generation"] == "1"
|
||||||
|
requests.append(request.url.path)
|
||||||
|
if request.url.path.endswith("/manifest"):
|
||||||
|
assert request.read() == package.canonical_bytes
|
||||||
|
return httpx.Response(200, json=plan(False))
|
||||||
|
if "/members/" in request.url.path:
|
||||||
|
assert request.url.path.endswith(member_id)
|
||||||
|
assert request.read() == result_payload
|
||||||
|
return httpx.Response(200, json=plan(True))
|
||||||
|
receipt_identity: dict[str, object] = {
|
||||||
|
"schema_version": PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA,
|
||||||
|
"job_id": job.job_id,
|
||||||
|
"job_identity_sha256": job.identity_sha256,
|
||||||
|
"claim_generation": job.claim_generation,
|
||||||
|
"claim_token_sha256": hashlib.sha256(
|
||||||
|
CLAIM_TOKEN.encode("ascii")
|
||||||
|
).hexdigest(),
|
||||||
|
"result_id": str(package.result["result_id"]),
|
||||||
|
"result_sha256": package.manifest_sha256,
|
||||||
|
"package_identity_sha256": package.identity_sha256,
|
||||||
|
"member_count": len(package.artifacts),
|
||||||
|
"total_bytes": sum(item.byte_length for item in package.artifacts),
|
||||||
|
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||||
|
}
|
||||||
|
receipt = {
|
||||||
|
**receipt_identity,
|
||||||
|
"receipt_sha256": hashlib.sha256(
|
||||||
|
canonical_json(receipt_identity)
|
||||||
|
).hexdigest(),
|
||||||
|
}
|
||||||
|
if receipt_mutator is not None:
|
||||||
|
receipt_mutator(receipt)
|
||||||
|
return httpx.Response(200, json=receipt)
|
||||||
|
|
||||||
|
return httpx.MockTransport(handler), requests
|
||||||
|
|
||||||
|
|
||||||
|
def test_http_gateway_uploads_atomic_package_and_verifies_receipt(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
job = _job(bundle_sha256="d" * 64, capability_sha256="e" * 64)
|
||||||
|
draft, package, result_payload = _result_package(tmp_path, job)
|
||||||
|
transport, requests = _result_handler(
|
||||||
|
job=job,
|
||||||
|
package=package,
|
||||||
|
result_payload=result_payload,
|
||||||
|
)
|
||||||
|
|
||||||
|
with ObservatoryWorkerHttpGateway(
|
||||||
|
base_url="https://mission-core.invalid",
|
||||||
|
bearer_token=BEARER_TOKEN,
|
||||||
|
work_root=tmp_path / "worker",
|
||||||
|
transport=transport,
|
||||||
|
) as gateway:
|
||||||
|
_cache_claim(gateway)
|
||||||
|
result = gateway.publish(job, draft)
|
||||||
|
|
||||||
|
assert result.result_id == draft.result_id
|
||||||
|
assert result.result_sha256 == draft.result_sha256
|
||||||
|
assert any(path.endswith("/manifest") for path in requests)
|
||||||
|
assert any("/members/" in path for path in requests)
|
||||||
|
assert any(path.endswith("/complete") for path in requests)
|
||||||
|
|
||||||
|
|
||||||
|
def test_http_gateway_rejects_changed_completion_receipt(tmp_path: Path) -> None:
|
||||||
|
job = _job(bundle_sha256="d" * 64, capability_sha256="e" * 64)
|
||||||
|
draft, package, result_payload = _result_package(tmp_path, job)
|
||||||
|
|
||||||
|
def mutate(receipt: dict[str, object]) -> None:
|
||||||
|
receipt["job_identity_sha256"] = "f" * 64
|
||||||
|
|
||||||
|
transport, _requests = _result_handler(
|
||||||
|
job=job,
|
||||||
|
package=package,
|
||||||
|
result_payload=result_payload,
|
||||||
|
receipt_mutator=mutate,
|
||||||
|
)
|
||||||
|
with ObservatoryWorkerHttpGateway(
|
||||||
|
base_url="http://[::1]:18080",
|
||||||
|
bearer_token=BEARER_TOKEN,
|
||||||
|
work_root=tmp_path / "worker",
|
||||||
|
transport=transport,
|
||||||
|
) as gateway:
|
||||||
|
_cache_claim(gateway)
|
||||||
|
with pytest.raises(
|
||||||
|
ObservatoryWorkerHttpError,
|
||||||
|
match="completion receipt differs",
|
||||||
|
):
|
||||||
|
gateway.publish(job, draft)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"base_url",
|
||||||
|
[
|
||||||
|
"http://mission-core.example",
|
||||||
|
"http://127.0.0.1:18080/api",
|
||||||
|
"https://user:secret@mission-core.example",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_http_gateway_rejects_unsafe_base_urls(
|
||||||
|
tmp_path: Path,
|
||||||
|
base_url: str,
|
||||||
|
) -> None:
|
||||||
|
with pytest.raises(ValueError, match="base URL|loopback"):
|
||||||
|
ObservatoryWorkerHttpGateway(
|
||||||
|
base_url=base_url,
|
||||||
|
bearer_token=BEARER_TOKEN,
|
||||||
|
work_root=tmp_path / "worker",
|
||||||
|
transport=httpx.MockTransport(
|
||||||
|
lambda _request: httpx.Response(500)
|
||||||
|
),
|
||||||
|
)
|
||||||
@@ -0,0 +1,207 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from threading import Event
|
||||||
|
from typing import cast
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry
|
||||||
|
from k1link.observatory.recorded_jobs import RecordedRunDefinition
|
||||||
|
from k1link.observatory.worker_agent import (
|
||||||
|
ObservatoryWorkerCycleReport,
|
||||||
|
ObservatoryWorkerExecutionResult,
|
||||||
|
ObservatoryWorkerExecutorIdentity,
|
||||||
|
ObservatoryWorkerExecutorRegistration,
|
||||||
|
ObservatoryWorkerExecutorRegistry,
|
||||||
|
SealedObservatoryRecordedJob,
|
||||||
|
)
|
||||||
|
from k1link.observatory.worker_http_transport import ObservatoryWorkerHttpError
|
||||||
|
from k1link.observatory.worker_service import (
|
||||||
|
InstalledObservatoryWorkerService,
|
||||||
|
ObservatoryWorkerServiceConfiguration,
|
||||||
|
ObservatoryWorkerServiceError,
|
||||||
|
load_observatory_worker_bearer_token,
|
||||||
|
require_ready_executor_coverage,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _definition(*, release_sha256: str = "3" * 64) -> RecordedRunDefinition:
|
||||||
|
return RecordedRunDefinition(
|
||||||
|
setup_id="portable-lab-v1",
|
||||||
|
definition_id="portable-lab-v1-definition",
|
||||||
|
definition_version=1,
|
||||||
|
definition_sha256="1" * 64,
|
||||||
|
source_adapter_id="sealed-session-source",
|
||||||
|
source_adapter_version=1,
|
||||||
|
source_adapter_sha256="2" * 64,
|
||||||
|
executor_release_id="portable-lab-v1-worker",
|
||||||
|
executor_release_sha256=release_sha256,
|
||||||
|
executor_image_sha256="4" * 64,
|
||||||
|
model_release_ids=("eomt-cityscapes-large", "ddrnet-39"),
|
||||||
|
model_manifest_sha256="5" * 64,
|
||||||
|
resource_profile_id="worker006-single-gpu",
|
||||||
|
resource_profile_sha256="6" * 64,
|
||||||
|
checkpoint_policy="cooperative",
|
||||||
|
allowed_checkpoints=("semantic-pass",),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class _ReadyDefinitions:
|
||||||
|
definitions: tuple[RecordedRunDefinition, ...]
|
||||||
|
|
||||||
|
def ready_recorded_definitions(self) -> tuple[RecordedRunDefinition, ...]:
|
||||||
|
return self.definitions
|
||||||
|
|
||||||
|
|
||||||
|
class _Executor:
|
||||||
|
def execute(
|
||||||
|
self,
|
||||||
|
job: SealedObservatoryRecordedJob,
|
||||||
|
) -> ObservatoryWorkerExecutionResult:
|
||||||
|
raise AssertionError(job)
|
||||||
|
|
||||||
|
|
||||||
|
def _identity(definition: RecordedRunDefinition) -> ObservatoryWorkerExecutorIdentity:
|
||||||
|
return ObservatoryWorkerExecutorIdentity(
|
||||||
|
release_sha256=definition.executor_release_sha256,
|
||||||
|
image_sha256=definition.executor_image_sha256,
|
||||||
|
model_manifest_sha256=definition.model_manifest_sha256,
|
||||||
|
resource_profile_sha256=definition.resource_profile_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _configuration(tmp_path: Path, **overrides: object) -> ObservatoryWorkerServiceConfiguration:
|
||||||
|
values: dict[str, object] = {
|
||||||
|
"base_url": "http://127.0.0.1:18080",
|
||||||
|
"bearer_token_file": tmp_path / "worker.token",
|
||||||
|
"work_root": tmp_path / "work",
|
||||||
|
"idle_poll_seconds": 0.05,
|
||||||
|
"transport_backoff_seconds": 0.05,
|
||||||
|
"max_consecutive_transport_failures": 2,
|
||||||
|
}
|
||||||
|
values.update(overrides)
|
||||||
|
return ObservatoryWorkerServiceConfiguration(**values) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_service_configuration_requires_loopback_for_plain_http(tmp_path: Path) -> None:
|
||||||
|
with pytest.raises(ValueError, match="loopback"):
|
||||||
|
_configuration(tmp_path, base_url="http://mission-core.internal:8000")
|
||||||
|
|
||||||
|
configured = ObservatoryWorkerServiceConfiguration.from_environment(
|
||||||
|
{
|
||||||
|
"MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE": str(tmp_path / "worker.token"),
|
||||||
|
"MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT": str(tmp_path / "work"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert configured.base_url == "http://127.0.0.1:18080"
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="failure bound"):
|
||||||
|
_configuration(tmp_path, max_consecutive_transport_failures=2.5)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="poll interval"):
|
||||||
|
_configuration(tmp_path, idle_poll_seconds=True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_token_loader_requires_private_exact_ascii_file(tmp_path: Path) -> None:
|
||||||
|
token = tmp_path / "worker.token"
|
||||||
|
token.write_text("worker-006-test-bearer-token-000001", encoding="ascii")
|
||||||
|
token.chmod(0o600)
|
||||||
|
|
||||||
|
assert load_observatory_worker_bearer_token(token) == (
|
||||||
|
"worker-006-test-bearer-token-000001"
|
||||||
|
)
|
||||||
|
|
||||||
|
token.chmod(0o644)
|
||||||
|
with pytest.raises(ObservatoryWorkerServiceError, match="permissions"):
|
||||||
|
load_observatory_worker_bearer_token(token)
|
||||||
|
|
||||||
|
link = tmp_path / "worker-link.token"
|
||||||
|
link.symlink_to(token)
|
||||||
|
with pytest.raises(ObservatoryWorkerServiceError, match="unavailable"):
|
||||||
|
load_observatory_worker_bearer_token(link)
|
||||||
|
|
||||||
|
|
||||||
|
def test_install_time_coverage_requires_each_ready_executor_identity() -> None:
|
||||||
|
definition = _definition()
|
||||||
|
definitions = cast(
|
||||||
|
PortableRunDefinitionRegistry,
|
||||||
|
_ReadyDefinitions((definition,)),
|
||||||
|
)
|
||||||
|
matching = ObservatoryWorkerExecutorRegistry(
|
||||||
|
(ObservatoryWorkerExecutorRegistration(_identity(definition), _Executor()),)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert require_ready_executor_coverage(
|
||||||
|
definitions=definitions,
|
||||||
|
executors=matching,
|
||||||
|
) == (_identity(definition),)
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryWorkerServiceError, match="exact local executor"):
|
||||||
|
require_ready_executor_coverage(
|
||||||
|
definitions=definitions,
|
||||||
|
executors=ObservatoryWorkerExecutorRegistry(()),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryWorkerServiceError, match="no portable"):
|
||||||
|
require_ready_executor_coverage(
|
||||||
|
definitions=cast(
|
||||||
|
PortableRunDefinitionRegistry,
|
||||||
|
_ReadyDefinitions(()),
|
||||||
|
),
|
||||||
|
executors=matching,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _FakeGateway:
|
||||||
|
closed: bool = False
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self.closed = True
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _FakeAgent:
|
||||||
|
reports: list[ObservatoryWorkerCycleReport]
|
||||||
|
transport_failure: bool = False
|
||||||
|
|
||||||
|
def run_once(self) -> ObservatoryWorkerCycleReport:
|
||||||
|
if self.transport_failure:
|
||||||
|
raise ObservatoryWorkerHttpError("offline")
|
||||||
|
return self.reports.pop(0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_polling_service_stops_cleanly_after_an_empty_cycle(tmp_path: Path) -> None:
|
||||||
|
stop = Event()
|
||||||
|
gateway = _FakeGateway()
|
||||||
|
report = ObservatoryWorkerCycleReport(
|
||||||
|
state="empty",
|
||||||
|
claim_request_id="worker-006:test",
|
||||||
|
)
|
||||||
|
service = InstalledObservatoryWorkerService(
|
||||||
|
configuration=_configuration(tmp_path),
|
||||||
|
gateway=cast(object, gateway), # type: ignore[arg-type]
|
||||||
|
agent=cast(object, _FakeAgent([report])), # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
|
||||||
|
service.run(stop=stop, on_cycle=lambda _: stop.set())
|
||||||
|
|
||||||
|
assert gateway.closed is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_polling_service_exits_after_bounded_transport_failures(tmp_path: Path) -> None:
|
||||||
|
gateway = _FakeGateway()
|
||||||
|
service = InstalledObservatoryWorkerService(
|
||||||
|
configuration=_configuration(tmp_path),
|
||||||
|
gateway=cast(object, gateway), # type: ignore[arg-type]
|
||||||
|
agent=cast(object, _FakeAgent([], transport_failure=True)), # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryWorkerServiceError, match="failure bound"):
|
||||||
|
service.run(stop=Event())
|
||||||
|
|
||||||
|
assert gateway.closed is True
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import plistlib
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from k1link.observatory.worker_tunnel_launchd import (
|
||||||
|
OBSERVATORY_WORKER_TUNNEL_LABEL,
|
||||||
|
ObservatoryWorkerTunnelPlanError,
|
||||||
|
plan_observatory_worker_tunnel_launch_agent,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _executable(path: Path) -> Path:
|
||||||
|
path.write_text("#!/bin/sh\n", encoding="utf-8")
|
||||||
|
path.chmod(0o700)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def test_tunnel_plan_is_reverse_loopback_only_and_contains_no_credential(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
data = tmp_path / "mission-core"
|
||||||
|
data.mkdir(mode=0o700)
|
||||||
|
ssh = _executable(tmp_path / "ssh")
|
||||||
|
agent = tmp_path / "worker-tunnel.plist"
|
||||||
|
|
||||||
|
plan = plan_observatory_worker_tunnel_launch_agent(
|
||||||
|
data_directory=data,
|
||||||
|
agent_path=agent,
|
||||||
|
ssh_path=ssh,
|
||||||
|
)
|
||||||
|
document = plistlib.loads(plan.desired_payload)
|
||||||
|
|
||||||
|
assert document["Label"] == OBSERVATORY_WORKER_TUNNEL_LABEL
|
||||||
|
assert document["ProgramArguments"][-3:] == [
|
||||||
|
"-R",
|
||||||
|
"127.0.0.1:18080:127.0.0.1:8000",
|
||||||
|
"mission-gpu",
|
||||||
|
]
|
||||||
|
assert "127.0.0.1:18080:127.0.0.1:8000" in document["ProgramArguments"]
|
||||||
|
assert document["KeepAlive"] is True
|
||||||
|
assert document["RunAtLoad"] is True
|
||||||
|
assert document["AbandonProcessGroup"] is False
|
||||||
|
assert "token" not in plan.desired_payload.decode("utf-8").lower()
|
||||||
|
assert plan.to_dict()["changes"]["durable_mutation_performed"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_tunnel_plan_rejects_nonprivate_data_directory(tmp_path: Path) -> None:
|
||||||
|
data = tmp_path / "mission-core"
|
||||||
|
data.mkdir(mode=0o755)
|
||||||
|
data.chmod(0o755)
|
||||||
|
|
||||||
|
with pytest.raises(ObservatoryWorkerTunnelPlanError, match="private"):
|
||||||
|
plan_observatory_worker_tunnel_launch_agent(
|
||||||
|
data_directory=data,
|
||||||
|
agent_path=tmp_path / "agent.plist",
|
||||||
|
ssh_path=_executable(tmp_path / "ssh"),
|
||||||
|
)
|
||||||
@@ -350,7 +350,7 @@ def test_session_router_exposes_immutable_lab_provenance(tmp_path: Path) -> None
|
|||||||
assert_no_local_paths((item, detail), repository)
|
assert_no_local_paths((item, detail), repository)
|
||||||
|
|
||||||
|
|
||||||
def test_session_router_rolls_capability_projections_out_only_in_v2(
|
def test_session_router_versions_capability_and_calculation_profile_projections(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
repository = tmp_path / "repo"
|
repository = tmp_path / "repo"
|
||||||
@@ -390,7 +390,21 @@ def test_session_router_rolls_capability_projections_out_only_in_v2(
|
|||||||
"method": lab_method(),
|
"method": lab_method(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
router = build_session_router(store)
|
calculation_profile = {
|
||||||
|
"schema_version": "missioncore.observatory-calculation-profile/v1",
|
||||||
|
"setup_id": "lab-v1-ravnoves004tree-final",
|
||||||
|
"display_name": "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||||
|
"origin": "existing-result",
|
||||||
|
"definition_id": None,
|
||||||
|
"definition_version": None,
|
||||||
|
"definition_sha256": None,
|
||||||
|
}
|
||||||
|
router = build_session_router(
|
||||||
|
store,
|
||||||
|
lab_calculation_profile_resolver=lambda summary: (
|
||||||
|
calculation_profile if summary.session_id == canonical.session_id else None
|
||||||
|
),
|
||||||
|
)
|
||||||
list_route = endpoint(router, "/api/v1/observation-sessions", "GET")
|
list_route = endpoint(router, "/api/v1/observation-sessions", "GET")
|
||||||
|
|
||||||
default_items = list_route(limit=20, cursor=None, scope="all")["items"]
|
default_items = list_route(limit=20, cursor=None, scope="all")["items"]
|
||||||
@@ -415,11 +429,30 @@ def test_session_router_rolls_capability_projections_out_only_in_v2(
|
|||||||
assert set(by_id) == {legacy.session_id, canonical.session_id}
|
assert set(by_id) == {legacy.session_id, canonical.session_id}
|
||||||
assert by_id[legacy.session_id]["lab"]["replay_capability"] is None
|
assert by_id[legacy.session_id]["lab"]["replay_capability"] is None
|
||||||
assert by_id[canonical.session_id]["lab"]["replay_capability"] == capability.as_dict()
|
assert by_id[canonical.session_id]["lab"]["replay_capability"] == capability.as_dict()
|
||||||
|
assert "calculation_profile" not in by_id[legacy.session_id]["lab"]
|
||||||
|
assert "calculation_profile" not in by_id[canonical.session_id]["lab"]
|
||||||
|
|
||||||
|
v3_labs = list_route(
|
||||||
|
limit=20,
|
||||||
|
cursor=None,
|
||||||
|
scope="laboratory",
|
||||||
|
lab_contract="v3",
|
||||||
|
)["items"]
|
||||||
|
v3_by_id = {item["id"]: item for item in v3_labs}
|
||||||
|
assert v3_by_id[legacy.session_id]["lab"]["calculation_profile"] is None
|
||||||
|
assert (
|
||||||
|
v3_by_id[canonical.session_id]["lab"]["calculation_profile"]
|
||||||
|
== calculation_profile
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
v3_by_id[canonical.session_id]["lab"]["replay_capability"]
|
||||||
|
== capability.as_dict()
|
||||||
|
)
|
||||||
|
|
||||||
application = FastAPI()
|
application = FastAPI()
|
||||||
application.include_router(router)
|
application.include_router(router)
|
||||||
assert TestClient(application).get(
|
assert TestClient(application).get(
|
||||||
"/api/v1/observation-sessions?lab_contract=v3"
|
"/api/v1/observation-sessions?lab_contract=v4"
|
||||||
).status_code == 422
|
).status_code == 422
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user