From 9beb534108a9df62997533c553396add609a2d8d Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Mon, 31 Aug 2026 15:42:56 +0300 Subject: [PATCH] feat(observatory): add portable calculation profiles --- .../core/observation/labCalculationProfile.ts | 140 + .../src/core/observation/sessionArchive.ts | 37 +- .../src/core/observatory/laboratorySetups.ts | 57 +- .../portableLaboratorySetupDecoder.ts | 45 +- .../observatory/portableLaboratorySetups.ts | 57 - .../src/core/observatory/recordedJobs.ts | 29 +- .../useObservatoryLaboratorySetups.ts | 14 +- .../observatory/useObservatoryRecordedJobs.ts | 17 +- .../observatory/ObservatoryWorkspace.tsx | 26 +- .../test/observationSessions.test.mjs | 78 +- .../test/observatoryCatalog.test.mjs | 3 +- .../test/observatoryLaboratorySetups.test.mjs | 105 +- .../test/observatoryRecordedJobs.test.mjs | 35 + .../test/observatoryWorkspace.test.mjs | 14 +- .../observatory-portable-run-definitions.json | 89 +- ...observatory-worker-runtime-candidates.json | 292 ++ .../lab-v1-eomt-ddrnet-portable-v2.json | 162 + config/perception/m49-tgs-portable-v2.json | 60 + docs/15_LABORATORY_RUN_CANON.md | 59 +- ...portable-observatory-result-publication.md | 121 + .../OBSERVATORY_PORTABLE_WORKER_006.md | 196 ++ .../Dockerfile.m49-portable-executor | 37 + ...ke-M49PortableExecutorCandidateInstall.ps1 | 401 +++ ...lab-v1-eomt-ddrnet-executor-candidate.json | 218 ++ .../m49-tgs-portable-runner-source.json | 51 + .../run_m49_tgs_portable.cpp | 266 ++ .../run_m49_tgs_portable.sh | 28 + .../smoke_m49_tgs_portable.sh | 29 + .../build_m49_portable_executor_release.py | 588 ++++ scripts/plan_observatory_worker_tunnel.py | 38 + src/k1link/observatory/__init__.py | 2 + .../observatory/m49_portable_executor.py | 523 +++ src/k1link/observatory/m49_portable_result.py | 1378 ++++++++ src/k1link/observatory/m49_portable_source.py | 1245 ++++++++ .../portable_artifact_transport.py | 1714 ++++++++++ .../observatory/portable_lab_v1_executor.py | 2790 +++++++++++++++++ .../observatory/portable_lab_v1_worker.py | 1225 ++++++++ .../observatory/portable_queue_binding.py | 7 +- .../observatory/portable_result_contract.py | 601 ++++ .../observatory/portable_result_publisher.py | 809 +++++ .../observatory/portable_run_definitions.py | 65 +- .../observatory/portable_setup_projection.py | 319 +- .../portable_worker_integration.py | 375 +++ .../observatory/portable_worker_runtime.py | 969 ++++++ src/k1link/observatory/recorded_jobs.py | 458 ++- src/k1link/observatory/setups.py | 48 +- src/k1link/observatory/source_admission.py | 157 +- src/k1link/observatory/worker_agent.py | 203 +- .../observatory/worker_http_transport.py | 1184 +++++++ src/k1link/observatory/worker_service.py | 357 +++ .../observatory/worker_tunnel_launchd.py | 142 + src/k1link/web/app.py | 235 +- src/k1link/web/observatory_api.py | 273 +- src/k1link/web/observatory_worker_api.py | 295 +- src/k1link/web/session_api.py | 29 +- tests/test_m49_portable_executor_release.py | 836 +++++ ...observatory_portable_artifact_transport.py | 600 ++++ ...st_observatory_portable_lab_v1_executor.py | 1265 ++++++++ ...test_observatory_portable_queue_binding.py | 28 + ...t_observatory_portable_result_publisher.py | 595 ++++ ...st_observatory_portable_run_definitions.py | 80 +- tests/test_observatory_portable_setup_api.py | 208 +- ...t_observatory_portable_setup_projection.py | 63 +- ...observatory_portable_worker_integration.py | 253 ++ ...est_observatory_portable_worker_runtime.py | 574 ++++ tests/test_observatory_recorded_jobs.py | 204 ++ tests/test_observatory_setups.py | 83 +- tests/test_observatory_source_admission.py | 116 + tests/test_observatory_worker_agent.py | 184 +- tests/test_observatory_worker_api.py | 132 +- tests/test_observatory_worker_app_wiring.py | 30 + .../test_observatory_worker_http_transport.py | 515 +++ tests/test_observatory_worker_service.py | 207 ++ .../test_observatory_worker_tunnel_launchd.py | 60 + tests/test_session_api.py | 39 +- 75 files changed, 24419 insertions(+), 348 deletions(-) create mode 100644 apps/control-station/src/core/observation/labCalculationProfile.ts create mode 100644 config/observatory-worker-runtime-candidates.json create mode 100644 config/perception/lab-v1-eomt-ddrnet-portable-v2.json create mode 100644 config/perception/m49-tgs-portable-v2.json create mode 100644 docs/adr/0047-verified-portable-observatory-result-publication.md create mode 100644 docs/runbooks/OBSERVATORY_PORTABLE_WORKER_006.md create mode 100644 experiments/perception/worker/observatory_portable/Dockerfile.m49-portable-executor create mode 100644 experiments/perception/worker/observatory_portable/Invoke-M49PortableExecutorCandidateInstall.ps1 create mode 100644 experiments/perception/worker/observatory_portable/lab-v1-eomt-ddrnet-executor-candidate.json create mode 100644 experiments/perception/worker/observatory_portable/m49-tgs-portable-runner-source.json create mode 100644 experiments/perception/worker/observatory_portable/run_m49_tgs_portable.cpp create mode 100755 experiments/perception/worker/observatory_portable/run_m49_tgs_portable.sh create mode 100644 experiments/perception/worker/observatory_portable/smoke_m49_tgs_portable.sh create mode 100644 scripts/build_m49_portable_executor_release.py create mode 100644 scripts/plan_observatory_worker_tunnel.py create mode 100644 src/k1link/observatory/m49_portable_executor.py create mode 100644 src/k1link/observatory/m49_portable_result.py create mode 100644 src/k1link/observatory/m49_portable_source.py create mode 100644 src/k1link/observatory/portable_artifact_transport.py create mode 100644 src/k1link/observatory/portable_lab_v1_executor.py create mode 100644 src/k1link/observatory/portable_lab_v1_worker.py create mode 100644 src/k1link/observatory/portable_result_contract.py create mode 100644 src/k1link/observatory/portable_result_publisher.py create mode 100644 src/k1link/observatory/portable_worker_integration.py create mode 100644 src/k1link/observatory/portable_worker_runtime.py create mode 100644 src/k1link/observatory/worker_http_transport.py create mode 100644 src/k1link/observatory/worker_service.py create mode 100644 src/k1link/observatory/worker_tunnel_launchd.py create mode 100644 tests/test_m49_portable_executor_release.py create mode 100644 tests/test_observatory_portable_artifact_transport.py create mode 100644 tests/test_observatory_portable_lab_v1_executor.py create mode 100644 tests/test_observatory_portable_result_publisher.py create mode 100644 tests/test_observatory_portable_worker_integration.py create mode 100644 tests/test_observatory_portable_worker_runtime.py create mode 100644 tests/test_observatory_worker_http_transport.py create mode 100644 tests/test_observatory_worker_service.py create mode 100644 tests/test_observatory_worker_tunnel_launchd.py diff --git a/apps/control-station/src/core/observation/labCalculationProfile.ts b/apps/control-station/src/core/observation/labCalculationProfile.ts new file mode 100644 index 0000000..9103d6a --- /dev/null +++ b/apps/control-station/src/core/observation/labCalculationProfile.ts @@ -0,0 +1,140 @@ +export const OBSERVATION_LAB_CALCULATION_PROFILE_SCHEMA = + "missioncore.observatory-calculation-profile/v1" as const; + +export type ObservationLabCalculationProfileOrigin = + | "archived-definition" + | "existing-result"; + +export interface ObservationLabCalculationProfile { + readonly schemaVersion: typeof OBSERVATION_LAB_CALCULATION_PROFILE_SCHEMA; + readonly setupId: string; + readonly displayName: string; + readonly origin: ObservationLabCalculationProfileOrigin; + readonly definitionId: string | null; + readonly definitionVersion: number | null; + readonly definitionSha256: string | null; +} + +export class ObservationLabCalculationProfileContractError extends Error { + constructor(message: string) { + super(message); + this.name = "ObservationLabCalculationProfileContractError"; + } +} + +const PROFILE_KEYS = new Set([ + "schema_version", + "setup_id", + "display_name", + "origin", + "definition_id", + "definition_version", + "definition_sha256", +]); +const IDENTIFIER = /^[a-z][a-z0-9-]{2,95}$/; +const SHA256 = /^[a-f0-9]{64}$/; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function requireText(value: unknown, field: string, maximum: number): string { + if ( + typeof value !== "string" + || value.length === 0 + || value.length > maximum + || value !== value.trim() + ) { + throw new ObservationLabCalculationProfileContractError( + `Поле calculation_profile.${field} должно быть непустой строкой.`, + ); + } + return value; +} + +function requireIdentifier(value: unknown, field: string): string { + const identifier = requireText(value, field, 96); + if (!IDENTIFIER.test(identifier)) { + throw new ObservationLabCalculationProfileContractError( + `Поле calculation_profile.${field} содержит некорректный идентификатор.`, + ); + } + return identifier; +} + +export function decodeObservationLabCalculationProfile( + value: unknown, + sessionId: string, +): ObservationLabCalculationProfile | null { + if (value === null) return null; + if (!isRecord(value) || Object.keys(value).some((key) => !PROFILE_KEYS.has(key))) { + throw new ObservationLabCalculationProfileContractError( + `Профиль расчёта LAB-сессии ${sessionId} нарушил контракт полей.`, + ); + } + if ( + Object.keys(value).length !== PROFILE_KEYS.size + || value.schema_version !== OBSERVATION_LAB_CALCULATION_PROFILE_SCHEMA + ) { + throw new ObservationLabCalculationProfileContractError( + `Профиль расчёта LAB-сессии ${sessionId} имеет неизвестную версию.`, + ); + } + + const origin = value.origin; + if (origin !== "archived-definition" && origin !== "existing-result") { + throw new ObservationLabCalculationProfileContractError( + `Профиль расчёта LAB-сессии ${sessionId} имеет неизвестное происхождение.`, + ); + } + const definitionId = value.definition_id === null + ? null + : requireIdentifier(value.definition_id, "definition_id"); + const definitionVersion = value.definition_version === null + ? null + : value.definition_version; + if ( + definitionVersion !== null + && ( + typeof definitionVersion !== "number" + || !Number.isInteger(definitionVersion) + || definitionVersion < 1 + ) + ) { + throw new ObservationLabCalculationProfileContractError( + `Профиль расчёта LAB-сессии ${sessionId} содержит некорректную версию определения.`, + ); + } + const definitionSha256 = value.definition_sha256 === null + ? null + : requireText(value.definition_sha256, "definition_sha256", 64); + if (definitionSha256 !== null && !SHA256.test(definitionSha256)) { + throw new ObservationLabCalculationProfileContractError( + `Профиль расчёта LAB-сессии ${sessionId} не содержит SHA-256 определения.`, + ); + } + + const definitionFieldCount = [ + definitionId, + definitionVersion, + definitionSha256, + ].filter((entry) => entry !== null).length; + if ( + (origin === "existing-result" && definitionFieldCount !== 0) + || (origin === "archived-definition" && definitionFieldCount !== 3) + ) { + throw new ObservationLabCalculationProfileContractError( + `Профиль расчёта LAB-сессии ${sessionId} содержит неполную идентичность определения.`, + ); + } + + return { + schemaVersion: OBSERVATION_LAB_CALCULATION_PROFILE_SCHEMA, + setupId: requireIdentifier(value.setup_id, "setup_id"), + displayName: requireText(value.display_name, "display_name", 256), + origin, + definitionId, + definitionVersion, + definitionSha256, + }; +} diff --git a/apps/control-station/src/core/observation/sessionArchive.ts b/apps/control-station/src/core/observation/sessionArchive.ts index 1d7466c..9f2c731 100644 --- a/apps/control-station/src/core/observation/sessionArchive.ts +++ b/apps/control-station/src/core/observation/sessionArchive.ts @@ -7,8 +7,14 @@ import { ObservationLabReplayCapabilityContractError, type ObservationLabReplayCapability, } from "./labReplayCapability"; +import { + decodeObservationLabCalculationProfile, + ObservationLabCalculationProfileContractError, + type ObservationLabCalculationProfile, +} from "./labCalculationProfile"; export type { ObservationLabReplayCapability } from "./labReplayCapability"; +export type { ObservationLabCalculationProfile } from "./labCalculationProfile"; export type ObservationSessionStatus = | "recording" @@ -29,6 +35,7 @@ export interface ObservationLabInstance { runCreatedAtUtc: string; publishedAtUtc: string; replayCapability: ObservationLabReplayCapability | null; + calculationProfile: ObservationLabCalculationProfile | null; provenance: Readonly>; } @@ -175,7 +182,11 @@ const LEGACY_LAB_KEYS = new Set([ "published_at_utc", "provenance", ]); -const LAB_KEYS = new Set([...LEGACY_LAB_KEYS, "replay_capability"]); +const LAB_V2_KEYS = new Set([...LEGACY_LAB_KEYS, "replay_capability"]); +const LAB_V3_KEYS = new Set([ + ...LAB_V2_KEYS, + "calculation_profile", +]); const CATALOG_PREPARATION_KEYS = new Set([ "preparation_id", "state", @@ -408,9 +419,17 @@ function decodeLabInstance( value, "replay_capability", ); + const hasCalculationProfile = Object.prototype.hasOwnProperty.call( + value, + "calculation_profile", + ); assertExactKeys( value, - hasTypedCapability ? LAB_KEYS : LEGACY_LAB_KEYS, + hasCalculationProfile + ? LAB_V3_KEYS + : hasTypedCapability + ? LAB_V2_KEYS + : LEGACY_LAB_KEYS, `LAB-привязка сессии ${sessionId}`, ); const labId = requireString(value.lab_id, `lab(${sessionId}).lab_id`, 36); @@ -459,6 +478,17 @@ function decodeLabInstance( } throw error; } + let calculationProfile: ObservationLabCalculationProfile | null; + try { + calculationProfile = hasCalculationProfile + ? decodeObservationLabCalculationProfile(value.calculation_profile, sessionId) + : null; + } catch (error) { + if (error instanceof ObservationLabCalculationProfileContractError) { + throw new ObservationSessionContractError(error.message); + } + throw error; + } return { labId, sourceSessionId, @@ -475,6 +505,7 @@ function decodeLabInstance( `lab(${sessionId}).published_at_utc`, ), replayCapability, + calculationProfile, provenance: value.provenance, }; } @@ -1211,7 +1242,7 @@ export async function fetchObservationSessionCatalog({ queryParameters.set("limit", String(Number(limit))); } if (scope !== "all") queryParameters.set("scope", scope); - if (scope === "laboratory") queryParameters.set("lab_contract", "v2"); + if (scope === "laboratory") queryParameters.set("lab_contract", "v3"); const serializedQuery = queryParameters.toString(); const query = serializedQuery ? `?${serializedQuery}` : ""; let response: Response; diff --git a/apps/control-station/src/core/observatory/laboratorySetups.ts b/apps/control-station/src/core/observatory/laboratorySetups.ts index 8c26266..6f77107 100644 --- a/apps/control-station/src/core/observatory/laboratorySetups.ts +++ b/apps/control-station/src/core/observatory/laboratorySetups.ts @@ -1,5 +1,3 @@ -import { preflightObservatoryPortableLaboratorySetup } from "./portableLaboratorySetups"; - export { fetchObservatoryPortableLaboratorySetups } from "./portableLaboratorySetups"; const CATALOG_SCHEMA = "missioncore.observatory-laboratory-setup-catalog/v1"; @@ -14,6 +12,7 @@ export type ObservatoryLaboratorySetupOrigin = export type ObservatoryLaboratorySetupAction = | "open-existing" | "open-legacy" + | "check" | "blocked"; export interface ObservatoryLaboratoryRunDefinition { @@ -62,7 +61,7 @@ export interface ObservatoryLaboratorySetup { }; readonly preservedResults: readonly ObservatoryLaboratoryPreservedResult[]; readonly preflight: { - readonly outcome: "existing" | "blocked"; + readonly outcome: "existing" | "ready" | "blocked"; readonly action: ObservatoryLaboratorySetupAction; readonly reason: string; readonly submissionAllowed: boolean; @@ -79,6 +78,7 @@ export interface ObservatoryLaboratoryRunPreflight { readonly sourceSessionId: string; readonly setupId: string; readonly definitionSha256: string | null; + readonly checkSha256: string | null; readonly outcome: "existing" | "queueable" | "blocked"; readonly submissionAllowed: boolean; readonly checks: readonly { @@ -142,9 +142,6 @@ export async function preflightObservatoryLaboratorySetup( fetcher?: ObservatoryLaboratorySetupFetch; } = {}, ): Promise { - if (setup.origin === "portable-definition") { - return preflightObservatoryPortableLaboratorySetup(sourceSessionId, setup); - } const response = await request( fetcher, "/api/v1/observatory/run-preflights", @@ -297,29 +294,55 @@ function decodePreservedResult(value: unknown): ObservatoryLaboratoryPreservedRe function decodePreflight(value: unknown): ObservatoryLaboratoryRunPreflight { const row = record(value, "preflight"); - exactKeys(row, [ + const baseKeys = [ "authority", "checks", "definition_sha256", "executor", "existing_result_ids", "outcome", "schema_version", "setup_id", "source_session_id", "submission_allowed", - ], "preflight"); + ] as const; + const hasPortableCheck = Object.hasOwn(row, "check_sha256"); + exactKeys( + row, + hasPortableCheck ? [...baseKeys, "check_sha256"] : baseKeys, + "preflight", + ); exact(row.schema_version, PREFLIGHT_SCHEMA, "preflight schema"); observationAuthority(row.authority); const digest = row.definition_sha256; if (digest !== null && (typeof digest !== "string" || !SHA256.test(digest))) { throw new ObservatoryLaboratorySetupContractError("Некорректный digest preflight."); } + const checkDigest = hasPortableCheck ? row.check_sha256 : null; + if ( + checkDigest !== null + && (typeof checkDigest !== "string" || !SHA256.test(checkDigest)) + ) { + throw new ObservatoryLaboratorySetupContractError( + "Некорректный check digest preflight.", + ); + } + const outcome = oneOf( + row.outcome, + ["existing", "queueable", "blocked"] as const, + "preflight outcome", + ); + const submissionAllowed = boolean( + row.submission_allowed, + "preflight submission_allowed", + ); + if ( + submissionAllowed !== (outcome === "queueable") + || (hasPortableCheck && outcome === "queueable" && checkDigest === null) + ) { + throw new ObservatoryLaboratorySetupContractError( + "Preflight содержит противоречивое разрешение постановки в очередь.", + ); + } return { sourceSessionId: text(row.source_session_id, "preflight source_session_id"), setupId: text(row.setup_id, "preflight setup_id"), definitionSha256: digest, - outcome: oneOf( - row.outcome, - ["existing", "queueable", "blocked"] as const, - "preflight outcome", - ), - submissionAllowed: boolean( - row.submission_allowed, - "preflight submission_allowed", - ), + checkSha256: checkDigest, + outcome, + submissionAllowed, checks: array(row.checks, "preflight checks").map((item) => { const check = record(item, "preflight check"); exactKeys(check, ["check_id", "message", "outcome", "reason_code"], "preflight check"); diff --git a/apps/control-station/src/core/observatory/portableLaboratorySetupDecoder.ts b/apps/control-station/src/core/observatory/portableLaboratorySetupDecoder.ts index 89d278f..9758394 100644 --- a/apps/control-station/src/core/observatory/portableLaboratorySetupDecoder.ts +++ b/apps/control-station/src/core/observatory/portableLaboratorySetupDecoder.ts @@ -55,7 +55,11 @@ function decodePortableSetup(value: unknown): ObservatoryLaboratorySetup { const compatibilityReason = text(compatibility.reason, "portable compatibility reason"); const executor = record(row.executor, "portable executor"); - exactKeys(executor, ["contour_id", "ready", "reason", "state"], "portable executor"); + exactKeys( + executor, + ["contour_id", "ready", "reason", "reason_code", "state"], + "portable executor", + ); const executorState = oneOf( executor.state, ["not-installed", "ready"] as const, @@ -70,20 +74,43 @@ function decodePortableSetup(value: unknown): ObservatoryLaboratorySetup { const executorReason = executor.reason === null ? "Исполнитель установлен." : text(executor.reason, "portable executor reason"); + const executorReasonCode = executor.reason_code === null + ? "portable-executor-ready" + : text(executor.reason_code, "portable executor reason_code"); + if ( + (executorReady && (executor.reason !== null || executor.reason_code !== null)) + || (!executorReady && (executor.reason === null || executor.reason_code === null)) + ) { + throw new ObservatoryPortableSetupDecodeError( + "Portable executor: причина недоступности противоречит состоянию.", + ); + } const preflight = record(row.preflight, "portable preflight"); exactKeys(preflight, [ "action", "existing_result_ids", "outcome", "reason", "submission_allowed", ], "portable preflight"); - exact(preflight.outcome, "blocked", "portable preflight outcome"); - exact(preflight.action, "blocked", "portable preflight action"); + const preflightOutcome = oneOf( + preflight.outcome, + ["ready", "blocked"] as const, + "portable preflight outcome", + ); + const preflightAction = oneOf( + preflight.action, + ["check", "blocked"] as const, + "portable preflight action", + ); const submissionAllowed = boolean( preflight.submission_allowed, "portable preflight submission_allowed", ); - if (submissionAllowed) { + if ( + submissionAllowed !== (preflightOutcome === "ready") + || (preflightOutcome === "ready" && preflightAction !== "check") + || (preflightOutcome === "blocked" && preflightAction !== "blocked") + ) { throw new ObservatoryPortableSetupDecodeError( - "Portable preflight: постановка в очередь ещё не поддерживается.", + "Portable preflight: состояние запуска противоречиво.", ); } const existingResults = array(row.existing_results, "portable existing_results"); @@ -112,15 +139,13 @@ function decodePortableSetup(value: unknown): ObservatoryLaboratorySetup { executor: { contourId: text(executor.contour_id, "portable executor contour_id"), state: executorState, - reasonCode: executorReady - ? "portable-executor-ready" - : "portable-executor-not-installed", + reasonCode: executorReasonCode, reason: executorReason, }, preservedResults: [], preflight: { - outcome: "blocked", - action: "blocked", + outcome: preflightOutcome, + action: preflightAction, reason: text(preflight.reason, "portable preflight reason"), submissionAllowed, existingResultIds: [], diff --git a/apps/control-station/src/core/observatory/portableLaboratorySetups.ts b/apps/control-station/src/core/observatory/portableLaboratorySetups.ts index c5b14c6..b1b6e0d 100644 --- a/apps/control-station/src/core/observatory/portableLaboratorySetups.ts +++ b/apps/control-station/src/core/observatory/portableLaboratorySetups.ts @@ -1,7 +1,5 @@ import { decodePortableCatalog } from "./portableLaboratorySetupDecoder"; import type { - ObservatoryLaboratoryRunPreflight, - ObservatoryLaboratorySetup, ObservatoryLaboratorySetupCatalog, } from "./laboratorySetups"; @@ -46,61 +44,6 @@ export async function fetchObservatoryPortableLaboratorySetups( return catalog; } -export function preflightObservatoryPortableLaboratorySetup( - sourceSessionId: string, - setup: ObservatoryLaboratorySetup, -): ObservatoryLaboratoryRunPreflight { - const definitionSha256 = setup.runDefinition?.definitionSha256 ?? null; - if (setup.origin !== "portable-definition" || definitionSha256 === null) { - throw new ObservatoryPortableLaboratorySetupContractError( - "Portable-сетап не содержит RunDefinition.", - ); - } - if ( - setup.preflight.outcome !== "blocked" - || setup.preflight.action !== "blocked" - || setup.preflight.existingResultIds.length > 0 - || setup.preservedResults.length > 0 - ) { - throw new ObservatoryPortableLaboratorySetupContractError( - "Portable-result ещё не имеет проверяемой привязки к RunDefinition.", - ); - } - return { - sourceSessionId, - setupId: setup.setupId, - definitionSha256, - outcome: "blocked", - submissionAllowed: false, - checks: [ - { - checkId: "source-compatibility", - outcome: setup.compatibility.compatible ? "pass" : "fail", - reasonCode: setup.compatibility.compatible - ? "source-capability-admitted" - : "source-capability-blocked", - message: setup.compatibility.compatible - ? "Запись соответствует portable-профилю LAB V1." - : setup.compatibility.reasons[0]?.message - ?? "Запись не соответствует portable-профилю LAB V1.", - }, - { - checkId: "executor", - outcome: setup.executor.state === "ready" ? "pass" : "fail", - reasonCode: setup.executor.reasonCode, - message: setup.executor.reason, - }, - { - checkId: "durable-queue", - outcome: "fail", - reasonCode: "portable-dispatch-unavailable", - message: setup.preflight.reason, - }, - ], - existingResultIds: [], - }; -} - async function request( fetcher: ObservatoryPortableLaboratorySetupFetch, input: string, diff --git a/apps/control-station/src/core/observatory/recordedJobs.ts b/apps/control-station/src/core/observatory/recordedJobs.ts index 25bc18c..550e2f2 100644 --- a/apps/control-station/src/core/observatory/recordedJobs.ts +++ b/apps/control-station/src/core/observatory/recordedJobs.ts @@ -79,6 +79,10 @@ export async function submitObservatoryRecordedJob( sourceSessionId: string, setupId: string, idempotencyKey: string, + portableBinding: { + readonly definitionSha256: string; + readonly checkSha256: string; + } | null, { signal, fetcher = globalThis.fetch, @@ -95,6 +99,10 @@ export async function submitObservatoryRecordedJob( idempotency_key: idempotencyKey, source_session_id: sourceSessionId, setup_id: setupId, + ...(portableBinding === null ? {} : { + definition_sha256: portableBinding.definitionSha256, + check_sha256: portableBinding.checkSha256, + }), }), signal, }); @@ -112,7 +120,7 @@ export async function submitObservatoryRecordedJob( function decodeJob(value: unknown): ObservatoryRecordedJob { const row = record(value, "расчёт"); exactKeys(row, [ - "authority", "checkpoint_policy", "claim_generation", "created_at_utc", "executor", + "authority", "checkpoint_policy", "claim_generation", "claim_lease", "created_at_utc", "executor", "idempotency_key", "identity_sha256", "job_id", "preemption_receipt_sha256", "preemption_requested", "priority", "request_sha256", "restart_from_zero", "result", "schema_version", "setup", "source", "state", "submission_receipt_sha256", "terminal", @@ -137,6 +145,18 @@ function decodeJob(value: unknown): ObservatoryRecordedJob { "accepted", "queued", "claimed", "running", "paused", "preemption-pending", "succeeded", "failed", "reconciliation-required", ] as const, "state"); + if (row.claim_lease !== null) { + const lease = record(row.claim_lease, "claim_lease"); + exactKeys( + lease, + ["claimed_at_utc", "expires_at_utc", "heartbeat_at_utc", "renewal_count"], + "claim_lease", + ); + text(lease.claimed_at_utc, "claim_lease.claimed_at_utc"); + text(lease.expires_at_utc, "claim_lease.expires_at_utc"); + text(lease.heartbeat_at_utc, "claim_lease.heartbeat_at_utc"); + nonNegativeInteger(lease.renewal_count, "claim_lease.renewal_count"); + } const result = row.result === null ? null : record(row.result, "result"); if (result !== null) exactKeys(result, ["result_id", "sha256"], "result"); const terminal = row.terminal === null ? null : record(row.terminal, "terminal"); @@ -220,6 +240,13 @@ function boolean(value: unknown, label: string): boolean { return value; } +function nonNegativeInteger(value: unknown, label: string): number { + if (!Number.isInteger(value) || Number(value) < 0) { + throw new ObservatoryRecordedJobContractError(`${label}: ожидалось целое число.`); + } + return Number(value); +} + function exact(value: unknown, expected: T, label: string): T { if (value !== expected) throw new ObservatoryRecordedJobContractError(`${label}: значение изменилось.`); return expected; diff --git a/apps/control-station/src/core/observatory/useObservatoryLaboratorySetups.ts b/apps/control-station/src/core/observatory/useObservatoryLaboratorySetups.ts index a8eebf3..3e470fb 100644 --- a/apps/control-station/src/core/observatory/useObservatoryLaboratorySetups.ts +++ b/apps/control-station/src/core/observatory/useObservatoryLaboratorySetups.ts @@ -73,7 +73,7 @@ export function useObservatoryLaboratorySetups(sourceSessionId: string) { publishSetupCatalog(legacyCatalog, setCatalog, setSelectedSetupId); setError(catalogErrorMessage( caught, - "Portable-каталог LAB V1 нарушил локальный контракт.", + "Portable-каталог профилей нарушил локальный контракт.", )); } return; @@ -81,7 +81,7 @@ export function useObservatoryLaboratorySetups(sourceSessionId: string) { publishSetupCatalog(legacyCatalog, setCatalog, setSelectedSetupId); setError(catalogErrorMessage( optionalPortable.reason, - "Portable-каталог LAB V1 недоступен.", + "Portable-каталог профилей недоступен.", )); }) .catch((caught: unknown) => { @@ -195,7 +195,15 @@ function mergeSetupCatalogs( if (legacy.sourceSessionId !== portable.sourceSessionId) { throw new Error("Каталоги сетапов относятся к разным исходным сессиям."); } - const setups = [...legacy.setups, ...portable.setups]; + const portableProfileNames = new Set( + portable.setups.map((setup) => setup.displayName), + ); + const setups = [ + ...legacy.setups.filter( + (setup) => !portableProfileNames.has(setup.displayName), + ), + ...portable.setups, + ]; if (new Set(setups.map((setup) => setup.setupId)).size !== setups.length) { throw new Error("Каталоги сетапов содержат повторяющиеся идентификаторы."); } diff --git a/apps/control-station/src/core/observatory/useObservatoryRecordedJobs.ts b/apps/control-station/src/core/observatory/useObservatoryRecordedJobs.ts index 3efa279..82f8dda 100644 --- a/apps/control-station/src/core/observatory/useObservatoryRecordedJobs.ts +++ b/apps/control-station/src/core/observatory/useObservatoryRecordedJobs.ts @@ -68,7 +68,12 @@ export function useObservatoryRecordedJobs(sourceSessionId: string, setupId: str const refresh = useCallback(() => setRevision((value) => value + 1), []); - const submit = useCallback(async (): Promise => { + const submit = useCallback(async ( + portableBinding: { + readonly definitionSha256: string; + readonly checkSha256: string; + } | null = null, + ): Promise => { if (!sourceSessionId || !setupId || state === "submitting") return null; if (activeJob) return activeJob; requestRef.current?.abort(); @@ -80,9 +85,13 @@ export function useObservatoryRecordedJobs(sourceSessionId: string, setupId: str setState("submitting"); setError(null); try { - const job = await submitObservatoryRecordedJob(sourceSessionId, setupId, key, { - signal: request.signal, - }); + const job = await submitObservatoryRecordedJob( + sourceSessionId, + setupId, + key, + portableBinding, + { signal: request.signal }, + ); if (request.signal.aborted || requestRef.current !== request) return null; setJobs((current) => [job, ...current.filter((candidate) => candidate.jobId !== job.jobId)]); setState("ready"); diff --git a/apps/control-station/src/workspaces/observatory/ObservatoryWorkspace.tsx b/apps/control-station/src/workspaces/observatory/ObservatoryWorkspace.tsx index 9ec08cb..8f37bfd 100644 --- a/apps/control-station/src/workspaces/observatory/ObservatoryWorkspace.tsx +++ b/apps/control-station/src/workspaces/observatory/ObservatoryWorkspace.tsx @@ -150,6 +150,11 @@ function mutationErrorMessage(error: unknown): string { : "Не удалось изменить лабораторный результат в Обсерватории."; } +function evidenceResultSubtitle(evidence: ObservatoryEvidence): string { + const profileName = evidence.lab.calculationProfile?.displayName; + return profileName ? `${evidence.label} · ${profileName}` : evidence.label; +} + export function ObservatoryWorkspace({ definition, }: { @@ -214,7 +219,7 @@ export function ObservatoryWorkspace({ ? "Готовый результат" : setup.origin === "portable-definition" ? setup.executor.state === "ready" - ? "Запись совместима · Worker установлен, запуск закрыт" + ? "Запись совместима · Worker готов к проверке" : "Запись совместима · Worker не установлен" : "Совместимый архивный сетап" : "Несовместим с выбранной сессией", @@ -512,8 +517,8 @@ export function ObservatoryWorkspace({ > {setupController.selectedSetup.compatibility.compatible ? setupController.selectedSetup.executor.state === "not-installed" - ? "Worker LAB V1 не установлен" - : "Запуск LAB V1 недоступен" + ? "Worker-профиль не установлен" + : "Запуск профиля недоступен" : "Запись несовместима"} ) : null} @@ -521,7 +526,18 @@ export function ObservatoryWorkspace({ @@ -631,7 +647,7 @@ export function ObservatoryWorkspace({
{evidence.lab.labId} - {evidence.label} + {evidenceResultSubtitle(evidence)} {evidence.lab.resultKind} · {formatTimestamp(evidence.publishedAtUtc)} diff --git a/apps/control-station/test/observationSessions.test.mjs b/apps/control-station/test/observationSessions.test.mjs index 89aa315..e95e0f4 100644 --- a/apps/control-station/test/observationSessions.test.mjs +++ b/apps/control-station/test/observationSessions.test.mjs @@ -101,6 +101,19 @@ function canonicalLab(overrides = {}) { }; } +function legacyCalculationProfile(overrides = {}) { + return { + schema_version: "missioncore.observatory-calculation-profile/v1", + setup_id: "lab-v1-ravnoves004tree-final", + display_name: "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39", + origin: "existing-result", + definition_id: null, + definition_version: null, + definition_sha256: null, + ...overrides, + }; +} + function canonicalProjectionProvenance(resultId, replayCapability) { return { schema_version: "missioncore.canonical-recorded-lab-projection/v1", @@ -260,6 +273,69 @@ test("session catalog strictly decodes the explicit recorded LAB replay capabili ); }); +test("session catalog decodes a typed calculation profile without inferring unknown results", () => { + const resultId = `lab-v1-vegetation-shadow-${"8".repeat(64)}`; + const decoded = decodeObservationSessionCatalog({ + items: [session({ + id: resultId, + lab: canonicalLab({ calculation_profile: legacyCalculationProfile() }), + })], + }).items[0].lab; + + assert.deepEqual(decoded.calculationProfile, { + schemaVersion: "missioncore.observatory-calculation-profile/v1", + setupId: "lab-v1-ravnoves004tree-final", + displayName: "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39", + origin: "existing-result", + definitionId: null, + definitionVersion: null, + definitionSha256: null, + }); + assert.equal( + decodeObservationSessionCatalog({ + items: [session({ id: resultId, lab: canonicalLab() })], + }).items[0].lab.calculationProfile, + null, + ); + assert.equal( + decodeObservationSessionCatalog({ + items: [session({ + id: resultId, + lab: canonicalLab({ calculation_profile: null }), + })], + }).items[0].lab.calculationProfile, + null, + ); + assert.throws( + () => decodeObservationSessionCatalog({ + items: [session({ + id: resultId, + lab: canonicalLab({ + calculation_profile: legacyCalculationProfile({ + origin: "archived-definition", + definition_id: "lab-v1-portable", + }), + }), + })], + }), + ObservationSessionContractError, + ); + assert.throws( + () => decodeObservationSessionCatalog({ + items: [session({ + id: resultId, + lab: canonicalLab({ + calculation_profile: { + ...legacyCalculationProfile(), + guessed_from_provenance: true, + }, + }), + })], + }), + ObservationSessionContractError, + ); +}); + test("opened archive is named in the scene header and trash hover has no pill", async () => { const appSource = await readFile(new URL("../src/App.tsx", import.meta.url), "utf8"); const styles = await readFile( @@ -365,7 +441,7 @@ test("source and laboratory catalogs are requested as disjoint backend projectio assert.deepEqual(calls, [ "/api/v1/observation-sessions?limit=100&scope=source", - "/api/v1/observation-sessions?limit=100&scope=laboratory&lab_contract=v2", + "/api/v1/observation-sessions?limit=100&scope=laboratory&lab_contract=v3", ]); }); diff --git a/apps/control-station/test/observatoryCatalog.test.mjs b/apps/control-station/test/observatoryCatalog.test.mjs index aa03c0c..d9d1530 100644 --- a/apps/control-station/test/observatoryCatalog.test.mjs +++ b/apps/control-station/test/observatoryCatalog.test.mjs @@ -69,6 +69,7 @@ function evidence(id, sourceSessionId, publishedAtUtc) { runCreatedAtUtc: publishedAtUtc, publishedAtUtc, replayCapability: null, + calculationProfile: null, provenance: { verdict: "must-not-be-inferred" }, }, }; @@ -174,7 +175,7 @@ test("Observatory fetches disjoint read-only source and laboratory projections", assert.deepEqual( calls.map(({ input }) => input).sort(), [ - "/api/v1/observation-sessions?limit=50&scope=laboratory&lab_contract=v2", + "/api/v1/observation-sessions?limit=50&scope=laboratory&lab_contract=v3", "/api/v1/observation-sessions?limit=50&scope=source", ], ); diff --git a/apps/control-station/test/observatoryLaboratorySetups.test.mjs b/apps/control-station/test/observatoryLaboratorySetups.test.mjs index 52e7baa..c4c98c8 100644 --- a/apps/control-station/test/observatoryLaboratorySetups.test.mjs +++ b/apps/control-station/test/observatoryLaboratorySetups.test.mjs @@ -122,6 +122,7 @@ function portableSetup() { contour_id: "worker-006", state: "not-installed", ready: false, + reason_code: "eomt-executor-release-unsealed", reason: "Immutable executor release не установлен.", }, existing_results: [], @@ -136,6 +137,26 @@ function portableSetup() { }; } +function portableM49Setup() { + const profile = portableSetup(); + profile.setup_id = "m49-tgs-portable-v2"; + profile.display_name = "M4.9T5 · TRAVEL TGS · CPU-only, без ML"; + profile.description = "Динамический TGS-разбор записанной K1-сессии без ML; только наблюдение."; + profile.run_definition = { + definition_id: "m49-tgs-portable", + version: 2, + definition_sha256: "4".repeat(64), + result_schema: "missioncore.recorded-travel-tgs-review/v2", + result_kind: "recorded-source-paced-tgs-shadow", + models: [], + }; + profile.source_compatibility.reason = "Запись соответствует требованиям TRAVEL TGS."; + profile.executor.reason_code = "m49-executor-release-unsealed"; + profile.executor.reason = "Immutable executor release не установлен."; + profile.preflight.reason = "Переносимый вычислительный контур M4.9T5 пока недоступен."; + return profile; +} + before(async () => { server = await createServer({ appType: "custom", @@ -178,7 +199,7 @@ test("Observatory setup catalog keeps definition identity separate from executor assert.equal(calls[0].init.method, "GET"); }); -test("portable LAB V1 reports compatible source separately from unavailable Worker", async () => { +test("portable profiles report compatible source separately from unavailable Worker", async () => { const calls = []; const selected = (await fetchObservatoryPortableLaboratorySetups("source-a", { fetcher: async (input, init) => { @@ -186,7 +207,7 @@ test("portable LAB V1 reports compatible source separately from unavailable Work return new Response(JSON.stringify({ schema_version: "missioncore.observatory-portable-setup-catalog/v2", source_session_id: "source-a", - setups: [portableSetup()], + setups: [portableSetup(), portableM49Setup()], authority, }), { status: 200 }); }, @@ -200,52 +221,81 @@ test("portable LAB V1 reports compatible source separately from unavailable Work "EoMT Cityscapes Large 1024", "DDRNet-39", ]); + const m49 = (await fetchObservatoryPortableLaboratorySetups("source-a", { + fetcher: async () => new Response(JSON.stringify({ + schema_version: "missioncore.observatory-portable-setup-catalog/v2", + source_session_id: "source-a", + setups: [portableSetup(), portableM49Setup()], + authority, + }), { status: 200 }), + })).setups[1]; + assert.equal(m49.displayName, "M4.9T5 · TRAVEL TGS · CPU-only, без ML"); + assert.deepEqual(m49.runDefinition.models, []); assert.equal( calls[0].input, "/api/v1/observatory/portable-laboratory-setups?source_session_id=source-a", ); - let unexpectedNetworkCall = false; + let preflightRequest; const preflight = await preflightObservatoryLaboratorySetup("source-a", selected, { - fetcher: async () => { - unexpectedNetworkCall = true; - throw new Error("portable preflight must use its server projection"); + fetcher: async (input, init) => { + preflightRequest = { input: String(input), init }; + return new Response(JSON.stringify({ + schema_version: "missioncore.observatory-run-preflight/v1", + source_session_id: "source-a", + setup_id: selected.setupId, + definition_sha256: "3".repeat(64), + check_sha256: null, + outcome: "blocked", + submission_allowed: false, + checks: [{ + check_id: "executor", + outcome: "fail", + reason_code: "eomt-executor-release-unsealed", + message: "Immutable executor release не установлен.", + }], + existing_result_ids: [], + executor: portableSetup().executor, + authority, + }), { status: 200 }); }, }); - assert.equal(unexpectedNetworkCall, false); + assert.equal(preflightRequest.input, "/api/v1/observatory/run-preflights"); + assert.equal(preflightRequest.init.method, "POST"); assert.equal(preflight.outcome, "blocked"); assert.equal(preflight.submissionAllowed, false); - assert.equal(preflight.checks[0].outcome, "pass"); - assert.equal(preflight.checks[1].outcome, "fail"); + assert.equal(preflight.checkSha256, null); + assert.equal(preflight.checks[0].outcome, "fail"); }); -test("portable LAB V1 rejects a premature enqueue projection", async () => { - const premature = portableSetup(); - premature.executor = { +test("portable profile accepts a consistent ready projection", async () => { + const ready = portableSetup(); + ready.executor = { contour_id: "worker-006", state: "ready", ready: true, + reason_code: null, reason: null, }; - premature.preflight = { + ready.preflight = { outcome: "ready", - action: "enqueue", + action: "check", reason: "Запись готова к постановке в очередь.", submission_allowed: true, existing_result_ids: [], }; - await assert.rejects( - fetchObservatoryPortableLaboratorySetups("source-a", { - fetcher: async () => new Response(JSON.stringify({ - schema_version: "missioncore.observatory-portable-setup-catalog/v2", - source_session_id: "source-a", - setups: [premature], - authority, - }), { status: 200 }), - }), - /значение изменилось|значение не поддерживается|постановка в очередь ещё не поддерживается/, - ); + const catalog = await fetchObservatoryPortableLaboratorySetups("source-a", { + fetcher: async () => new Response(JSON.stringify({ + schema_version: "missioncore.observatory-portable-setup-catalog/v2", + source_session_id: "source-a", + setups: [ready], + authority, + }), { status: 200 }), + }); + assert.equal(catalog.setups[0].executor.state, "ready"); + assert.equal(catalog.setups[0].preflight.outcome, "ready"); + assert.equal(catalog.setups[0].preflight.submissionAllowed, true); }); test("portable LAB V1 rejects an unbound existing result projection", async () => { @@ -272,7 +322,7 @@ test("portable LAB V1 rejects an unbound existing result projection", async () = authority, }), { status: 200 }), }), - /значение изменилось|проверяемая привязка результата/, + /значение изменилось|значение не поддерживается|проверяемая привязка результата/, ); }); @@ -315,6 +365,7 @@ test("Observatory preflight sends the exact selected definition and never submit source_session_id: "source-a", setup_id: selected.setupId, definition_sha256: "b".repeat(64), + check_sha256: null, outcome: "blocked", submission_allowed: false, checks: [{ @@ -340,6 +391,7 @@ test("Observatory preflight sends the exact selected definition and never submit }); assert.equal(preflight.outcome, "blocked"); assert.equal(preflight.submissionAllowed, false); + assert.equal(preflight.checkSha256, null); }); test("Observatory dynamic preflight admits only an explicit queueable response", async () => { @@ -374,6 +426,7 @@ test("Observatory dynamic preflight admits only an explicit queueable response", assert.equal(preflight.outcome, "queueable"); assert.equal(preflight.submissionAllowed, true); + assert.equal(preflight.checkSha256, null); }); test("Observatory setup contract rejects authority escalation and response drift", async () => { diff --git a/apps/control-station/test/observatoryRecordedJobs.test.mjs b/apps/control-station/test/observatoryRecordedJobs.test.mjs index 9b556f2..9150eb6 100644 --- a/apps/control-station/test/observatoryRecordedJobs.test.mjs +++ b/apps/control-station/test/observatoryRecordedJobs.test.mjs @@ -61,6 +61,7 @@ function job(state = "queued") { restart_from_zero: state === "paused", preemption_receipt_sha256: null, claim_generation: 0, + claim_lease: null, result: state === "succeeded" ? { result_id: "m49-result", sha256: "d".repeat(64) } : null, @@ -132,6 +133,7 @@ test("Observatory submits only public identities and accepts every durable state "source-a", "m49-tgs", "observatory-ui:source-a:m49-tgs:request-a", + null, { fetcher: async (input, init) => { request = { input: String(input), init }; @@ -172,8 +174,41 @@ test("Observatory queue contract rejects authority escalation and response drift "source-a", "m49-tgs", "observatory-ui:source-a:m49-tgs:request-a", + null, { fetcher: async () => new Response(JSON.stringify(drifted), { status: 200 }) }, ), ObservatoryRecordedJobContractError, ); }); + +test("portable submission carries the exact definition/check fence", async () => { + let request; + await submitObservatoryRecordedJob( + "source-a", + "lab-v1-eomt-ddrnet-portable-v1", + "observatory-ui:source-a:lab-v1:request-a", + { + definitionSha256: "e".repeat(64), + checkSha256: "f".repeat(64), + }, + { + fetcher: async (input, init) => { + request = { input: String(input), init }; + const response = job("queued"); + response.setup.setup_id = "lab-v1-eomt-ddrnet-portable-v1"; + response.setup.definition_sha256 = "e".repeat(64); + response.source.session_id = "source-a"; + return new Response(JSON.stringify(response), { status: 202 }); + }, + }, + ); + + assert.deepEqual(JSON.parse(request.init.body), { + schema_version: "missioncore.observatory-recorded-run-submit/v1", + idempotency_key: "observatory-ui:source-a:lab-v1:request-a", + source_session_id: "source-a", + setup_id: "lab-v1-eomt-ddrnet-portable-v1", + definition_sha256: "e".repeat(64), + check_sha256: "f".repeat(64), + }); +}); diff --git a/apps/control-station/test/observatoryWorkspace.test.mjs b/apps/control-station/test/observatoryWorkspace.test.mjs index 875179b..b138596 100644 --- a/apps/control-station/test/observatoryWorkspace.test.mjs +++ b/apps/control-station/test/observatoryWorkspace.test.mjs @@ -84,6 +84,14 @@ test("Observatory mounts the one shared canonical replay only after explicit adm assert.match(workspace, /observatory-session-stack/); assert.match(workspace, /observatory-session-summary__facts/); assert.match(workspace, /observatory-evidence-card__copy/); + assert.match( + workspace, + /function evidenceResultSubtitle[\s\S]*calculationProfile\?\.displayName[\s\S]*`\$\{evidence\.label\} · \$\{profileName\}`[\s\S]*: evidence\.label/, + ); + assert.match( + workspace, + /\{evidenceResultSubtitle\(evidence\)\}<\/span>/, + ); assert.match( workspace, /evidence\.recordedRun \? \([\s\S]*name="trash"[\s\S]*name="edit"[\s\S]*Открыть визуальный разбор:[\s\S]*name="eye"/, @@ -242,8 +250,12 @@ test("Observatory keeps one compact selector axis without the obsolete setup det assert.doesNotMatch(setupHook, /Promise\.allSettled/); assert.match(setupHook, /publishSetupCatalog\(\s*legacyCatalog/); assert.match(setupHook, /preserveUnknownSelection: true/); + assert.match( + setupHook, + /portableProfileNames[\s\S]*legacy\.setups\.filter\([\s\S]*!portableProfileNames\.has\(setup\.displayName\)[\s\S]*\.\.\.portable\.setups/, + ); assert.match(setupHook, /selectedSetupId, sourceSessionId/); - assert.match(workspace, /Worker установлен, запуск закрыт/); + assert.match(workspace, /Worker готов к проверке/); assert.match( workspace, /preflightCandidate\.definitionSha256[\s\S]*selectedSetup\?\.runDefinition\?\.definitionSha256/, diff --git a/config/observatory-portable-run-definitions.json b/config/observatory-portable-run-definitions.json index 88463a8..88e128b 100644 --- a/config/observatory-portable-run-definitions.json +++ b/config/observatory-portable-run-definitions.json @@ -4,8 +4,8 @@ { "setup_id": "lab-v1-eomt-ddrnet-portable-v1", "definition_id": "lab-v1-eomt-ddrnet-portable", - "version": 1, - "definition_sha256": "57bf8f0859e10e54e30322c9a8aa28b427699f6fe6b5267e279ec3390fa78466", + "version": 2, + "definition_sha256": "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2", "source_requirements": { "plugin_id": "nodedc.device.xgrids-lixelkity-k1", "archive_id": "xgrids-k1.viewer-live.evidence", @@ -32,9 +32,9 @@ }, "components": [ { - "component_id": "ddrnet-full-route-runtime-config-v1", + "component_id": "ddrnet-portable-runtime-config-v2", "kind": "configuration", - "sha256": "ec7464c2818a707c79aadaa6494625b76d45fb4fe59b74b5208ff9bbd7c9006a" + "sha256": "c0ff657dadc86607d77d232e84d041fbf2d8b63e86d02319e9cd607220d00f21" }, { "component_id": "eomt-recorded-dependency-set-v1", @@ -145,8 +145,85 @@ "release_id": null, "release_sha256": null, "image_sha256": null, - "reason_code": "eomt-executor-release-unsealed", - "reason": "Immutable EoMT plus DDRNet executor release and image are not sealed or installed on Worker 006." + "reason_code": "lab-v1-portable-v2-uninstalled", + "reason": "Portable LAB V1 v2 is not sealed or installed; a commit-bound combined image and Worker 006 receipt are required." + }, + "authority": { + "commands_enabled": false, + "actuation_allowed": false, + "navigation_or_safety_accepted": false, + "production_accepted": false + } + }, + { + "setup_id": "m49-tgs-portable-v2", + "definition_id": "m49-tgs-portable", + "version": 2, + "definition_sha256": "73611f24d70319ea1edca428726d6538a3cbad012a415cc0c1a7ecb7d9b4d910", + "source_requirements": { + "plugin_id": "nodedc.device.xgrids-lixelkity-k1", + "archive_id": "xgrids-k1.viewer-live.evidence", + "required_modalities": [ + "point-cloud", + "trajectory", + "video" + ], + "camera_source_id": "sensor.camera.right", + "camera_semantic_channel_id": "camera.video.recorded", + "recorded_media_type": "video/mp4; codecs=\"avc1.641028\"", + "recorded_media_init_sha256": "e2279963e16d84c91d68e7dbb1f7efed840533387dfeb844b7398bff45fbde38", + "camera_width": 800, + "camera_height": 600, + "calibration_slot": "camera_1", + "calibration_identity_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9", + "exactly_one_media_epoch": true, + "seekable": true + }, + "source_adapter": { + "adapter_id": "xgrids-k1-recorded-observatory-v2", + "version": 2, + "contract_sha256": "4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de" + }, + "components": [ + { + "component_id": "k1-camera-1-calibration-v1", + "kind": "calibration", + "sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9" + }, + { + "component_id": "m49-tgs-portable-profile-v2", + "kind": "configuration", + "sha256": "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9" + } + ], + "models": [], + "resource_profile": { + "schema_version": "missioncore.observatory-portable-resource-profile/v2", + "profile_id": "worker006-cpu-single-run-portable-v2", + "contour_id": "worker-006", + "accelerator_id": "cpu-only", + "concurrency": 1, + "checkpoint_policy": "non-checkpointable", + "allowed_checkpoints": [], + "profile_sha256": "49e373f1cbf314e7fab2d176db295f3d3d9ddbbcbd724bbde988ef60ad767dee" + }, + "result_contract": { + "schema_version": "missioncore.observatory-portable-result-contract/v2", + "contract_id": "m49-tgs-portable-review-v2", + "version": 2, + "result_schema": "missioncore.recorded-tgs-costmap-review/v2", + "result_kind": "recorded-perception-qualification", + "publication": "observatory", + "contract_sha256": "9dd80c8e2504559d2156fca933de6eb27901e35305e6853aeb84707e1cb13892" + }, + "executor": { + "contour_id": "worker-006", + "state": "not-installed", + "release_id": null, + "release_sha256": null, + "image_sha256": null, + "reason_code": "m49-portable-executor-not-installed", + "reason": "A source-independent M4.9 TGS executor release and image are not sealed or installed on Worker 006." }, "authority": { "commands_enabled": false, diff --git a/config/observatory-worker-runtime-candidates.json b/config/observatory-worker-runtime-candidates.json new file mode 100644 index 0000000..151c12c --- /dev/null +++ b/config/observatory-worker-runtime-candidates.json @@ -0,0 +1,292 @@ +{ + "schema_version": "missioncore.observatory-portable-worker-runtime-registry/v1", + "candidates": [ + { + "schema_version": "missioncore.observatory-portable-worker-runtime-candidate/v1", + "adapter_id": "lab-v1-eomt-ddrnet-worker006-v2", + "setup_id": "lab-v1-eomt-ddrnet-portable-v1", + "definition_id": "lab-v1-eomt-ddrnet-portable", + "definition_version": 2, + "definition_sha256": "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2", + "source_adapter_sha256": "4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de", + "model_manifest_sha256": "3fd2d43af73bd73f89d9ffae95d8770cfdeb46033ec967509124fac6ae4afe56", + "resource_profile_sha256": "7468138cad115210eda18e7a3350e3423d3d39ac0aae650b3266cdc2bc63fd5d", + "result_contract_sha256": "b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a", + "state": "blocked", + "executor": null, + "reusable_assets": [ + { + "asset_id": "ddrnet-checkpoint", + "kind": "model-artifact", + "sha256": "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6", + "byte_length": 259419077, + "component_id": null, + "model_release_id": "lab-v1-ddrnet-39-goose-fine-64-v1", + "model_artifact_role": "checkpoint" + }, + { + "asset_id": "ddrnet-goose-image", + "kind": "container-image", + "sha256": "591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd", + "byte_length": null, + "component_id": null, + "model_release_id": null, + "model_artifact_role": null + }, + { + "asset_id": "ddrnet-goose-runner", + "kind": "local-file", + "sha256": "b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1", + "byte_length": 32877, + "component_id": null, + "model_release_id": null, + "model_artifact_role": null + }, + { + "asset_id": "ddrnet-portable-config", + "kind": "definition-component", + "sha256": "c0ff657dadc86607d77d232e84d041fbf2d8b63e86d02319e9cd607220d00f21", + "byte_length": 4324, + "component_id": "ddrnet-portable-runtime-config-v2", + "model_release_id": null, + "model_artifact_role": null + }, + { + "asset_id": "eomt-config-json", + "kind": "model-artifact", + "sha256": "7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650", + "byte_length": 1575, + "component_id": null, + "model_release_id": "eomt-cityscapes-large-1024-v1", + "model_artifact_role": "config-json" + }, + { + "asset_id": "eomt-image", + "kind": "container-image", + "sha256": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794", + "byte_length": null, + "component_id": null, + "model_release_id": null, + "model_artifact_role": null + }, + { + "asset_id": "eomt-model-weights", + "kind": "model-artifact", + "sha256": "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782", + "byte_length": 1276175488, + "component_id": null, + "model_release_id": "eomt-cityscapes-large-1024-v1", + "model_artifact_role": "model-weights" + }, + { + "asset_id": "eomt-orchestrator", + "kind": "definition-component", + "sha256": "d3e9435939444ab35b27a744ac314e289ebd66a13fa56e3d59f121e088d22774", + "byte_length": 21489, + "component_id": "eomt-recorded-orchestrator-v1", + "model_release_id": null, + "model_artifact_role": null + }, + { + "asset_id": "eomt-preprocessor-config", + "kind": "model-artifact", + "sha256": "97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7", + "byte_length": 666, + "component_id": null, + "model_release_id": "eomt-cityscapes-large-1024-v1", + "model_artifact_role": "preprocessor-config" + }, + { + "asset_id": "eomt-profile", + "kind": "definition-component", + "sha256": "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875", + "byte_length": 3805, + "component_id": "eomt-recorded-profile-v1", + "model_release_id": null, + "model_artifact_role": null + }, + { + "asset_id": "eomt-runner", + "kind": "definition-component", + "sha256": "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4", + "byte_length": 30720, + "component_id": "eomt-recorded-runner-v1", + "model_release_id": null, + "model_artifact_role": null + }, + { + "asset_id": "vegetation-policy", + "kind": "definition-component", + "sha256": "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35", + "byte_length": 3022, + "component_id": "vegetation-mission-policy-v1", + "model_release_id": null, + "model_artifact_role": null + }, + { + "asset_id": "vegetation-provider-map", + "kind": "definition-component", + "sha256": "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352", + "byte_length": 2756, + "component_id": "vegetation-provider-label-map-v1", + "model_release_id": null, + "model_artifact_role": null + } + ], + "phases": [ + { + "phase_id": "source-delivery", + "state": "implemented" + }, + { + "phase_id": "eomt-runtime", + "state": "implemented" + }, + { + "phase_id": "ddrnet-portable-runtime", + "state": "missing" + }, + { + "phase_id": "portable-lab-orchestrator", + "state": "implemented" + }, + { + "phase_id": "result-v2-assembler", + "state": "implemented" + }, + { + "phase_id": "observatory-result-publisher", + "state": "implemented" + } + ], + "blockers": [ + "combined-executor-image-unsealed", + "commit-bound-source-unavailable", + "ddrnet-component-port-uninstalled", + "eomt-component-port-uninstalled", + "executor-release-unsealed", + "fixture-smoke-unaccepted", + "worker-installation-receipt-unavailable" + ], + "authority": { + "commands_enabled": false, + "actuation_allowed": false, + "navigation_or_safety_accepted": false, + "production_accepted": false + }, + "candidate_sha256": "b22b8fa16cca6dd68cf1ee68abeea84b33f4c420bb3844931b4dc7fd19434a81" + }, + { + "schema_version": "missioncore.observatory-portable-worker-runtime-candidate/v1", + "adapter_id": "m49-tgs-worker006-portable-v2", + "setup_id": "m49-tgs-portable-v2", + "definition_id": "m49-tgs-portable", + "definition_version": 2, + "definition_sha256": "73611f24d70319ea1edca428726d6538a3cbad012a415cc0c1a7ecb7d9b4d910", + "source_adapter_sha256": "4e12be6d2503e2e237eddb290b28d6a7d16cf983855b6d8e6b4e3b65d2feb0de", + "model_manifest_sha256": "489a43448f720a9b5c7993dc8279d167b77191a586f0d87b6d38b81cf728e2f1", + "resource_profile_sha256": "49e373f1cbf314e7fab2d176db295f3d3d9ddbbcbd724bbde988ef60ad767dee", + "result_contract_sha256": "9dd80c8e2504559d2156fca933de6eb27901e35305e6853aeb84707e1cb13892", + "state": "blocked", + "executor": null, + "reusable_assets": [ + { + "asset_id": "m49-portable-profile", + "kind": "definition-component", + "sha256": "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9", + "byte_length": 1683, + "component_id": "m49-tgs-portable-profile-v2", + "model_release_id": null, + "model_artifact_role": null + }, + { + "asset_id": "m49-portable-runner-manifest", + "kind": "local-file", + "sha256": "264c79258195c91448cadad2fff68fdee067c539cccdf1227657a4417ed4cc85", + "byte_length": 1825, + "component_id": null, + "model_release_id": null, + "model_artifact_role": null + }, + { + "asset_id": "m49-portable-runner-source", + "kind": "local-file", + "sha256": "52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9", + "byte_length": 11052, + "component_id": null, + "model_release_id": null, + "model_artifact_role": null + }, + { + "asset_id": "m49-portable-runner-wrapper", + "kind": "local-file", + "sha256": "2d6c32560682647f868e4ce4c2605749f17c60482a609f8c03ff951411f48ffb", + "byte_length": 858, + "component_id": null, + "model_release_id": null, + "model_artifact_role": null + }, + { + "asset_id": "m49-portable-smoke", + "kind": "local-file", + "sha256": "e5da73c2cf89ed0671de8ac52e13014a7faf79ce1ff1f97a090154063a61c18d", + "byte_length": 1117, + "component_id": null, + "model_release_id": null, + "model_artifact_role": null + }, + { + "asset_id": "travel-tgs-image", + "kind": "container-image", + "sha256": "7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f", + "byte_length": null, + "component_id": null, + "model_release_id": null, + "model_artifact_role": null + } + ], + "phases": [ + { + "phase_id": "source-delivery", + "state": "missing" + }, + { + "phase_id": "camera-lidar-timeline-materializer", + "state": "missing" + }, + { + "phase_id": "portable-tgs-input-materializer", + "state": "missing" + }, + { + "phase_id": "portable-tgs-runner", + "state": "implemented" + }, + { + "phase_id": "result-v2-assembler", + "state": "missing" + }, + { + "phase_id": "observatory-result-publisher", + "state": "missing" + } + ], + "blockers": [ + "executor-release-unsealed", + "portable-camera-lidar-timeline-unimplemented", + "portable-result-assembler-unimplemented", + "portable-source-delivery-unimplemented", + "portable-tgs-input-unimplemented", + "portable-tgs-runner-unsealed", + "result-publisher-integration-unaccepted" + ], + "authority": { + "commands_enabled": false, + "actuation_allowed": false, + "navigation_or_safety_accepted": false, + "production_accepted": false + }, + "candidate_sha256": "e2085c25e45a46de58f693d1128914dc25febbe18a28341cdf57c79f3472f180" + } + ] +} diff --git a/config/perception/lab-v1-eomt-ddrnet-portable-v2.json b/config/perception/lab-v1-eomt-ddrnet-portable-v2.json new file mode 100644 index 0000000..e8e4195 --- /dev/null +++ b/config/perception/lab-v1-eomt-ddrnet-portable-v2.json @@ -0,0 +1,162 @@ +{ + "schema_version": "missioncore.lab-v1-eomt-ddrnet-portable-profile/v2", + "profile_id": "lab-v1-eomt-ddrnet-portable-v2", + "lab_id": "LAB-V1", + "worker_id": "worker-006", + "source_binding": { + "mode": "admitted-k1-recording", + "camera_source_id": "sensor.camera.right", + "camera_timeline": "dynamic", + "filesystem_paths": "executor-resolved", + "expected_width": 800, + "expected_height": 600, + "expected_frame_count": "source-derived", + "frame_indices": "source-derived-representative", + "crop_contract": "center-600-square-to-512; outside-crop-is-undefined" + }, + "effective_config_contract": { + "schema_version": "missioncore.lab-v1-goose-vegetation-benchmark/v1", + "dynamic_field": "ravnoves", + "source_id": "sealed-source-derived", + "source_sha256": "sealed-camera-input-derived", + "base_m4_result_id": null + }, + "runtime": { + "super_gradients_version": "3.2.0", + "super_gradients_revision": "54d062ecb1081944a672ce447cf3e96a36708ff9", + "python_version": "3.9", + "numpy_version": "1.23.0", + "cmake_version": "3.31.6", + "onnxsim_version": "0.4.36", + "opencv_python_version": "4.8.1.78", + "pytorch_version": "1.13.1", + "torchvision_version": "0.14.1", + "pytorch_cuda_version": "11.7", + "container_base": "nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04@sha256:ad6d59a3bbf3e82c1c849c9ac09cfc2a3e0bbb8655042fd899be6681b3fe2a85", + "miniconda_installer": "Miniconda3-py39_24.11.1-0-Linux-x86_64.sh", + "miniconda_installer_sha256": "3ea8373098d72140e08aac9217822b047ec094eb457e7f73945af7c6f68bf6f5" + }, + "dataset": { + "dataset_id": "goose-2d-validation-visible-rgb", + "relative_root": "goose-2d/validation", + "mapping_relative_path": "goose_label_mapping.csv", + "mapping_sha256": "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f", + "image_glob": "images/val/**/*_windshield_vis.png", + "expected_pair_count": 962, + "input_size": [ + 512, + 512 + ], + "preprocessing": [ + "center-square-crop", + "nearest-neighbor-resize", + "rgb-to-tensor-0-1" + ] + }, + "candidates": { + "ddrnet": { + "candidate_id": "goose-ddrnet-class-512", + "model_names": [ + "ddrnet_39" + ], + "checkpoint_relative_path": "models/goose/ddrnet_class_512.pth", + "checkpoint_size_bytes": 259419077, + "checkpoint_sha256": "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6", + "published_validation_miou_percent": 46.53 + } + }, + "vegetation_class_names": [ + "leaves", + "forest", + "bush", + "moss", + "tree_crown", + "tree_trunk", + "crops", + "low_grass", + "high_grass", + "scenery_vegetation", + "hedge", + "tree_root" + ], + "visual_case_contract": { + "selection_basis": "ground-truth-class-support-only", + "case_count": 12, + "minimum_focus_pixels": 2048, + "strata": [ + { + "class_name": "high_grass", + "count": 2 + }, + { + "class_name": "low_grass", + "count": 2 + }, + { + "class_name": "bush", + "count": 2 + }, + { + "class_name": "tree_trunk", + "count": 2 + }, + { + "class_name": "tree_crown", + "count": 1 + }, + { + "class_name": "hedge", + "count": 1 + }, + { + "class_name": "forest", + "count": 1 + }, + { + "class_name": "crops", + "count": 1 + } + ], + "error_overlay": { + "correct_material_rgba": [ + 34, + 197, + 94, + 72 + ], + "missed_vegetation_rgba": [ + 239, + 68, + 68, + 220 + ], + "false_vegetation_rgba": [ + 245, + 158, + 11, + 220 + ], + "wrong_vegetation_material_rgba": [ + 168, + 85, + 247, + 220 + ] + } + }, + "policy_action_colors": { + "ALLOW": "#22c55e", + "HIGH_COST": "#f59e0b", + "NO_GO": "#ef4444" + }, + "invariants": { + "one_heavy_candidate_at_a_time": true, + "raw_fisheye_is_immutable": true, + "outside_center_crop_is_free": false, + "missing_or_unknown_is_free": false, + "camera_semantics_can_clear_rigid_geometry": false, + "navigation_authority": false, + "actuation_authority": false, + "canonical_triton_mutation_allowed": false + } +} diff --git a/config/perception/m49-tgs-portable-v2.json b/config/perception/m49-tgs-portable-v2.json new file mode 100644 index 0000000..ff96c85 --- /dev/null +++ b/config/perception/m49-tgs-portable-v2.json @@ -0,0 +1,60 @@ +{ + "schema_version": "missioncore.m49-tgs-portable-profile/v2", + "profile_id": "m49-tgs-portable-v2", + "source_binding": { + "mode": "admitted-k1-recording", + "camera_timeline": "dynamic", + "lidar_replay": "dynamic", + "trajectory": "dynamic", + "frame_counts": "source-derived", + "filesystem_paths": "executor-resolved" + }, + "alignment": { + "timeline": "recorded-camera-host-arrival", + "lidar_selection": "latest-not-newer-than-camera-frame", + "maximum_lidar_age_seconds": 1.0, + "future_frames_allowed": false + }, + "tgs": { + "max_range_m": 80.0, + "min_range_m": 1.0, + "resolution_m": 8.0, + "num_iterations": 3, + "num_lowest_representative_points": 5, + "minimum_points": 10, + "seed_threshold_m": 0.5, + "distance_threshold_m": 0.125, + "outlier_threshold_m": 0.3, + "normal_threshold": 0.94, + "weight_threshold": 200.0, + "lcc_normal_similarity": 0.03, + "lcc_planar_distance_m": 0.1, + "obstacle_height_m": 1.0, + "refine_mode": true + }, + "rolling_profile": { + "history_seconds": 1.0, + "local_radius_m": 12.0, + "missing_lidar_policy": "all-cells-unobserved" + }, + "costmap": { + "coordinate_frame": "map-gravity-local", + "cell_size_m": 0.45, + "radius_m": 12.0, + "state_priority": [ + "NONGROUND_OCCUPIED", + "UNKNOWN_REJECTED", + "GROUND_SUPPORT", + "UNOBSERVED" + ] + }, + "invariants": { + "aos_allowed": false, + "gpu_allowed": false, + "missing_support_means_free": false, + "missing_lidar_means_unobserved": true, + "unobserved_cells_are_emitted": true, + "camera_projection_is_authoritative": false, + "navigation_or_actuation_allowed": false + } +} diff --git a/docs/15_LABORATORY_RUN_CANON.md b/docs/15_LABORATORY_RUN_CANON.md index 19ea021..9da87b5 100644 --- a/docs/15_LABORATORY_RUN_CANON.md +++ b/docs/15_LABORATORY_RUN_CANON.md @@ -278,11 +278,13 @@ an integrity or product need justifies a targeted migration. ## Observatory durable recorded queue and Worker dispatch Selecting a source and a laboratory setup in Observatory is not itself a run. -The public submission contains only `source_session_id`, `setup_id` and an -idempotency key. It cannot supply commands, executable text, filesystem paths, -container images, model identities, resource limits or priority. The server -resolves the allowlisted pair and seals all executable identity into one durable -record: +The exact legacy submission contains only `source_session_id`, `setup_id` and an +idempotency key. A portable submission additionally returns the server-owned +`definition_sha256` and one content-bound `check_sha256` obtained from preflight; +both must be echoed unchanged during submit. Neither request can supply commands, +executable text, filesystem paths, container images, model identities, resource +limits or priority. The server resolves the allowlisted identities and seals all +executable identity into one durable record: - the current source-catalog snapshot captured at admission, plus immutable source bundle and source-capability-manifest SHA-256 identities; @@ -307,22 +309,33 @@ not an unknown model dependency. The binding pins the exact source pack rather than a volatile whole-catalog digest; the server captures and seals the current catalog snapshot into each admitted job. -`LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39` now has a source-independent -portable RunDefinition and lightweight recorded-source capability probe. The -definition seals the exact model/component/resource identities, observation-only -authority and generic `missioncore.recorded-eomt-ddrnet-review/v2` result -contract; it contains no source Session id or label. Compatibility is derived -from the selected Session's real K1 capabilities rather than its name. A -compatible source may therefore report capability `pass` independently from -executor readiness. +Two source-independent definitions are projected by the portable catalog: -That portable foundation is not an executable product path yet. Its executor is -`not-installed`, and there is no accepted server-side definition-SHA/check-SHA -fenced check/submit API, generic v2 result assembler/publisher or deployed Worker -executor. Preflight consequently remains blocked and the UI must not promise or -expose enqueue. The old `missioncore.lab-v1-vegetation-shadow/v1` result is not an -exact/existing result of the generic portable definition, even for its original -source; it remains available only in the immutable legacy LAB catalog. +- `LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39` seals the exact + model/component/resource identities and the + `missioncore.recorded-eomt-ddrnet-review/v2` result contract; +- `M4.9T5 · TRAVEL TGS · CPU-only, без ML` v2 seals an explicitly empty model + manifest, dynamic source-derived frame counts, causal TGS invariants and the + `missioncore.recorded-tgs-costmap-review/v2` result contract. + +Neither portable definition contains a source Session id, label, fixed frame +count or filesystem path. Compatibility is derived independently for every +setup from the selected Session's real K1 capabilities. A compatible source may +therefore report capability `pass` while that setup's executor remains blocked. +Blocked definitions stay projectable and do not prevent an unrelated ready +definition from entering the durable queue allowlist. + +The server now owns a definition-SHA/check-SHA fenced portable check/submit +boundary. Submission still fails closed unless that exact definition has a +sealed `ready` executor release and image and is present in the durable queue. +Both repository portable definitions currently declare `not-installed`, so +their preflight remains blocked; no release or image hash is fabricated. The +generic v2 assemblers, verified publisher and Worker transport exist as a +dormant fail-closed implementation, but exact executor installation, central +storage, authentication, tunnel acceptance and end-to-end canaries remain +pending. The old `missioncore.lab-v1-vegetation-shadow/v1` result and the exact +RAVNOVES00 M4.9 result remain only in their existing immutable catalogs; +neither is reclassified as a result of a portable definition. Recorded work has priority rank `100`. A future live K1 lease has rank `0` and closes new recorded claims while it is pending or active. Cooperative executors @@ -342,8 +355,10 @@ foundation. The production app hard-disables the Worker router even when a valid credential is present until an expiring claim lease and verified result publisher are accepted. Installation of the exact executors, Worker deployment and wiring from the real K1 lifecycle to live-lease triggers remain pending. Therefore a -submitted M4.9T5 job may honestly wait in `queued` without implying that Worker -006 can execute it yet; portable LAB V1 cannot currently be submitted at all. +submitted exact legacy M4.9T5 job may honestly wait in `queued` without implying +that Worker 006 can execute it yet. Portable LAB V1 and portable M4.9T5 cannot +currently be submitted because their executor states are `not-installed` and +the authenticated Worker dispatch gate is closed. Worker telemetry remains secondary observation evidence. It does not replace the authoritative queue ledger, result validation or common laboratory receipt. K1 diff --git a/docs/adr/0047-verified-portable-observatory-result-publication.md b/docs/adr/0047-verified-portable-observatory-result-publication.md new file mode 100644 index 0000000..e62ec2c --- /dev/null +++ b/docs/adr/0047-verified-portable-observatory-result-publication.md @@ -0,0 +1,121 @@ +# ADR 0047: Verified portable Observatory result publication + +Date: 2026-08-31 +Status: accepted as a backend foundation; production validators, transport and +executor wiring remain blocked + +## Context + +The durable Observatory queue deliberately treats a Worker `succeed` call as a +transport acknowledgement. Its `result_id` and SHA-256 do not prove that an +artifact exists, that it was computed from the admitted source, that it obeys +the selected RunDefinition, or that it has observation-only authority. Publishing +that acknowledgement directly as a LAB session would let incomplete, corrupt or +mislabelled output enter the same catalog as immutable evidence. + +The current canonical LAB V1 result is a separate preserved legacy result. It +must not be reinterpreted as a portable v2 result, rewritten with new provenance, +or used as evidence that the portable executor exists. + +## Decision + +Portable results cross a new backend-only verification boundary implemented in +`k1link.observatory.portable_result_contract` and +`k1link.observatory.portable_result_publisher`. The boundary does not change the +queue, the source-admission service, Session API, setup projection, UI, K1, +Simulation, or the legacy canonical publisher. + +A Worker-side assembler must produce one directory whose basename is the +SHA-256 of its canonical `manifest.json`. The manifest schema is +`missioncore.observatory-portable-result-package/v1`; JSON bytes are canonical +UTF-8 with sorted keys and no insignificant whitespace. Its separately hashed +identity binds: + +1. queue `job_id`, request identity, execution identity, submission receipt and + claim generation; +2. source Session, catalog snapshot, source bundle, capability manifest and + source-adapter identities; +3. the complete portable RunDefinition identity, including source requirements, + components, models, resource profile, result contract, executor and authority; +4. result id, result schema, result kind and result-contract SHA-256; +5. a canonical, role-sorted artifact list with confined relative paths, media + types, byte lengths and SHA-256 identities; +6. observation-only authority. + +Exactly one non-empty `result-document` JSON artifact is required. Package roots, +the manifest, every path component and every artifact are checked without +following symlinks outside the package. The queue's terminal result SHA must equal +the canonical manifest SHA and the package directory name. + +Before catalog publication, the server also: + +- resolves the exact `(setup_id, definition_sha256)` in the portable registry and + proves every recorded queue field equals the resulting RunDefinition; +- rejects not-installed executors and the reserved legacy + `lab-v1-vegetation-shadow-` namespace; +- rechecks the current SessionStore catalog snapshot; +- reads both source-admission documents from + `observatory-portable-source-contracts/.json`, verifies their bytes, + schemas, cross-links, adapter, source and authority; +- invokes a validator registered by exact result-contract SHA-256; +- copies the package manifest and all output artifacts into the central immutable + content-addressed artifact store and records its manifest id; +- publishes one idempotent `LabSessionBinding` through `SessionStore`. + +There is intentionally no generic “JSON looks plausible” validator. An unknown +result contract fails before artifact-store or SessionStore mutation. A validator +must understand the exact result schema and determine that the result document +and supporting artifacts are accepted evidence. + +## Calculation-profile provenance + +The publisher does not read a browser selection or infer a profile from a result +name. It requires a server-owned policy bound to the exact definition id, version +and SHA-256. The resulting immutable provenance contains +`missioncore.observatory-calculation-profile/v1` with: + +- `setup_id`; +- full display name; +- origin `archived-definition`; +- definition id, version and SHA-256. + +It also stores a SHA-256 of that calculation-profile document. A later Session API +projection can therefore read `calculation_profile` from the result's provenance +instead of reporting whichever setup happens to be selected now. + +No replay capability is invented. The portable result package is preserved in +the central artifact store, while a result-schema-specific viewer/replay adapter +must be accepted separately before the catalog binding can claim visual replay. + +## Current fail-closed blockers + +The publisher and focused contract tests are implemented, but the end-to-end +production loop remains unavailable for concrete reasons: + +1. neither `recorded-eomt-ddrnet-review-v2` nor + `m49-tgs-portable-review-v2` has an installed exact result-contract validator; +2. the current Worker protocol returns only `result_id` and manifest SHA-256; it + has no accepted package upload/CAS handoff that gives Mission Core the matching + content-addressed directory; +3. portable executor releases that emit this package contract are not installed + and physically accepted on Worker 006; +4. the application has not registered definition-bound calculation-profile + publication policies or wired the publisher into the terminal Worker flow; +5. no result-schema-specific replay-capability adapter has been accepted. + +These are explicit blockers. The backend must not fabricate a package, reuse a +legacy result, trust a Worker success receipt, guess model/profile provenance, or +expose an enqueue/result promise to bypass them. + +## Consequences + +- Queue success and Observatory publication remain distinct evidence states. +- Exact retry is safe: content-addressed artifact publication and the immutable + SessionStore binding are idempotent; a conflicting result id fails closed. +- Source, definition, model, resource-profile and calculation-profile identities + remain available in one portable provenance document. +- Artifact-store writes may leave harmless immutable unreferenced objects if the + final SessionStore transaction detects a conflict; they cannot overwrite an + existing identity. +- The legacy canonical LAB V1 result and its admission logic remain byte-for-byte + outside this publisher. diff --git a/docs/runbooks/OBSERVATORY_PORTABLE_WORKER_006.md b/docs/runbooks/OBSERVATORY_PORTABLE_WORKER_006.md new file mode 100644 index 0000000..3328d7a --- /dev/null +++ b/docs/runbooks/OBSERVATORY_PORTABLE_WORKER_006.md @@ -0,0 +1,196 @@ +# Portable Observatory Worker 006 operational boundary + +## Scope and current state + +This runbook covers only recorded, observation-only Observatory jobs. It does +not change K1 acquisition/control, Simulation/Gaussian, legacy LAB execution, +navigation or safety authority. + +The server queue, renewable claim lease, verified source/result transport and +result publisher exist, but the production Worker API remains deliberately +disabled. All three application gates stay `False` until exact executors are +installed and a complete transport smoke has passed: + +- `OBSERVATORY_WORKER_CLAIM_LEASE_READY`; +- `OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY`; +- `OBSERVATORY_WORKER_PRODUCTION_API_ENABLED`. + +The canonical central artifact store is already declared in the installed +Mission Core LaunchAgent as: + +```text +MISSIONCORE_ARTIFACT_STORE_ROOT=/Volumes/docker/nodedc-mission-core/artifact-store +``` + +On 2026-08-31 `/Volumes/docker` was not mounted. This is a fail-closed +preflight failure, not permission to create a checkout-local substitute. The +portable Worker server composition now requires `central_status=ready` before +it can be constructed. + +The installed `com.nodedc.mission-core.local` LaunchAgent was running from the +M5 Observatory feature worktree while retaining the established Mission Core +data directory in the main checkout. It declares the central artifact-store +root and cache limits, but not the two portable storage roots below. Do not +silently edit or restart this hybrid local service; carry the environment and +working-directory change through one separately reviewed, hash-gated service +transition after the code release is sealed. + +Large portable inputs and in-flight results have no Mission Core data-directory +fallback. The server requires both roots through environment-only +configuration: + +```text +MISSIONCORE_OBSERVATORY_WORKER_SOURCE_CAS_ROOT=/Volumes/docker/nodedc-mission-core/observatory-worker/source-cas +MISSIONCORE_OBSERVATORY_WORKER_RESULT_STAGING_ROOT=/Volumes/docker/nodedc-mission-core/observatory-worker/result-staging +``` + +Both directories must already exist, be canonical non-symlink directories, +remain inside `/Volumes/docker/nodedc-mission-core`, and be disjoint from each +other and from `artifact-store`. Mission Core never creates these configured +roots. If `/Volumes/docker` is absent or is only a local directory rather than +a mounted volume, composition fails closed before queue or Worker API exposure. +Provision the directories only through the reviewed server deployment after +the SMB mount preflight succeeds. + +Read-only Worker evidence on the same date: + +- strict-pinned `ssh -o BatchMode=yes mission-gpu` succeeds; +- Worker host identity remains `DESKTOP-OPJ8J04`; +- Windows OpenSSH `sshd` is running; +- `AllowTcpForwarding` and `GatewayPorts` use OpenSSH defaults: forwarding is + allowed and remote listeners are not exposed beyond loopback; +- Worker loopback port `18080` had no listener. + +## Network shape + +Mission Core remains the only backend on Mac loopback port `8000`. The Mac +owns one reverse SSH tunnel through the existing strict-pinned `mission-gpu` +alias: + +```text +Worker 006 process + -> http://127.0.0.1:18080 + -> encrypted SSH reverse forwarding + -> Mac http://127.0.0.1:8000 +``` + +The exact forwarding declaration is: + +```text +-R 127.0.0.1:18080:127.0.0.1:8000 +``` + +It neither opens a LAN listener nor sends a credential in process arguments. +`ObservatoryWorkerHttpGateway` independently rejects plaintext HTTP to any +non-loopback host. `worker_tunnel_launchd.py` builds a pure, hashable launchd +plan; `scripts/plan_observatory_worker_tunnel.py` prints that plan and performs +no installation. + +## Bearer credential + +The server reads one credential from the fixed private data path: + +```text +/worker-auth/observatory-worker.token +``` + +The installed Worker release receives the same secret through its own private, +runner-managed file and passes only its path as +`MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE`. The token is ASCII, 32–512 bytes, +has no newline, is never stored in Git, an artifact, a plist, an environment +value, a command line or Ops plaintext, and is read with no-follow semantics. +The admitted service runtime is POSIX and requires mode `0600` or narrower; +native Windows ACL handling is intentionally not guessed. + +The Worker service accepts only these non-executable settings: + +```text +MISSIONCORE_OBSERVATORY_WORKER_BASE_URL=http://127.0.0.1:18080 +MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE= +MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT= +MISSIONCORE_OBSERVATORY_WORKER_IDLE_POLL_SECONDS=1 +MISSIONCORE_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS=5 +MISSIONCORE_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES=12 +``` + +There is no configurable module, command, image or executable entrypoint. + +## Install-time executor seam + +`compose_installed_observatory_worker_service` is called only by a reviewed +Worker release. That release injects a constructed +`ObservatoryWorkerExecutorRegistry` directly in memory. Before opening the +HTTP gateway, service composition resolves the four-digest identity of every +portable RunDefinition whose executor state is `ready`: + +- executor release SHA-256; +- executor image SHA-256; +- model manifest SHA-256; +- resource profile SHA-256. + +No ready definitions, an empty registry, or one missing identity stops the +service before its first claim. Blocked candidates cannot be selected through +configuration and are not silently registered. + +## Staged deployment and smoke sequence + +No step below was applied by this implementation increment. + +1. Seal each executor release and installation receipt. Change a portable + RunDefinition to `ready` only when its exact release/image identities and + local adapter admission agree. +2. Mount the existing canonical SMB artifact store. Require Mission Core + artifact status `central_status=ready`; do not initialize a local surrogate. + Through the reviewed server deployment, provision the two disjoint portable + roots above, set both environment values, and verify they resolve inside the + same mounted `/Volumes/docker/nodedc-mission-core` boundary. +3. Provision one bearer credential through the deployment-owned secret path + on both Mac and the admitted POSIX Worker service runtime. Verify file type, + no-link handling and private permissions without printing the value. +4. Generate the reverse-tunnel launchd plan, review its SHA-256 and exact + arguments, then install it through a separate hash-gated local-service + change. Accept only Worker-side `127.0.0.1:18080/api/health` reaching the + canonical Mac service; no second backend is started. +5. Install the exact Worker release. Its fixed entrypoint loads the sealed + RunDefinition registry and its in-memory executor registry, then calls + `compose_installed_observatory_worker_service`. The process must refuse a + missing token, non-loopback plaintext URL, unsafe work root, absent ready + definition or executor coverage gap. +6. Restart the canonical Mission Core process once with CAS and server token + available. Keep the Worker route disabled and verify that K1, + Simulation/Gaussian and legacy LAB surfaces are unchanged. +7. In a separate reviewed source gate, flip the claim-lease, verified-publisher + and production-API flags together. Restart only the canonical port `8000` + service. An authenticated empty claim must return `204`; missing/wrong bearer + and wrong contour identity must remain `401/403`. +8. Submit one short recorded K1 canary for each profile. Require exact source + materialization, lease heartbeat, result-package digest validation, central + artifact publication, immutable calculation-profile provenance and a + reopenable Observatory result. +9. During a bounded recorded canary, start the existing live K1 priority + transition. Require the recorded job to pause/defer and resume only after + live K1 releases the single Worker resource. This test never grants control + or navigation authority. + +## Remaining blockers + +- canonical SMB artifact store is currently unmounted on the Mac; +- portable source CAS and result-staging directories/environment values are not + provisioned; +- the installed Mission Core LaunchAgent has not received a sealed + working-directory/environment transition for this release; +- the shared bearer credential has not been provisioned; +- the reverse tunnel plan has not been installed or smoked; +- the Worker polling entrypoint has not been packaged into an admitted POSIX + Worker release; +- both exact executor releases/install receipts still need their own seals; +- production flags and the server route remain off by design; +- no end-to-end result has yet crossed Worker 006 -> central store -> + Observatory under this new path. + +The Synology root-owned `nodedc-deploy` registry has no +`mission-core-worker` component. Older Mission Core Worker shadow artifacts +explicitly declare that they are outside that registry. Do not route a Windows +Worker install through an unrelated NAS component or weaken the deploy canon; +the durable Worker release needs its own exact reviewed installation transition +and rollback evidence. diff --git a/experiments/perception/worker/observatory_portable/Dockerfile.m49-portable-executor b/experiments/perception/worker/observatory_portable/Dockerfile.m49-portable-executor new file mode 100644 index 0000000..9e6ca2c --- /dev/null +++ b/experiments/perception/worker/observatory_portable/Dockerfile.m49-portable-executor @@ -0,0 +1,37 @@ +FROM ndc/mission-core-m49-t3-travel:20260826 AS builder + +COPY run_m49_tgs_portable.cpp /build/run_m49_tgs_portable.cpp + +RUN echo "52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9 /build/run_m49_tgs_portable.cpp" \ + | sha256sum --check --strict \ + && g++ -std=c++17 -O3 -DNDEBUG -pthread \ + -I/opt/travel/src/TRAVEL/cpp/travel/core \ + -I/usr/include/eigen3 \ + /build/run_m49_tgs_portable.cpp \ + -o /build/run_m49_tgs_portable + +FROM ndc/mission-core-m49-t3-travel:20260826 + +COPY --from=builder /build/run_m49_tgs_portable /opt/nodedc/m49-tgs-portable/bin/run_m49_tgs_portable +COPY m49-tgs-portable-v2.json /opt/nodedc/m49-tgs-portable/profile/m49-tgs-portable-v2.json +COPY smoke_m49_tgs_portable.sh /opt/nodedc/m49-tgs-portable/bin/smoke_m49_tgs_portable + +RUN echo "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9 /opt/nodedc/m49-tgs-portable/profile/m49-tgs-portable-v2.json" \ + | sha256sum --check --strict \ + && echo "e5da73c2cf89ed0671de8ac52e13014a7faf79ce1ff1f97a090154063a61c18d /opt/nodedc/m49-tgs-portable/bin/smoke_m49_tgs_portable" \ + | sha256sum --check --strict \ + && chmod 0555 /opt/nodedc/m49-tgs-portable/bin/smoke_m49_tgs_portable \ + && test -x /opt/nodedc/m49-tgs-portable/bin/run_m49_tgs_portable + +LABEL com.nodedc.product="mission-core" \ + com.nodedc.stack="ndc-mission-core-observatory" \ + com.nodedc.role="m49-tgs-portable-executor" \ + com.nodedc.managed-by="mission-core-worker-release" \ + com.nodedc.worker-contour="worker-006" \ + com.nodedc.authority="observation-only" \ + com.nodedc.base-image.sha256="7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f" \ + com.nodedc.runner-source.sha256="52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9" \ + com.nodedc.fixture-smoke.sha256="e5da73c2cf89ed0671de8ac52e13014a7faf79ce1ff1f97a090154063a61c18d" \ + com.nodedc.profile.sha256="6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9" + +ENTRYPOINT ["/opt/nodedc/m49-tgs-portable/bin/run_m49_tgs_portable"] diff --git a/experiments/perception/worker/observatory_portable/Invoke-M49PortableExecutorCandidateInstall.ps1 b/experiments/perception/worker/observatory_portable/Invoke-M49PortableExecutorCandidateInstall.ps1 new file mode 100644 index 0000000..ec3688b --- /dev/null +++ b/experiments/perception/worker/observatory_portable/Invoke-M49PortableExecutorCandidateInstall.ps1 @@ -0,0 +1,401 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$CandidateId, + + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$ArchiveSha256 +) + +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$WorkerId = "worker-006" +$ExpectedComputer = "DESKTOP-OPJ8J04" +$ReleaseId = "m49-tgs-portable-executor-v1" +$ReleaseParent = "D:\NDC_MISSIONCORE\runtime\releases\observatory-portable" +$BaseImage = "ndc/mission-core-m49-t3-travel:20260826" +$BaseImageId = "sha256:7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f" +$ProfileSha256 = "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9" +$RunnerSourceSha256 = "52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9" +$RunnerWrapperSha256 = "2d6c32560682647f868e4ce4c2605749f17c60482a609f8c03ff951411f48ffb" +$FixtureSmokeSha256 = "e5da73c2cf89ed0671de8ac52e13014a7faf79ce1ff1f97a090154063a61c18d" +$ResultContractSha256 = "9dd80c8e2504559d2156fca933de6eb27901e35305e6853aeb84707e1cb13892" +$ProtectedContainers = @( + "ndc-mission-core-triton", + "ndc-mission-core-perception-worker", + "ndc-gaussian-pipeline-gaussian-gateway-1", + "ndc-gaussian-pipeline-gaussian-pipeline-1", + "ndc-gaussian-pipeline-gaussian-terrain-executor-1" +) +$Authority = [ordered]@{ + commands_enabled = $false + actuation_allowed = $false + navigation_or_safety_accepted = $false + production_accepted = $false +} + +function Assert-LastExitCode([string]$Operation) { + if ($LASTEXITCODE -ne 0) { + throw "$Operation failed with exit code $LASTEXITCODE" + } +} + +function Resolve-DDirectory([string]$Path, [string]$Label) { + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ( + -not $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:" + ) { + throw "$Label must be a real D: directory" + } + return $item.FullName +} + +function Resolve-DFile([string]$Path, [string]$Label) { + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ( + $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:" + ) { + throw "$Label must be a real D: file" + } + return $item.FullName +} + +function Get-Sha256([string]$Path) { + return (Get-FileHash -Algorithm SHA256 -LiteralPath $Path).Hash.ToLowerInvariant() +} + +function Write-Utf8NoBom([string]$Path, [string]$Value) { + $encoding = New-Object System.Text.UTF8Encoding($false) + [IO.File]::WriteAllText($Path, $Value, $encoding) +} + +function Get-ProtectedRuntime() { + $rows = @(((& docker inspect $ProtectedContainers) | ConvertFrom-Json)) + Assert-LastExitCode "protected runtime inspection" + if ($rows.Count -ne $ProtectedContainers.Count) { + throw "protected runtime inventory is incomplete" + } + return $rows +} + +function Assert-ProtectedRuntime([object[]]$Before, [object[]]$After) { + $beforeByName = @{} + foreach ($row in $Before) { + $beforeByName[[string]$row.Name] = [string]$row.Id + } + foreach ($row in $After) { + $name = [string]$row.Name + if (-not $beforeByName.ContainsKey($name) -or $beforeByName[$name] -cne [string]$row.Id) { + throw "protected runtime identity changed: $name" + } + } + $triton = @($After | Where-Object { [string]$_.Name -ceq "/ndc-mission-core-triton" }) + if ( + $triton.Count -ne 1 -or + -not $triton[0].State.Running -or + [string]$triton[0].State.Health.Status -cne "healthy" + ) { + throw "canonical Mission Core Triton is not healthy" + } +} + +if ($env:COMPUTERNAME -cne $ExpectedComputer) { + throw "portable M4.9 executor is pinned to Worker 006" +} + +$releaseCandidate = Join-Path $ReleaseParent "m49-tgs-portable-candidate-$CandidateId" +$releaseRoot = Resolve-DDirectory $releaseCandidate "portable M4.9 candidate release" +$archive = Resolve-DFile (Join-Path $releaseRoot "candidate.tgz") "portable M4.9 candidate archive" +if ((Get-Sha256 $archive) -cne $ArchiveSha256) { + throw "portable M4.9 candidate archive digest changed" +} + +$sourceRoot = Join-Path $releaseRoot "source-candidate" +if (Test-Path -LiteralPath $sourceRoot) { + throw "portable M4.9 source candidate was already extracted" +} +$null = New-Item -ItemType Directory -Path $sourceRoot +& tar -xzf $archive -C $sourceRoot +Assert-LastExitCode "portable M4.9 candidate extraction" +$sourceRoot = Resolve-DDirectory $sourceRoot "portable M4.9 source candidate" + +$manifestPath = Resolve-DFile (Join-Path $sourceRoot "release-manifest.json") "portable M4.9 candidate manifest" +$manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json +if ( + [string]$manifest.candidate_sha256 -cne $CandidateId -or + [string]$manifest.source_state -cne "uncommitted-candidate" -or + [string]$manifest.state -cne "blocked" -or + [string]$manifest.travel_build_image_sha256 -cne $BaseImageId.Substring(7) -or + [string]$manifest.profile_sha256 -cne $ProfileSha256 -or + [string]$manifest.result_contract_sha256 -cne $ResultContractSha256 +) { + throw "portable M4.9 candidate manifest identity changed" +} + +$base = @(((& docker image inspect $BaseImage) | ConvertFrom-Json)) +Assert-LastExitCode "TRAVEL predecessor image inspection" +if ($base.Count -ne 1 -or [string]$base[0].Id -cne $BaseImageId) { + throw "exact TRAVEL predecessor image changed" +} + +$protectedBefore = @(Get-ProtectedRuntime) +Assert-ProtectedRuntime $protectedBefore $protectedBefore +$legacyTaskBefore = Get-ScheduledTask -TaskName "MissionCore-M49TgsFullShadow" -ErrorAction SilentlyContinue + +$shortCandidate = $CandidateId.Substring(0, 16) +$imageTag = "ndc/mission-core-m49-tgs-portable-executor:$shortCandidate-candidate" +$existingImageTags = @( + & docker image ls --format "{{.Repository}}:{{.Tag}}" --filter "reference=$imageTag" +) +Assert-LastExitCode "portable M4.9 executor image collision check" +if ($existingImageTags -contains $imageTag) { + throw "portable M4.9 executor image tag already exists" +} + +$payload = Resolve-DDirectory (Join-Path $sourceRoot "payload") "portable M4.9 candidate payload" +$dockerfile = Resolve-DFile ( + Join-Path $payload "experiments\perception\worker\observatory_portable\Dockerfile.m49-portable-executor" +) "portable M4.9 Dockerfile" +$runnerSource = Resolve-DFile ( + Join-Path $payload "experiments\perception\worker\observatory_portable\run_m49_tgs_portable.cpp" +) "portable M4.9 runner source" +$runnerWrapper = Resolve-DFile ( + Join-Path $payload "experiments\perception\worker\observatory_portable\run_m49_tgs_portable.sh" +) "portable M4.9 runner wrapper" +$fixtureSmoke = Resolve-DFile ( + Join-Path $payload "experiments\perception\worker\observatory_portable\smoke_m49_tgs_portable.sh" +) "portable M4.9 fixture smoke" +$profile = Resolve-DFile ( + Join-Path $payload "config\perception\m49-tgs-portable-v2.json" +) "portable M4.9 profile" +if ( + (Get-Sha256 $runnerSource) -cne $RunnerSourceSha256 -or + (Get-Sha256 $runnerWrapper) -cne $RunnerWrapperSha256 -or + (Get-Sha256 $fixtureSmoke) -cne $FixtureSmokeSha256 -or + (Get-Sha256 $profile) -cne $ProfileSha256 +) { + throw "portable M4.9 exact source files changed" +} + +$context = Join-Path $releaseRoot "build-context" +if (Test-Path -LiteralPath $context) { + throw "portable M4.9 immutable build context already exists" +} +$null = New-Item -ItemType Directory -Path $context +Copy-Item -LiteralPath $dockerfile -Destination (Join-Path $context "Dockerfile") +Copy-Item -LiteralPath $runnerSource -Destination (Join-Path $context "run_m49_tgs_portable.cpp") +Copy-Item -LiteralPath $profile -Destination (Join-Path $context "m49-tgs-portable-v2.json") +Copy-Item -LiteralPath $fixtureSmoke -Destination (Join-Path $context "smoke_m49_tgs_portable.sh") +$context = Resolve-DDirectory $context "portable M4.9 immutable build context" + +$dockerConfig = Join-Path $releaseRoot "docker-config" +$null = New-Item -ItemType Directory -Path $dockerConfig +Write-Utf8NoBom (Join-Path $dockerConfig "config.json") '{"auths":{}}' +$env:DOCKER_CONFIG = $dockerConfig + +$buildOutput = @(& docker build --quiet --pull=false --no-cache --network none --tag $imageTag $context) +Assert-LastExitCode "portable M4.9 executor image build" +$builtImageId = ([string]$buildOutput[-1]).Trim() +$image = @(((& docker image inspect $imageTag) | ConvertFrom-Json)) +Assert-LastExitCode "portable M4.9 executor image inspection" +if ($image.Count -ne 1 -or [string]$image[0].Id -cne $builtImageId) { + throw "portable M4.9 executor image identity is ambiguous" +} +$expectedLabels = [ordered]@{ + "com.nodedc.product" = "mission-core" + "com.nodedc.stack" = "ndc-mission-core-observatory" + "com.nodedc.role" = "m49-tgs-portable-executor" + "com.nodedc.managed-by" = "mission-core-worker-release" + "com.nodedc.worker-contour" = $WorkerId + "com.nodedc.authority" = "observation-only" + "com.nodedc.base-image.sha256" = $BaseImageId.Substring(7) + "com.nodedc.runner-source.sha256" = $RunnerSourceSha256 + "com.nodedc.fixture-smoke.sha256" = $FixtureSmokeSha256 + "com.nodedc.profile.sha256" = $ProfileSha256 +} +foreach ($entry in $expectedLabels.GetEnumerator()) { + if ([string]$image[0].Config.Labels.($entry.Key) -cne [string]$entry.Value) { + throw "portable M4.9 executor image label changed: $($entry.Key)" + } +} + +$installed = Join-Path $releaseRoot "installed" +$null = New-Item -ItemType Directory -Path $installed +$extractName = "ndc-mission-core-m49-portable-extract-$shortCandidate" +if (& docker ps -a --format "{{.Names}}" --filter "name=^/$extractName$") { + throw "portable M4.9 extraction container already exists" +} +try { + $null = & docker create --name $extractName $imageTag + Assert-LastExitCode "portable M4.9 extraction container creation" + & docker cp ( + "$extractName`:/opt/nodedc/m49-tgs-portable/bin/run_m49_tgs_portable" + ) (Join-Path $installed "run_m49_tgs_portable") + Assert-LastExitCode "portable M4.9 binary extraction" + & docker cp ( + "$extractName`:/opt/nodedc/m49-tgs-portable/profile/m49-tgs-portable-v2.json" + ) (Join-Path $installed "m49-tgs-portable-v2.json") + Assert-LastExitCode "portable M4.9 profile extraction" +} finally { + if (& docker ps -a --format "{{.Names}}" --filter "name=^/$extractName$") { + & docker rm --force $extractName *> $null + } +} + +$installedBinary = Resolve-DFile (Join-Path $installed "run_m49_tgs_portable") "installed portable M4.9 runner" +$installedProfile = Resolve-DFile (Join-Path $installed "m49-tgs-portable-v2.json") "installed portable M4.9 profile" +if ((Get-Sha256 $installedProfile) -cne $ProfileSha256) { + throw "installed portable M4.9 profile changed" +} +$binarySha256 = Get-Sha256 $installedBinary +$binaryLength = (Get-Item -LiteralPath $installedBinary).Length + +$buildSeal = [ordered]@{ + schema_version = "missioncore.m49-tgs-portable-compiled-runner-build-candidate/v1" + source_revision = [string]$manifest.source_revision + source_state = [string]$manifest.source_state + candidate_sha256 = $CandidateId + build_image_sha256 = $BaseImageId.Substring(7) + profile_sha256 = $ProfileSha256 + runner_source_sha256 = $RunnerSourceSha256 + runner_wrapper_sha256 = $RunnerWrapperSha256 + compiler_contract = [ordered]@{ + compiler = "g++" + language_standard = "c++17" + flags = @("-O3", "-DNDEBUG", "-pthread") + travel_include = "/opt/travel/src/TRAVEL/cpp/travel/core" + eigen_include = "/usr/include/eigen3" + } + binary = [ordered]@{ + file_name = "run_m49_tgs_portable" + format = "elf" + byte_length = [long]$binaryLength + sha256 = $binarySha256 + } + authority = $Authority +} +$buildSealPath = Join-Path $installed "compiled-runner-build-candidate.json" +Write-Utf8NoBom $buildSealPath (($buildSeal | ConvertTo-Json -Depth 8 -Compress) + "`n") +$buildSealSha256 = Get-Sha256 $buildSealPath + +$smokeName = "ndc-mission-core-m49-portable-smoke-$shortCandidate" +if (& docker ps -a --format "{{.Names}}" --filter "name=^/$smokeName$") { + throw "portable M4.9 smoke container already exists" +} +try { + $smokeOutput = @(& docker run --rm --name $smokeName --network none --read-only --cpus 2 --memory 2g --pids-limit 256 --tmpfs "/smoke:rw,nosuid,nodev,size=32m" --entrypoint /opt/nodedc/m49-tgs-portable/bin/smoke_m49_tgs_portable $imageTag) + Assert-LastExitCode "portable M4.9 fixture smoke" +} finally { + if (& docker ps -a --format "{{.Names}}" --filter "name=^/$smokeName$") { + & docker rm --force $smokeName *> $null + } +} +if ($smokeOutput[-1] -cne "M49_PORTABLE_FIXTURE_SMOKE_OK frames=2 points_per_frame=124668") { + throw "portable M4.9 fixture smoke receipt changed" +} + +$protectedAfter = @(Get-ProtectedRuntime) +Assert-ProtectedRuntime $protectedBefore $protectedAfter +$legacyTaskAfter = Get-ScheduledTask -TaskName "MissionCore-M49TgsFullShadow" -ErrorAction SilentlyContinue +if ( + ($null -eq $legacyTaskBefore) -ne ($null -eq $legacyTaskAfter) -or + ($null -ne $legacyTaskBefore -and [string]$legacyTaskBefore.State -cne [string]$legacyTaskAfter.State) +) { + throw "legacy M49 scheduled task changed during executor installation" +} + +$releaseDocument = [ordered]@{ + schema_version = "missioncore.m49-tgs-portable-executor-installed-candidate/v1" + release_id = $ReleaseId + state = "installed-candidate-blocked" + worker_id = $WorkerId + candidate_sha256 = $CandidateId + source_revision = [string]$manifest.source_revision + source_state = [string]$manifest.source_state + source_archive_sha256 = $ArchiveSha256 + base_image_sha256 = $BaseImageId.Substring(7) + executor_image = [ordered]@{ + tag = $imageTag + sha256 = ([string]$image[0].Id).Substring(7) + size_bytes = [long]$image[0].Size + } + compiled_runner = [ordered]@{ + relative_path = "installed/run_m49_tgs_portable" + byte_length = [long]$binaryLength + sha256 = $binarySha256 + } + compiled_runner_build_seal = [ordered]@{ + relative_path = "installed/compiled-runner-build-candidate.json" + sha256 = $buildSealSha256 + } + profile = [ordered]@{ + relative_path = "installed/m49-tgs-portable-v2.json" + sha256 = $ProfileSha256 + } + result_contract_sha256 = $ResultContractSha256 + fixture_smoke = [ordered]@{ + source = "pinned TRAVEL KITTI fixture" + frame_count = 2 + points_per_frame = 124668 + network = "none" + result = "passed" + } + blockers = @("committed-source-snapshot-missing") + authority = $Authority +} +$releasePath = Join-Path $installed "executor-release-candidate.json" +Write-Utf8NoBom $releasePath (($releaseDocument | ConvertTo-Json -Depth 10 -Compress) + "`n") +$releaseSha256 = Get-Sha256 $releasePath + +$protectedIdentities = @() +foreach ($row in $protectedAfter) { + $protectedIdentities += [ordered]@{ + name = ([string]$row.Name).TrimStart("/") + container_id = [string]$row.Id + } +} +$receipt = [ordered]@{ + schema_version = "missioncore.m49-tgs-portable-worker-installation-receipt/v1" + receipt_state = "installed-candidate-blocked" + worker_id = $WorkerId + computer_name = $env:COMPUTERNAME + installed_at_utc = [DateTimeOffset]::UtcNow.ToString("o") + release_id = $ReleaseId + release_sha256 = $releaseSha256 + release_root = $releaseRoot + executor_image_sha256 = ([string]$image[0].Id).Substring(7) + compiled_runner_sha256 = $binarySha256 + compiled_runner_build_seal_sha256 = $buildSealSha256 + fixture_smoke = "passed" + protected_runtime = $protectedIdentities + legacy_m49_task_state = if ($null -eq $legacyTaskAfter) { $null } else { [string]$legacyTaskAfter.State } + blockers = @("committed-source-snapshot-missing") + authority = $Authority +} +$receiptPath = Join-Path $installed "worker-installation-receipt.json" +Write-Utf8NoBom $receiptPath (($receipt | ConvertTo-Json -Depth 10 -Compress) + "`n") +$receiptSha256 = Get-Sha256 $receiptPath + +[pscustomobject]@{ + ok = $true + receipt_state = "installed-candidate-blocked" + worker_id = $WorkerId + candidate_sha256 = $CandidateId + release_id = $ReleaseId + release_sha256 = $releaseSha256 + executor_image_tag = $imageTag + executor_image_sha256 = ([string]$image[0].Id).Substring(7) + compiled_runner_sha256 = $binarySha256 + compiled_runner_build_seal_sha256 = $buildSealSha256 + worker_installation_receipt_sha256 = $receiptSha256 + fixture_smoke = "passed" + blocker = "committed-source-snapshot-missing" + release_root = $releaseRoot +} | ConvertTo-Json -Compress diff --git a/experiments/perception/worker/observatory_portable/lab-v1-eomt-ddrnet-executor-candidate.json b/experiments/perception/worker/observatory_portable/lab-v1-eomt-ddrnet-executor-candidate.json new file mode 100644 index 0000000..312a5f6 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/lab-v1-eomt-ddrnet-executor-candidate.json @@ -0,0 +1,218 @@ +{ + "assets": [ + { + "asset_id": "ddrnet-checkpoint", + "byte_length": 259419077, + "kind": "model-artifact", + "repository_path": null, + "sha256": "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6" + }, + { + "asset_id": "ddrnet-goose-image", + "byte_length": null, + "kind": "container-image", + "repository_path": null, + "sha256": "591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd" + }, + { + "asset_id": "ddrnet-goose-mapping", + "byte_length": null, + "kind": "runtime-artifact", + "repository_path": null, + "sha256": "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f" + }, + { + "asset_id": "ddrnet-goose-runner", + "byte_length": 32877, + "kind": "repository-file", + "repository_path": "experiments/perception/worker/lab_v1_vegetation_goose/run_goose_vegetation_benchmark.py", + "sha256": "b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1" + }, + { + "asset_id": "ddrnet-image-dockerfile", + "byte_length": 1793, + "kind": "repository-file", + "repository_path": "experiments/perception/worker/lab_v1_vegetation_goose/Dockerfile", + "sha256": "8203fd01e05d8f5bcce11c328dd39dbb6b97df54ca5d9690706536bafe3d3bad" + }, + { + "asset_id": "ddrnet-portable-config", + "byte_length": 4324, + "kind": "repository-file", + "repository_path": "config/perception/lab-v1-eomt-ddrnet-portable-v2.json", + "sha256": "c0ff657dadc86607d77d232e84d041fbf2d8b63e86d02319e9cd607220d00f21" + }, + { + "asset_id": "eomt-config-json", + "byte_length": 1575, + "kind": "model-artifact", + "repository_path": null, + "sha256": "7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650" + }, + { + "asset_id": "eomt-dependency-set", + "byte_length": null, + "kind": "definition-component", + "repository_path": null, + "sha256": "4eb1f8d33236806e74f9e5bb96b7dce2ac37623dc39b2184be2aa8d7d00e983e" + }, + { + "asset_id": "eomt-evaluation-helper", + "byte_length": 28915, + "kind": "repository-file", + "repository_path": "experiments/perception/worker/run_evaluation_prelabels.py", + "sha256": "25baf30c0df564734e08f38ace88cc4bc147cacf240c761622279511e361daa4" + }, + { + "asset_id": "eomt-image", + "byte_length": null, + "kind": "container-image", + "repository_path": null, + "sha256": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794" + }, + { + "asset_id": "eomt-model-weights", + "byte_length": 1276175488, + "kind": "model-artifact", + "repository_path": null, + "sha256": "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782" + }, + { + "asset_id": "eomt-orchestrator", + "byte_length": 21489, + "kind": "repository-file", + "repository_path": "experiments/perception/worker/Invoke-E4FullSessionSegmentation.ps1", + "sha256": "d3e9435939444ab35b27a744ac314e289ebd66a13fa56e3d59f121e088d22774" + }, + { + "asset_id": "eomt-preprocessor-config", + "byte_length": 666, + "kind": "model-artifact", + "repository_path": null, + "sha256": "97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7" + }, + { + "asset_id": "eomt-profile", + "byte_length": 3805, + "kind": "repository-file", + "repository_path": "experiments/perception/worker/e3_k1_camera1_profile.json", + "sha256": "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875" + }, + { + "asset_id": "eomt-profile-runtime", + "byte_length": 45789, + "kind": "repository-file", + "repository_path": "experiments/perception/worker/run_e3_rectified_segmentation.py", + "sha256": "01881862d4eaa218955f776a948124bf19c34be2b5ec282115daeacb15c53ae6" + }, + { + "asset_id": "eomt-recorded-runtime", + "byte_length": 34899, + "kind": "repository-file", + "repository_path": "experiments/perception/worker/run_recorded_perception_epoch.py", + "sha256": "4dcc4fc8bdf33702651a199be69d0dd4fadb243d2e65aee1c3d1ae7a58fdf675" + }, + { + "asset_id": "eomt-runner", + "byte_length": 30720, + "kind": "repository-file", + "repository_path": "experiments/perception/worker/run_e4_full_session_segmentation.py", + "sha256": "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4" + }, + { + "asset_id": "k1-calibration", + "byte_length": null, + "kind": "definition-component", + "repository_path": null, + "sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9" + }, + { + "asset_id": "k1-valid-fov-identity", + "byte_length": null, + "kind": "definition-component", + "repository_path": null, + "sha256": "b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2" + }, + { + "asset_id": "k1-valid-fov-mask", + "byte_length": null, + "kind": "definition-component", + "repository_path": null, + "sha256": "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63" + }, + { + "asset_id": "lab-v1-portable-contracts", + "byte_length": 114905, + "kind": "repository-file", + "repository_path": "src/k1link/observatory/portable_lab_v1_executor.py", + "sha256": "c8ba210336138617a1cdb02ed2795935b25a265ae7b905f75cd0338774338c55" + }, + { + "asset_id": "lab-v1-portable-worker", + "byte_length": 46859, + "kind": "repository-file", + "repository_path": "src/k1link/observatory/portable_lab_v1_worker.py", + "sha256": "ad00418d59b793328a281fa432e450919690fe4957f024f296e2d125c1b3c03a" + }, + { + "asset_id": "portable-result-contracts", + "byte_length": 22008, + "kind": "repository-file", + "repository_path": "src/k1link/observatory/portable_result_contract.py", + "sha256": "936d6f20789c26e9bed1c9e34ee51259368eb95bc7e3aa549821d1f7de79030c" + }, + { + "asset_id": "portable-worker-runtime", + "byte_length": 39120, + "kind": "repository-file", + "repository_path": "src/k1link/observatory/portable_worker_runtime.py", + "sha256": "205d32116f1ba75d5257d15cc79b5146574d93acd52028a5950f6fca4d8129cb" + }, + { + "asset_id": "vegetation-policy", + "byte_length": 3022, + "kind": "repository-file", + "repository_path": "config/perception/lab-v1-vegetation-mission-policy-v1.json", + "sha256": "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35" + }, + { + "asset_id": "vegetation-provider-map", + "byte_length": 2756, + "kind": "repository-file", + "repository_path": "config/perception/lab-v1-vegetation-provider-label-map-v1.json", + "sha256": "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352" + } + ], + "authority": { + "actuation_allowed": false, + "commands_enabled": false, + "navigation_or_safety_accepted": false, + "production_accepted": false + }, + "candidate_sha256": "6e7adbf25acbc3a9daedf8671ee57a7c00043ea73897bc9abf1e9947eaa5d0bd", + "declared_blockers": [ + "combined-executor-entrypoint-uninstalled", + "combined-executor-image-unsealed", + "commit-bound-source-unavailable", + "ddrnet-component-port-uninstalled", + "eomt-component-port-uninstalled", + "fixture-smoke-unaccepted", + "worker-installation-receipt-unavailable" + ], + "definition_id": "lab-v1-eomt-ddrnet-portable", + "definition_sha256": "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2", + "definition_version": 2, + "executor_image_sha256": null, + "phases": [ + "source-materialization", + "eomt-full-session", + "ddrnet-full-session", + "result-v2-assembly", + "result-v2-validation", + "portable-result-packaging" + ], + "release_id": "lab-v1-eomt-ddrnet-worker006-candidate-v2", + "result_contract_sha256": "b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a", + "schema_version": "missioncore.observatory-portable-lab-v1-executor-candidate/v1", + "setup_id": "lab-v1-eomt-ddrnet-portable-v1" +} diff --git a/experiments/perception/worker/observatory_portable/m49-tgs-portable-runner-source.json b/experiments/perception/worker/observatory_portable/m49-tgs-portable-runner-source.json new file mode 100644 index 0000000..a26f3c9 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/m49-tgs-portable-runner-source.json @@ -0,0 +1,51 @@ +{ + "schema_version": "missioncore.m49-tgs-portable-runner-source/v1", + "release_id": "m49-tgs-portable-runner-source-v1", + "worker_contour_id": "worker-006", + "container_image_sha256": "7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f", + "profile_sha256": "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9", + "input_contract": { + "schema_version": "missioncore.m49-tgs-portable-source-stage/v1", + "sequence": "ordered-kitti-xyzi-float32-files/v1", + "schedule_columns": [ + "timeline_frame_index", + "source_frame_index", + "session_seconds", + "available_slot", + "point_count" + ], + "timeline_frame_count": "source-derived", + "available_lidar_frame_count": "source-derived", + "missing_lidar_policy": "all-cells-unobserved" + }, + "intermediate_result_contract": { + "schema_version": "missioncore.m49-tgs-portable-intermediate/v1", + "outputs": "ground-and-nonground-xyzi-by-timeline-frame", + "timing": "one-row-per-timeline-frame", + "publication_ready": false + }, + "files": [ + { + "name": "run_m49_tgs_portable.cpp", + "byte_length": 11052, + "sha256": "52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9" + }, + { + "name": "run_m49_tgs_portable.sh", + "byte_length": 858, + "sha256": "2d6c32560682647f868e4ce4c2605749f17c60482a609f8c03ff951411f48ffb" + }, + { + "name": "smoke_m49_tgs_portable.sh", + "byte_length": 1117, + "sha256": "e5da73c2cf89ed0671de8ac52e13014a7faf79ce1ff1f97a090154063a61c18d" + } + ], + "authority": { + "commands_enabled": false, + "actuation_allowed": false, + "navigation_or_safety_accepted": false, + "production_accepted": false + }, + "source_release_sha256": "50818f50892ded83285a0fe2875066e32f2c5a13aa8af6c6ce55f63eef863569" +} diff --git a/experiments/perception/worker/observatory_portable/run_m49_tgs_portable.cpp b/experiments/perception/worker/observatory_portable/run_m49_tgs_portable.cpp new file mode 100644 index 0000000..8e4b927 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/run_m49_tgs_portable.cpp @@ -0,0 +1,266 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "travel/point_types.hpp" +#include "travel/tgs.hpp" + +namespace { + +using Clock = std::chrono::steady_clock; + +struct ScheduleRow { + std::size_t timeline_frame_index; + long long source_frame_index; + double session_seconds; + long long available_slot; + std::size_t point_count; +}; + +struct Schedule { + std::vector rows; + std::size_t available_count; +}; + +Schedule readSchedule(const std::string& path) { + std::ifstream input(path); + if (!input) { + throw std::runtime_error("cannot open portable TGS schedule"); + } + std::string line; + std::getline(input, line); + if (line != + "timeline_frame_index\tsource_frame_index\tsession_seconds\tavailable_slot\tpoint_count") { + throw std::runtime_error("portable TGS schedule header changed"); + } + + std::vector rows; + std::size_t available_count = 0; + long long previous_source_frame = -1; + double previous_session_seconds = -1.0; + while (std::getline(input, line)) { + if (line.empty()) { + throw std::runtime_error("portable TGS schedule contains an empty row"); + } + std::istringstream stream(line); + ScheduleRow row{}; + std::string trailing; + if (!(stream >> row.timeline_frame_index >> row.source_frame_index >> row.session_seconds >> + row.available_slot >> row.point_count) || + (stream >> trailing)) { + throw std::runtime_error("portable TGS schedule row is invalid"); + } + if (row.timeline_frame_index != rows.size() || row.source_frame_index < 0 || + row.source_frame_index <= previous_source_frame || !std::isfinite(row.session_seconds) || + row.session_seconds <= previous_session_seconds) { + throw std::runtime_error("portable TGS schedule order changed"); + } + if (row.available_slot == -1) { + if (row.point_count != 0) { + throw std::runtime_error("missing portable TGS frame contains points"); + } + } else if (row.available_slot < 0 || + static_cast(row.available_slot) != available_count || + row.point_count == 0) { + throw std::runtime_error("portable TGS available slot order changed"); + } else { + ++available_count; + } + rows.push_back(row); + previous_source_frame = row.source_frame_index; + previous_session_seconds = row.session_seconds; + } + if (rows.size() < 2 || available_count == 0) { + throw std::runtime_error("portable TGS schedule is incomplete"); + } + return Schedule{std::move(rows), available_count}; +} + +std::string framePath(const std::string& directory, std::size_t slot) { + std::ostringstream name; + name << directory << "/" << std::setw(6) << std::setfill('0') << slot << ".bin"; + return name.str(); +} + +void verifySequence(const std::string& directory, std::size_t expected_count) { + if (!std::filesystem::is_directory(directory)) { + throw std::runtime_error("portable TGS sequence directory is unavailable"); + } + std::size_t actual_count = 0; + for (const auto& entry : std::filesystem::directory_iterator(directory)) { + if (!entry.is_regular_file() || entry.path().extension() != ".bin") { + throw std::runtime_error("portable TGS sequence contains an unexpected member"); + } + ++actual_count; + } + if (actual_count != expected_count) { + throw std::runtime_error("portable TGS sequence and schedule differ"); + } + for (std::size_t slot = 0; slot < expected_count; ++slot) { + if (!std::filesystem::is_regular_file(framePath(directory, slot))) { + throw std::runtime_error("portable TGS sequence order changed"); + } + } +} + +travel::PointCloud::Ptr readXYZI( + const std::string& directory, + std::size_t slot, + std::size_t point_count) { + if (point_count > std::numeric_limits::max() / (4 * sizeof(float))) { + throw std::runtime_error("portable TGS point count exceeds the addressable bound"); + } + const auto path = framePath(directory, slot); + const auto expected_bytes = point_count * 4 * sizeof(float); + if (std::filesystem::file_size(path) != expected_bytes) { + throw std::runtime_error("portable TGS input byte count changed"); + } + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("cannot open portable TGS input"); + } + std::vector values(point_count * 4); + input.read( + reinterpret_cast(values.data()), + static_cast(expected_bytes)); + if (!input || input.peek() != std::ifstream::traits_type::eof()) { + throw std::runtime_error("cannot read exact portable TGS input"); + } + auto cloud = std::make_shared>(); + cloud->resize(point_count); + for (std::size_t index = 0; index < point_count; ++index) { + auto& point = cloud->at(index); + point.x = values[index * 4]; + point.y = values[index * 4 + 1]; + point.z = values[index * 4 + 2]; + point.intensity = values[index * 4 + 3]; + if (!std::isfinite(point.x) || !std::isfinite(point.y) || + !std::isfinite(point.z) || !std::isfinite(point.intensity)) { + throw std::runtime_error("portable TGS input contains a non-finite value"); + } + } + return cloud; +} + +void writeXYZI(const std::string& path, const travel::PointCloud& cloud) { + std::ofstream output(path, std::ios::binary); + if (!output) { + throw std::runtime_error("cannot open portable TGS output"); + } + for (const auto& point : cloud.points) { + const float row[4] = {point.x, point.y, point.z, point.intensity}; + output.write(reinterpret_cast(row), sizeof(row)); + } + if (!output) { + throw std::runtime_error("cannot write portable TGS output"); + } +} + +double milliseconds(Clock::duration duration) { + return std::chrono::duration(duration).count(); +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 5) { + std::cerr << "Usage: run_m49_tgs_portable " + " \n"; + return 1; + } + try { + const std::string sequence_dir = argv[1]; + const std::string schedule_path = argv[2]; + const std::string output_dir = argv[3]; + const std::string timing_path = argv[4]; + const auto schedule = readSchedule(schedule_path); + verifySequence(sequence_dir, schedule.available_count); + if (std::filesystem::exists(output_dir) || std::filesystem::exists(timing_path)) { + throw std::runtime_error("portable TGS output already exists"); + } + std::filesystem::create_directories(output_dir); + std::ofstream timing(timing_path); + if (!timing) { + throw std::runtime_error("cannot open portable TGS timing output"); + } + timing << "timeline_frame_index\tsource_frame_index\tsession_seconds\tsample_available" + << "\tavailable_slot\tinput_points\tground_points\tnonground_points\ttgs_ms" + << "\tstage_wall_ms\n"; + timing << std::fixed << std::setprecision(6); + + std::size_t completed_slots = 0; + for (const auto& row : schedule.rows) { + const auto stage_started = Clock::now(); + std::size_t input_points = 0; + std::size_t ground_points = 0; + std::size_t nonground_points = 0; + double tgs_seconds = 0.0; + if (row.available_slot >= 0) { + const auto slot = static_cast(row.available_slot); + auto input_xyzi = readXYZI(sequence_dir, slot, row.point_count); + if (input_xyzi->size() != row.point_count) { + throw std::runtime_error("portable TGS input point count changed"); + } + auto input = std::make_shared>(); + input->reserve(input_xyzi->size()); + for (const auto& point : input_xyzi->points) { + PointXYZILID value{}; + value.x = point.x; + value.y = point.y; + value.z = point.z; + value.intensity = point.intensity; + value.label = 0; + value.id = 0; + input->emplace_back(value); + } + travel::TravelGroundSeg tgs; + tgs.setParams( + 80.0, 1.0, 8.0, 3, 5, 10, 0.5, 0.125, 0.3, 0.940, 200.0, 0.03, + 0.1, 1.0, true, false); + travel::PointCloud ground; + travel::PointCloud nonground; + tgs.estimateGround(*input, ground, nonground, tgs_seconds); + input_points = input->size(); + ground_points = ground.size(); + nonground_points = nonground.size(); + const std::string base = output_dir + "/" + + std::to_string(row.timeline_frame_index); + writeXYZI(base + "_ground.bin", ground); + writeXYZI(base + "_nonground.bin", nonground); + ++completed_slots; + } + const auto completed = Clock::now(); + timing << row.timeline_frame_index << '\t' << row.source_frame_index << '\t' + << row.session_seconds << '\t' << (row.available_slot >= 0 ? 1 : 0) << '\t' + << row.available_slot << '\t' << input_points << '\t' << ground_points << '\t' + << nonground_points << '\t' << (tgs_seconds * 1000.0) << '\t' + << milliseconds(completed - stage_started) << '\n'; + if ((row.timeline_frame_index + 1) % 100 == 0) { + timing.flush(); + std::cout << "[TGS-PORTABLE] frame=" << (row.timeline_frame_index + 1) << "/" + << schedule.rows.size() << " available=" << completed_slots << "/" + << schedule.available_count << "\n"; + } + } + timing.flush(); + if (completed_slots != schedule.available_count) { + throw std::runtime_error("portable TGS available frame accounting changed"); + } + std::cout << "[TGS-PORTABLE] complete timeline=" << schedule.rows.size() + << " available=" << completed_slots << "\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "[TGS-PORTABLE] " << error.what() << '\n'; + return 2; + } +} diff --git a/experiments/perception/worker/observatory_portable/run_m49_tgs_portable.sh b/experiments/perception/worker/observatory_portable/run_m49_tgs_portable.sh new file mode 100755 index 0000000..9838254 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/run_m49_tgs_portable.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly PROFILE=/release/m49-tgs-portable-v2.json +readonly PROFILE_SHA256=6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9 +readonly SOURCE=/release/run_m49_tgs_portable.cpp +readonly INPUT_ROOT=/work/source/tgs +readonly OUTPUT_ROOT=/work/result +readonly BINARY=/tmp/run_m49_tgs_portable + +test -f "${PROFILE}" +test -f "${SOURCE}" +test -f "${INPUT_ROOT}/schedule.tsv" +test -d "${INPUT_ROOT}/sequence" +test ! -e "${OUTPUT_ROOT}" +echo "${PROFILE_SHA256} ${PROFILE}" | sha256sum --check --strict + +g++ -std=c++17 -O3 -DNDEBUG -pthread \ + -I/opt/travel/src/TRAVEL/cpp/travel/core \ + -I/usr/include/eigen3 \ + "${SOURCE}" \ + -o "${BINARY}" + +exec /usr/bin/time -v "${BINARY}" \ + "${INPUT_ROOT}/sequence/velodyne" \ + "${INPUT_ROOT}/schedule.tsv" \ + "${OUTPUT_ROOT}/outputs" \ + "${OUTPUT_ROOT}/timing.tsv" diff --git a/experiments/perception/worker/observatory_portable/smoke_m49_tgs_portable.sh b/experiments/perception/worker/observatory_portable/smoke_m49_tgs_portable.sh new file mode 100644 index 0000000..0057b44 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/smoke_m49_tgs_portable.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly SMOKE_ROOT=/smoke +readonly FIXTURE=/opt/travel/fixture/00/velodyne/000000.bin +readonly RUNNER=/opt/nodedc/m49-tgs-portable/bin/run_m49_tgs_portable +readonly EXPECTED_FRAME_BYTES=1994688 +readonly EXPECTED_POINTS=124668 + +mkdir -p "${SMOKE_ROOT}/sequence" +cp "${FIXTURE}" "${SMOKE_ROOT}/sequence/000000.bin" +cp "${FIXTURE}" "${SMOKE_ROOT}/sequence/000001.bin" +printf 'timeline_frame_index\tsource_frame_index\tsession_seconds\tavailable_slot\tpoint_count\n0\t0\t0.0\t0\t124668\n1\t1\t0.1\t1\t124668\n' \ + > "${SMOKE_ROOT}/schedule.tsv" + +"${RUNNER}" \ + "${SMOKE_ROOT}/sequence" \ + "${SMOKE_ROOT}/schedule.tsv" \ + "${SMOKE_ROOT}/outputs" \ + "${SMOKE_ROOT}/timing.tsv" + +test "$(wc -l < "${SMOKE_ROOT}/timing.tsv")" -eq 3 +for index in 0 1; do + ground_bytes="$(stat -c %s "${SMOKE_ROOT}/outputs/${index}_ground.bin")" + nonground_bytes="$(stat -c %s "${SMOKE_ROOT}/outputs/${index}_nonground.bin")" + test "$((ground_bytes + nonground_bytes))" -eq "${EXPECTED_FRAME_BYTES}" +done + +printf 'M49_PORTABLE_FIXTURE_SMOKE_OK frames=2 points_per_frame=%s\n' "${EXPECTED_POINTS}" diff --git a/scripts/build_m49_portable_executor_release.py b/scripts/build_m49_portable_executor_release.py new file mode 100644 index 0000000..9767069 --- /dev/null +++ b/scripts/build_m49_portable_executor_release.py @@ -0,0 +1,588 @@ +#!/usr/bin/env python3 +"""Build and verify a deterministic blocked M4.9 executor release candidate. + +This builder deliberately cannot mark the executor ready. It seals the exact +portable source materializer, generic TRAVEL/TGS runner, result-v2 assembler +and validator into a reproducible candidate archive. A later installation +step must additionally provide an exact compiled runner, an executor image and +an installation receipt before the runtime registry can become ready. +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import io +import json +import os +import re +import stat +import subprocess +import tarfile +import tempfile +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Final, cast + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +M49_EXECUTOR_RELEASE_CANDIDATE_SCHEMA: Final = ( + "missioncore.m49-tgs-portable-executor-release-candidate/v1" +) +M49_COMPILED_RUNNER_BUILD_SCHEMA: Final = "missioncore.m49-tgs-portable-compiled-runner-build/v1" +M49_EXECUTOR_RELEASE_ID: Final = "m49-tgs-portable-executor-v1" +M49_TRAVEL_IMAGE_SHA256: Final = "7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f" +M49_PROFILE_SHA256: Final = "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9" +M49_RESULT_CONTRACT_SHA256: Final = ( + "9dd80c8e2504559d2156fca933de6eb27901e35305e6853aeb84707e1cb13892" +) +M49_RUNNER_SOURCE_SHA256: Final = "52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9" +M49_RUNNER_WRAPPER_SHA256: Final = ( + "2d6c32560682647f868e4ce4c2605749f17c60482a609f8c03ff951411f48ffb" +) +M49_COMPILER_CONTRACT: Final = { + "compiler": "g++", + "language_standard": "c++17", + "flags": ["-O3", "-DNDEBUG", "-pthread"], + "travel_include": "/opt/travel/src/TRAVEL/cpp/travel/core", + "eigen_include": "/usr/include/eigen3", +} +M49_RELEASE_BLOCKERS: Final = ( + "compiled-runner-artifact-missing", + "exact-executor-image-missing", + "worker-installation-receipt-missing", +) +M49_RELEASE_SOURCES: Final = ( + Path("config/perception/m49-tgs-portable-v2.json"), + Path("experiments/perception/worker/observatory_portable/Dockerfile.m49-portable-executor"), + Path( + "experiments/perception/worker/observatory_portable/" + "Invoke-M49PortableExecutorCandidateInstall.ps1" + ), + Path("experiments/perception/worker/observatory_portable/m49-tgs-portable-runner-source.json"), + Path("experiments/perception/worker/observatory_portable/run_m49_tgs_portable.cpp"), + Path("experiments/perception/worker/observatory_portable/run_m49_tgs_portable.sh"), + Path("experiments/perception/worker/observatory_portable/smoke_m49_tgs_portable.sh"), + Path("experiments/perception/worker/m49_t3_travel/build_tgs_fail_closed_evidence.py"), + Path("experiments/perception/worker/m49_t3_travel/build_tgs_full_shadow_binary.sh"), + Path("experiments/perception/worker/m49_t3_travel/build_tgs_full_shadow_evidence.py"), + Path("src/k1link/compute/lidar_replay.py"), + Path("src/k1link/observatory/m49_portable_executor.py"), + Path("src/k1link/observatory/m49_portable_result.py"), + Path("src/k1link/observatory/m49_portable_source.py"), + Path("src/k1link/observatory/portable_result_contract.py"), +) +_AUTHORITY: Final = { + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, + "production_accepted": False, +} +_REVISION: Final = re.compile(r"^[a-f0-9]{40}$") +_SHA256: Final = re.compile(r"^[a-f0-9]{64}$") +_HASH_CHUNK_BYTES: Final = 1024 * 1024 + + +class M49ExecutorReleaseBuildError(RuntimeError): + """The exact portable executor candidate cannot be built or verified.""" + + +@dataclass(frozen=True, slots=True) +class BuiltM49ExecutorReleaseCandidate: + archive: Path + archive_sha256: str + candidate_sha256: str + manifest: dict[str, object] + + +@dataclass(frozen=True, slots=True) +class BuiltM49CompiledRunnerSeal: + binary: Path + manifest_path: Path + manifest_sha256: str + manifest: dict[str, object] + + +def seal_m49_compiled_runner_build( + *, + source_root: Path, + source_revision: str, + binary_path: Path, + manifest_path: Path, +) -> BuiltM49CompiledRunnerSeal: + """Seal, but never execute, a binary built in the exact TRAVEL image.""" + + if _REVISION.fullmatch(source_revision) is None: + raise M49ExecutorReleaseBuildError("M4.9 compiled-runner revision is invalid") + root = source_root.expanduser().resolve(strict=True) + if root.is_symlink() or not root.is_dir(): + raise M49ExecutorReleaseBuildError("M4.9 compiled-runner source root is unsafe") + source = root / "experiments/perception/worker/observatory_portable/run_m49_tgs_portable.cpp" + wrapper = root / "experiments/perception/worker/observatory_portable/run_m49_tgs_portable.sh" + profile = root / "config/perception/m49-tgs-portable-v2.json" + if ( + _sha256_file(source) != M49_RUNNER_SOURCE_SHA256 + or _sha256_file(wrapper) != M49_RUNNER_WRAPPER_SHA256 + or _sha256_file(profile) != M49_PROFILE_SHA256 + ): + raise M49ExecutorReleaseBuildError( + "M4.9 compiled-runner sources differ from the exact release" + ) + binary = binary_path.expanduser().absolute() + try: + metadata = binary.lstat() + resolved_binary = binary.resolve(strict=True) + except OSError as exc: + raise M49ExecutorReleaseBuildError("M4.9 compiled runner is unavailable") from exc + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or not os.path.samefile(binary, resolved_binary) + or not os.access(resolved_binary, os.X_OK) + or not _is_elf(resolved_binary) + ): + raise M49ExecutorReleaseBuildError("M4.9 compiled runner is not an executable ELF artifact") + manifest: dict[str, object] = { + "schema_version": M49_COMPILED_RUNNER_BUILD_SCHEMA, + "source_revision": source_revision, + "source_state": "committed-snapshot", + "build_image_sha256": M49_TRAVEL_IMAGE_SHA256, + "profile_sha256": M49_PROFILE_SHA256, + "runner_source_sha256": M49_RUNNER_SOURCE_SHA256, + "runner_wrapper_sha256": M49_RUNNER_WRAPPER_SHA256, + "compiler_contract": dict(M49_COMPILER_CONTRACT), + "binary": { + "file_name": "run_m49_tgs_portable", + "format": "elf", + "byte_length": resolved_binary.stat().st_size, + "sha256": _sha256_file(resolved_binary), + }, + "authority": dict(_AUTHORITY), + } + payload = _canonical_json(manifest) + target = manifest_path.expanduser().absolute() + if target.exists(): + raise M49ExecutorReleaseBuildError("M4.9 compiled-runner seal already exists") + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + if target.parent.is_symlink() or not target.parent.is_dir(): + raise M49ExecutorReleaseBuildError("M4.9 compiled-runner seal root is unsafe") + target.write_bytes(payload) + return BuiltM49CompiledRunnerSeal( + binary=resolved_binary, + manifest_path=target, + manifest_sha256=hashlib.sha256(payload).hexdigest(), + manifest=manifest, + ) + + +def git_revision(repository_root: Path = REPOSITORY_ROOT) -> str: + completed = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repository_root, + check=False, + capture_output=True, + text=True, + ) + revision = completed.stdout.strip() + if completed.returncode != 0 or _REVISION.fullmatch(revision) is None: + raise M49ExecutorReleaseBuildError("M4.9 release revision is unavailable") + return revision + + +def materialize_revision( + *, revision: str, destination: Path, repository_root: Path = REPOSITORY_ROOT +) -> None: + """Extract only the explicit release source set from one exact commit.""" + + if _REVISION.fullmatch(revision) is None or destination.exists(): + raise M49ExecutorReleaseBuildError("M4.9 release revision request is invalid") + verified = subprocess.run( + ["git", "rev-parse", "--verify", f"{revision}^{{commit}}"], + cwd=repository_root, + check=False, + capture_output=True, + text=True, + ) + if verified.returncode != 0 or verified.stdout.strip() != revision: + raise M49ExecutorReleaseBuildError("M4.9 release revision is not a commit") + archive_path = destination.parent / "source.tar" + archived = subprocess.run( + [ + "git", + "archive", + "--format=tar", + "--output", + str(archive_path), + revision, + "--", + *(path.as_posix() for path in M49_RELEASE_SOURCES), + ], + cwd=repository_root, + check=False, + capture_output=True, + text=True, + ) + if archived.returncode != 0: + raise M49ExecutorReleaseBuildError( + "M4.9 release sources are not all present in the selected commit" + ) + destination.mkdir() + resolved_root = destination.resolve() + with tarfile.open(archive_path, "r:") as archive: + members = archive.getmembers() + for member in members: + target = (destination / member.name).resolve() + if ( + target != resolved_root + and resolved_root not in target.parents + or not (member.isdir() or member.isreg()) + ): + raise M49ExecutorReleaseBuildError("M4.9 Git archive contains an unsafe member") + for member in members: + target = destination / member.name + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + continue + source = archive.extractfile(member) + if source is None: + raise M49ExecutorReleaseBuildError("M4.9 Git archive member is unreadable") + target.parent.mkdir(parents=True, exist_ok=True) + with source, target.open("wb") as output: + while chunk := source.read(_HASH_CHUNK_BYTES): + output.write(chunk) + + +def build_m49_executor_release_candidate( + *, + source_root: Path, + output_directory: Path, + source_revision: str, + source_state: str, +) -> BuiltM49ExecutorReleaseCandidate: + """Build a deterministic candidate; never a ready installation artifact.""" + + if _REVISION.fullmatch(source_revision) is None: + raise M49ExecutorReleaseBuildError("M4.9 source revision is invalid") + if source_state not in {"committed-snapshot", "uncommitted-candidate"}: + raise M49ExecutorReleaseBuildError("M4.9 source state is invalid") + root = source_root.expanduser().resolve(strict=True) + if root.is_symlink() or not root.is_dir(): + raise M49ExecutorReleaseBuildError("M4.9 source root is unsafe") + files = _source_inventory(root) + identity: dict[str, object] = { + "schema_version": M49_EXECUTOR_RELEASE_CANDIDATE_SCHEMA, + "release_id": M49_EXECUTOR_RELEASE_ID, + "state": "blocked", + "source_revision": source_revision, + "source_state": source_state, + "worker_contour_id": "worker-006", + "travel_build_image_sha256": M49_TRAVEL_IMAGE_SHA256, + "executor_image_sha256": None, + "compiled_runner": None, + "profile_sha256": M49_PROFILE_SHA256, + "result_contract_sha256": M49_RESULT_CONTRACT_SHA256, + "source_contract": { + "camera": "exact-admitted-fmp4-members", + "spatial_replay": "exact-admitted-k1mqtt-member", + "host_time_metadata": "separate-exact-admitted-member-required", + "server_paths_or_commands_allowed": False, + }, + "phases": [ + {"phase_id": "source-materializer", "state": "implemented"}, + {"phase_id": "travel-tgs-runner", "state": "implemented"}, + {"phase_id": "result-v2-assembler", "state": "implemented"}, + {"phase_id": "exact-result-validator", "state": "implemented"}, + {"phase_id": "package-sealer", "state": "implemented"}, + ], + "files": files, + "blockers": list(M49_RELEASE_BLOCKERS), + "authority": dict(_AUTHORITY), + } + candidate_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + manifest: dict[str, object] = { + **identity, + "candidate_sha256": candidate_sha256, + } + output = output_directory.expanduser().absolute() + output.mkdir(parents=True, exist_ok=True) + if output.is_symlink() or not output.is_dir(): + raise M49ExecutorReleaseBuildError("M4.9 release output is unsafe") + target = output / f"m49-tgs-portable-executor-{candidate_sha256}.tgz" + if target.exists(): + verified = verify_m49_executor_release_candidate(target) + if verified.candidate_sha256 != candidate_sha256: + raise M49ExecutorReleaseBuildError( + "existing M4.9 release candidate has another identity" + ) + return verified + with tempfile.TemporaryDirectory(prefix="m49-executor-candidate-") as temporary: + stage = Path(temporary) + payload = stage / "payload" + for source in M49_RELEASE_SOURCES: + destination = payload / source + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes((root / source).read_bytes()) + (stage / "release-manifest.json").write_bytes(_canonical_json(manifest)) + _write_archive(stage, target) + verified = verify_m49_executor_release_candidate(target) + if verified.candidate_sha256 != candidate_sha256: + raise M49ExecutorReleaseBuildError("built M4.9 release candidate changed identity") + return verified + + +def verify_m49_executor_release_candidate( + archive_path: Path, +) -> BuiltM49ExecutorReleaseCandidate: + candidate = archive_path.expanduser().resolve(strict=True) + if candidate.is_symlink() or not candidate.is_file(): + raise M49ExecutorReleaseBuildError("M4.9 release archive is unsafe") + with tarfile.open(candidate, "r:gz") as archive: + members = archive.getmembers() + names = [member.name for member in members] + payload_names: set[str] = set() + for source in M49_RELEASE_SOURCES: + parts = PurePosixPath("payload", *source.parts) + for parent in reversed(parts.parents): + name = parent.as_posix() + if name not in {".", "payload"}: + payload_names.add(name) + payload_names.add(parts.as_posix()) + expected_names = [ + "release-manifest.json", + "payload", + *sorted(payload_names), + ] + if names != expected_names: + raise M49ExecutorReleaseBuildError("M4.9 release archive member set changed") + by_name = {member.name: member for member in members} + for member in members: + path = PurePosixPath(member.name) + if ( + path.is_absolute() + or any(part in {"", ".", ".."} for part in path.parts) + or not (member.isdir() or member.isreg()) + or member.uid != 0 + or member.gid != 0 + or member.mtime != 0 + ): + raise M49ExecutorReleaseBuildError("M4.9 release archive metadata is unsafe") + manifest_stream = archive.extractfile(by_name["release-manifest.json"]) + if manifest_stream is None: + raise M49ExecutorReleaseBuildError("M4.9 release manifest is unavailable") + manifest_payload = manifest_stream.read() + try: + manifest = _object(json.loads(manifest_payload), "M4.9 release manifest") + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise M49ExecutorReleaseBuildError("M4.9 release manifest is invalid JSON") from exc + if manifest_payload != _canonical_json(manifest): + raise M49ExecutorReleaseBuildError("M4.9 release manifest is not canonical JSON") + candidate_sha256 = cast(str, manifest.pop("candidate_sha256", None)) + if ( + _SHA256.fullmatch(candidate_sha256 or "") is None + or manifest.get("schema_version") != M49_EXECUTOR_RELEASE_CANDIDATE_SCHEMA + or manifest.get("state") != "blocked" + or tuple(cast(list[object], manifest.get("blockers"))) != M49_RELEASE_BLOCKERS + or manifest.get("authority") != _AUTHORITY + or hashlib.sha256(_canonical_json(manifest)).hexdigest() != candidate_sha256 + ): + raise M49ExecutorReleaseBuildError("M4.9 release candidate identity changed") + files = cast(list[object], manifest.get("files")) + expected_files = {path.as_posix() for path in M49_RELEASE_SOURCES} + if {cast(dict[str, object], row).get("relative_path") for row in files} != expected_files: + raise M49ExecutorReleaseBuildError("M4.9 release file set changed") + for value in files: + row = _object(value, "M4.9 release file") + relative = cast(str, row["relative_path"]) + member = by_name[f"payload/{relative}"] + stream = archive.extractfile(member) + if stream is None: + raise M49ExecutorReleaseBuildError("M4.9 release file is unavailable") + payload = stream.read() + if ( + row.get("byte_length") != len(payload) + or row.get("sha256") != hashlib.sha256(payload).hexdigest() + ): + raise M49ExecutorReleaseBuildError("M4.9 release file changed") + restored_manifest = {**manifest, "candidate_sha256": candidate_sha256} + return BuiltM49ExecutorReleaseCandidate( + archive=candidate, + archive_sha256=_sha256_file(candidate), + candidate_sha256=candidate_sha256, + manifest=restored_manifest, + ) + + +def write_worktree_candidate_manifest( + *, source_root: Path, target: Path, source_revision: str +) -> dict[str, object]: + """Write a reviewable manifest without pretending it is a Git release.""" + + files = _source_inventory(source_root.expanduser().resolve(strict=True)) + identity: dict[str, object] = { + "schema_version": M49_EXECUTOR_RELEASE_CANDIDATE_SCHEMA, + "release_id": M49_EXECUTOR_RELEASE_ID, + "state": "blocked", + "source_revision": source_revision, + "source_state": "uncommitted-candidate", + "worker_contour_id": "worker-006", + "travel_build_image_sha256": M49_TRAVEL_IMAGE_SHA256, + "executor_image_sha256": None, + "compiled_runner": None, + "profile_sha256": M49_PROFILE_SHA256, + "result_contract_sha256": M49_RESULT_CONTRACT_SHA256, + "source_contract": { + "camera": "exact-admitted-fmp4-members", + "spatial_replay": "exact-admitted-k1mqtt-member", + "host_time_metadata": "separate-exact-admitted-member-required", + "server_paths_or_commands_allowed": False, + }, + "phases": [ + {"phase_id": "source-materializer", "state": "implemented"}, + {"phase_id": "travel-tgs-runner", "state": "implemented"}, + {"phase_id": "result-v2-assembler", "state": "implemented"}, + {"phase_id": "exact-result-validator", "state": "implemented"}, + {"phase_id": "package-sealer", "state": "implemented"}, + ], + "files": files, + "blockers": [ + *M49_RELEASE_BLOCKERS, + "committed-source-snapshot-missing", + ], + "authority": dict(_AUTHORITY), + } + manifest: dict[str, object] = { + **identity, + "candidate_sha256": hashlib.sha256(_canonical_json(identity)).hexdigest(), + } + destination = target.expanduser().absolute() + if destination.exists(): + raise M49ExecutorReleaseBuildError("worktree candidate manifest already exists") + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(_canonical_json(manifest)) + return manifest + + +def _source_inventory(root: Path) -> list[dict[str, object]]: + rows: list[dict[str, object]] = [] + for relative in M49_RELEASE_SOURCES: + path = root / relative + try: + resolved = path.resolve(strict=True) + except OSError as exc: + raise M49ExecutorReleaseBuildError( + f"M4.9 release source is unavailable: {relative}" + ) from exc + if path.is_symlink() or not resolved.is_file() or not resolved.is_relative_to(root): + raise M49ExecutorReleaseBuildError(f"M4.9 release source is unsafe: {relative}") + rows.append( + { + "relative_path": relative.as_posix(), + "byte_length": resolved.stat().st_size, + "sha256": _sha256_file(resolved), + } + ) + return rows + + +def _write_archive(stage: Path, target: Path) -> None: + members = [stage / "release-manifest.json", stage / "payload"] + members.extend(sorted((stage / "payload").rglob("*"))) + target.parent.mkdir(parents=True, exist_ok=True) + with ( + target.open("wb") as raw, + gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed, + tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive, + ): + for path in members: + relative = path.relative_to(stage).as_posix() + info = tarfile.TarInfo(relative) + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + info.mtime = 0 + if path.is_dir(): + info.type = tarfile.DIRTYPE + info.mode = 0o755 + archive.addfile(info, io.BytesIO()) + else: + info.type = tarfile.REGTYPE + info.mode = 0o755 if path.suffix in {".sh", ".py"} else 0o644 + info.size = path.stat().st_size + with path.open("rb") as stream: + archive.addfile(info, stream) + + +def _canonical_json(value: object) -> bytes: + try: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise M49ExecutorReleaseBuildError("M4.9 release manifest is not JSON-compatible") from exc + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(_HASH_CHUNK_BYTES), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _is_elf(path: Path) -> bool: + try: + with path.open("rb") as stream: + return stream.read(4) == b"\x7fELF" + except OSError: + return False + + +def _object(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise M49ExecutorReleaseBuildError(f"{label} must be an object") + return cast(dict[str, object], value) + + +def _parse_arguments(arguments: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--output-directory", type=Path, required=True) + parser.add_argument("--revision") + return parser.parse_args(arguments) + + +def main(arguments: Sequence[str] | None = None) -> int: + options = _parse_arguments(arguments) + revision = options.revision or git_revision() + with tempfile.TemporaryDirectory(prefix="m49-revision-") as temporary: + snapshot = Path(temporary) / "source" + materialize_revision(revision=revision, destination=snapshot) + built = build_m49_executor_release_candidate( + source_root=snapshot, + output_directory=options.output_directory, + source_revision=revision, + source_state="committed-snapshot", + ) + print( + json.dumps( + { + "ok": True, + "state": "blocked", + "artifact": str(built.archive), + "sha256": built.archive_sha256, + "candidate_sha256": built.candidate_sha256, + "blockers": list(M49_RELEASE_BLOCKERS), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/plan_observatory_worker_tunnel.py b/scripts/plan_observatory_worker_tunnel.py new file mode 100644 index 0000000..e5b90c0 --- /dev/null +++ b/scripts/plan_observatory_worker_tunnel.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Print the immutable Mac launchd plan for the Worker 006 reverse tunnel.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from k1link.observatory.worker_tunnel_launchd import ( + plan_observatory_worker_tunnel_launch_agent, +) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--data-directory", type=Path, required=True) + parser.add_argument( + "--agent-path", + type=Path, + default=( + Path.home() + / "Library/LaunchAgents/com.nodedc.observatory-worker-tunnel.local.plist" + ), + ) + parser.add_argument("--ssh-path", type=Path, default=Path("/usr/bin/ssh")) + arguments = parser.parse_args() + plan = plan_observatory_worker_tunnel_launch_agent( + data_directory=arguments.data_directory, + agent_path=arguments.agent_path, + ssh_path=arguments.ssh_path, + ) + print(json.dumps(plan.to_dict(), indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/k1link/observatory/__init__.py b/src/k1link/observatory/__init__.py index 965836e..42211f8 100644 --- a/src/k1link/observatory/__init__.py +++ b/src/k1link/observatory/__init__.py @@ -24,6 +24,7 @@ from k1link.observatory.run_preparations import ( from k1link.observatory.setups import ( LABORATORY_SETUP_CATALOG_SCHEMA, LABORATORY_SETUP_REGISTRY_SCHEMA, + OBSERVATORY_CALCULATION_PROFILE_SCHEMA, LaboratorySetupRegistry, LaboratorySetupRegistryError, ) @@ -31,6 +32,7 @@ from k1link.observatory.setups import ( __all__ = [ "LABORATORY_SETUP_CATALOG_SCHEMA", "LABORATORY_SETUP_REGISTRY_SCHEMA", + "OBSERVATORY_CALCULATION_PROFILE_SCHEMA", "MAX_RUN_PREPARATION_RECORDS", "MAX_RUN_PREPARATION_STORAGE_BYTES", "OBSERVATORY_RUN_PREPARATION_REQUEST_SCHEMA", diff --git a/src/k1link/observatory/m49_portable_executor.py b/src/k1link/observatory/m49_portable_executor.py new file mode 100644 index 0000000..3e848ea --- /dev/null +++ b/src/k1link/observatory/m49_portable_executor.py @@ -0,0 +1,523 @@ +"""Exact local composition for the portable M4.9 TRAVEL/TGS executor. + +The server supplies only a sealed recorded-job identity. Source delivery and +result upload remain implementations of the shared portable Worker ports; all +filesystem locations, the compiled runner and its build seal are selected by +reviewed Worker-local configuration. + +This module does not register an executor or make a blocked runtime ready. It +is the adapter that an installer may bind only after the release archive, +compiled runner, executor image and local admission have all been sealed. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import stat +import subprocess +import tempfile +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Protocol, cast + +from k1link.observatory.m49_portable_result import ( + M49_PORTABLE_PROFILE_SHA256, + M49_PORTABLE_RESULT_CONTRACT_SHA256, + assemble_m49_portable_result, +) +from k1link.observatory.m49_portable_source import ( + M49_PORTABLE_STAGE_SCHEDULE, + M49_PORTABLE_TGS_SEQUENCE, + M49PortableSourceStage, + materialize_m49_portable_source_from_worker_stage, + validate_m49_portable_source_stage, +) +from k1link.observatory.portable_result_contract import ( + OBSERVATION_ONLY_AUTHORITY, + canonical_json, +) +from k1link.observatory.portable_run_definitions import PortableRunDefinition +from k1link.observatory.portable_worker_runtime import ( + PortableWorkerExecutorAdapter, + PortableWorkerResultDraft, + PortableWorkerResultPublisher, + PortableWorkerRuntimeAdmission, + PortableWorkerRuntimeCandidate, + PortableWorkerRuntimeJobRejectedError, + PortableWorkerRuntimePlan, + PortableWorkerRuntimeUnavailableError, + PortableWorkerSourceMaterializer, + PortableWorkerSourceStage, +) +from k1link.observatory.worker_agent import SealedObservatoryRecordedJob + +M49_COMPILED_RUNNER_BUILD_SCHEMA: Final = "missioncore.m49-tgs-portable-compiled-runner-build/v1" +M49_PORTABLE_RUNNER_SOURCE_SHA256: Final = ( + "52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9" +) +M49_PORTABLE_RUNNER_WRAPPER_SHA256: Final = ( + "2d6c32560682647f868e4ce4c2605749f17c60482a609f8c03ff951411f48ffb" +) +M49_PORTABLE_TRAVEL_BUILD_IMAGE_SHA256: Final = ( + "7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f" +) +M49_PORTABLE_RUNTIME_PHASES: Final = ( + "source-delivery", + "camera-lidar-timeline-materializer", + "portable-tgs-input-materializer", + "portable-tgs-runner", + "result-v2-assembler", + "observatory-result-publisher", +) +M49_PORTABLE_COMPILED_RUNNER_ASSET_ID: Final = "m49-portable-compiled-runner" +M49_PORTABLE_COMPILED_RUNNER_BUILD_SEAL_ASSET_ID: Final = "m49-portable-compiled-runner-build-seal" +M49_PORTABLE_PROFILE_ASSET_ID: Final = "m49-portable-profile" +M49_PORTABLE_TRAVEL_IMAGE_ASSET_ID: Final = "travel-tgs-image" +M49_PORTABLE_COMPILER_CONTRACT: Final = { + "compiler": "g++", + "language_standard": "c++17", + "flags": ["-O3", "-DNDEBUG", "-pthread"], + "travel_include": "/opt/travel/src/TRAVEL/cpp/travel/core", + "eigen_include": "/usr/include/eigen3", +} + +_MAX_BUILD_SEAL_BYTES: Final = 128 * 1024 +_MAX_RUN_SECONDS: Final = 24 * 60 * 60 + + +class M49PortableExecutorError(PortableWorkerRuntimeUnavailableError): + """The local M4.9 executor installation or invocation is not exact.""" + + +class M49PortableRunnerInvoker(Protocol): + def __call__( + self, + *, + binary: Path, + sequence: Path, + schedule: Path, + output: Path, + timing: Path, + workspace: Path, + timeout_seconds: int, + ) -> None: ... + + +@dataclass(frozen=True, slots=True) +class M49PortableRunnerInstallation: + """Worker-local paths bound to an exact build-only runner seal.""" + + profile_path: Path + runner_binary_path: Path + runner_build_seal_path: Path + runner_build_seal_sha256: str + output_parent: Path + timeout_seconds: int = _MAX_RUN_SECONDS + + def __post_init__(self) -> None: + profile = _exact_file( + self.profile_path, + M49_PORTABLE_PROFILE_SHA256, + "portable M4.9 profile", + ) + seal = _exact_file( + self.runner_build_seal_path, + self.runner_build_seal_sha256, + "portable M4.9 runner build seal", + ) + binary = _regular_file(self.runner_binary_path, "portable M4.9 compiled runner") + document = _read_build_seal(seal) + binary_row = _object(document["binary"], "portable M4.9 build-seal binary") + if binary_row != { + "file_name": "run_m49_tgs_portable", + "format": "elf", + "byte_length": binary.stat().st_size, + "sha256": _sha256_file(binary), + } or document["profile_sha256"] != _sha256_file(profile): + raise M49PortableExecutorError( + "portable M4.9 compiled runner differs from its build seal" + ) + if not os.access(binary, os.X_OK): + raise M49PortableExecutorError("portable M4.9 compiled runner is not executable") + if not _is_elf(binary): + raise M49PortableExecutorError("portable M4.9 compiled runner is not ELF") + parent = self.output_parent.expanduser().absolute() + parent.mkdir(mode=0o700, parents=True, exist_ok=True) + if parent.is_symlink() or not parent.is_dir(): + raise M49PortableExecutorError("portable M4.9 output root is unsafe") + if ( + isinstance(self.timeout_seconds, bool) + or not isinstance(self.timeout_seconds, int) + or not 1 <= self.timeout_seconds <= _MAX_RUN_SECONDS + ): + raise ValueError("portable M4.9 runner timeout is invalid") + object.__setattr__(self, "profile_path", profile) + object.__setattr__(self, "runner_binary_path", binary) + object.__setattr__(self, "runner_build_seal_path", seal) + object.__setattr__(self, "output_parent", parent) + + @property + def runner_binary_sha256(self) -> str: + binary = _object( + _read_build_seal(self.runner_build_seal_path)["binary"], + "portable M4.9 build-seal binary", + ) + return cast(str, binary["sha256"]) + + +@dataclass(frozen=True, slots=True) +class M49PortableBoundSourceStage(PortableWorkerSourceStage): + """Claim-bound in-memory extension of the shared source-stage port.""" + + job: SealedObservatoryRecordedJob + m49_stage: M49PortableSourceStage + + def __post_init__(self) -> None: + PortableWorkerSourceStage.__post_init__(self) + if ( + self.root != self.m49_stage.root + or self.source_bundle_sha256 != self.job.source_bundle_sha256 + or self.source_capability_manifest_sha256 != self.job.source_capability_manifest_sha256 + or self.source_adapter_sha256 != self.job.source_adapter_sha256 + ): + raise M49PortableExecutorError("portable M4.9 bound source differs from its sealed job") + + +@dataclass(frozen=True, slots=True) +class M49PortableSourceMaterializerAdapter: + """Adapt exact Worker transport delivery to the M4.9 TGS source port.""" + + upstream: PortableWorkerSourceMaterializer + profile_path: Path + output_parent: Path + + def __post_init__(self) -> None: + profile = _exact_file( + self.profile_path, + M49_PORTABLE_PROFILE_SHA256, + "portable M4.9 profile", + ) + parent = self.output_parent.expanduser().absolute() + parent.mkdir(mode=0o700, parents=True, exist_ok=True) + if parent.is_symlink() or not parent.is_dir(): + raise M49PortableExecutorError("portable M4.9 source output root is unsafe") + object.__setattr__(self, "profile_path", profile) + object.__setattr__(self, "output_parent", parent) + + def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage: + delivered = self.upstream.materialize(job) + materialized = materialize_m49_portable_source_from_worker_stage( + worker_stage=delivered, + job=job, + profile_path=self.profile_path, + output_parent=self.output_parent, + ) + return M49PortableBoundSourceStage( + root=materialized.root, + source_bundle_sha256=job.source_bundle_sha256, + source_capability_manifest_sha256=job.source_capability_manifest_sha256, + source_adapter_sha256=job.source_adapter_sha256, + job=job, + m49_stage=materialized, + ) + + +@dataclass(frozen=True, slots=True) +class M49PortableProfileRunnerAdapter: + """Run the exact installed binary and assemble the deterministic result-v2.""" + + definition: PortableRunDefinition + installation: M49PortableRunnerInstallation + created_at_utc: Callable[[], str] + invoker: M49PortableRunnerInvoker | None = None + + def __post_init__(self) -> None: + _verify_definition(self.definition) + + def run( + self, + plan: PortableWorkerRuntimePlan, + source: PortableWorkerSourceStage, + ) -> PortableWorkerResultDraft: + if not isinstance(source, M49PortableBoundSourceStage): + raise PortableWorkerRuntimeJobRejectedError( + "portable M4.9 runner requires its claim-bound source stage" + ) + job = source.job + _verify_plan(plan, job=job, definition=self.definition) + stage = validate_m49_portable_source_stage(source.root) + _verify_installation_unchanged(self.installation) + workspace = Path( + tempfile.mkdtemp(prefix=".m49-portable-run-", dir=self.installation.output_parent) + ) + try: + output = workspace / "outputs" + timing = workspace / "timing.tsv" + invoker = self.invoker or _invoke_exact_runner + invoker( + binary=self.installation.runner_binary_path, + sequence=stage.root / M49_PORTABLE_TGS_SEQUENCE, + schedule=stage.root / M49_PORTABLE_STAGE_SCHEDULE, + output=output, + timing=timing, + workspace=workspace, + timeout_seconds=self.installation.timeout_seconds, + ) + package = assemble_m49_portable_result( + source_stage_root=stage.root, + runner_output_root=output, + runner_timing_path=timing, + profile_path=self.installation.profile_path, + output_parent=self.installation.output_parent / "result-packages", + job=job, + definition=self.definition, + created_at_utc=self.created_at_utc(), + ) + return PortableWorkerResultDraft( + root=package.root, + result_id=package.result_id, + result_sha256=package.manifest.manifest_sha256, + result_contract_sha256=M49_PORTABLE_RESULT_CONTRACT_SHA256, + ) + finally: + shutil.rmtree(workspace, ignore_errors=True) + + +def compose_m49_portable_executor_adapter( + *, + candidate: PortableWorkerRuntimeCandidate, + definition: PortableRunDefinition, + admission: PortableWorkerRuntimeAdmission, + source_transport: PortableWorkerSourceMaterializer, + result_transport: PortableWorkerResultPublisher, + installation: M49PortableRunnerInstallation, + source_output_parent: Path, + created_at_utc: Callable[[], str], + invoker: M49PortableRunnerInvoker | None = None, +) -> PortableWorkerExecutorAdapter: + """Compose shared Worker ports without changing their API or registry state.""" + + _verify_candidate_assets(candidate, installation) + return PortableWorkerExecutorAdapter( + candidate=candidate, + definition=definition, + admission=admission, + source_materializer=M49PortableSourceMaterializerAdapter( + upstream=source_transport, + profile_path=installation.profile_path, + output_parent=source_output_parent, + ), + runner=M49PortableProfileRunnerAdapter( + definition=definition, + installation=installation, + created_at_utc=created_at_utc, + invoker=invoker, + ), + publisher=result_transport, + ) + + +def _invoke_exact_runner( + *, + binary: Path, + sequence: Path, + schedule: Path, + output: Path, + timing: Path, + workspace: Path, + timeout_seconds: int, +) -> None: + stdout = workspace / "runner.stdout.log" + stderr = workspace / "runner.stderr.log" + try: + with stdout.open("xb") as stdout_stream, stderr.open("xb") as stderr_stream: + completed = subprocess.run( + [str(binary), str(sequence), str(schedule), str(output), str(timing)], + cwd=workspace, + env={"LANG": "C", "LC_ALL": "C", "TZ": "UTC"}, + stdin=subprocess.DEVNULL, + stdout=stdout_stream, + stderr=stderr_stream, + check=False, + timeout=timeout_seconds, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise M49PortableExecutorError("portable M4.9 runner invocation failed") from exc + if completed.returncode != 0: + raise M49PortableExecutorError("portable M4.9 runner rejected its exact source stage") + + +def _verify_definition(definition: PortableRunDefinition) -> None: + components = {component.component_id: component for component in definition.components} + profile = components.get("m49-tgs-portable-profile-v2") + if ( + definition.setup_id != "m49-tgs-portable-v2" + or definition.definition_id != "m49-tgs-portable" + or definition.result_contract.contract_sha256 != M49_PORTABLE_RESULT_CONTRACT_SHA256 + or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY + or profile is None + or profile.sha256 != M49_PORTABLE_PROFILE_SHA256 + ): + raise M49PortableExecutorError("portable M4.9 RunDefinition identity changed") + + +def _verify_plan( + plan: PortableWorkerRuntimePlan, + *, + job: SealedObservatoryRecordedJob, + definition: PortableRunDefinition, +) -> None: + if ( + plan.job_id != job.job_id + or plan.setup_id != job.setup_id + or plan.definition_sha256 != job.definition_sha256 + or plan.source_bundle_sha256 != job.source_bundle_sha256 + or plan.source_capability_manifest_sha256 != job.source_capability_manifest_sha256 + or plan.result_contract_sha256 != M49_PORTABLE_RESULT_CONTRACT_SHA256 + or plan.phases != M49_PORTABLE_RUNTIME_PHASES + or job.definition_sha256 != definition.definition_sha256 + ): + raise PortableWorkerRuntimeJobRejectedError( + "portable M4.9 runtime plan differs from its sealed job" + ) + + +def _verify_candidate_assets( + candidate: PortableWorkerRuntimeCandidate, + installation: M49PortableRunnerInstallation, +) -> None: + assets = {asset.asset_id: asset for asset in candidate.reusable_assets} + binary = assets.get(M49_PORTABLE_COMPILED_RUNNER_ASSET_ID) + build_seal = assets.get(M49_PORTABLE_COMPILED_RUNNER_BUILD_SEAL_ASSET_ID) + profile = assets.get(M49_PORTABLE_PROFILE_ASSET_ID) + image = assets.get(M49_PORTABLE_TRAVEL_IMAGE_ASSET_ID) + if ( + binary is None + or binary.kind != "local-file" + or binary.sha256 != installation.runner_binary_sha256 + or binary.byte_length != installation.runner_binary_path.stat().st_size + or build_seal is None + or build_seal.kind != "local-file" + or build_seal.sha256 != installation.runner_build_seal_sha256 + or build_seal.byte_length != installation.runner_build_seal_path.stat().st_size + or profile is None + or profile.sha256 != M49_PORTABLE_PROFILE_SHA256 + or image is None + or image.kind != "container-image" + or image.sha256 != M49_PORTABLE_TRAVEL_BUILD_IMAGE_SHA256 + ): + raise M49PortableExecutorError( + "portable M4.9 runtime candidate lacks exact installed runner assets" + ) + + +def _verify_installation_unchanged(installation: M49PortableRunnerInstallation) -> None: + _exact_file( + installation.profile_path, + M49_PORTABLE_PROFILE_SHA256, + "portable M4.9 profile", + ) + _exact_file( + installation.runner_build_seal_path, + installation.runner_build_seal_sha256, + "portable M4.9 runner build seal", + ) + binary = _regular_file( + installation.runner_binary_path, + "portable M4.9 compiled runner", + ) + if ( + _sha256_file(binary) != installation.runner_binary_sha256 + or not os.access(binary, os.X_OK) + or not _is_elf(binary) + ): + raise M49PortableExecutorError("portable M4.9 installed runner changed") + + +def _read_build_seal(path: Path) -> dict[str, object]: + try: + payload = path.read_bytes() + decoded: object = json.loads(payload.decode("utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise M49PortableExecutorError("portable M4.9 build seal is unreadable") from exc + if not 0 < len(payload) <= _MAX_BUILD_SEAL_BYTES: + raise M49PortableExecutorError("portable M4.9 build seal size is invalid") + document = _object(decoded, "portable M4.9 build seal") + if ( + set(document) + != { + "schema_version", + "source_revision", + "source_state", + "build_image_sha256", + "profile_sha256", + "runner_source_sha256", + "runner_wrapper_sha256", + "compiler_contract", + "binary", + "authority", + } + or payload != canonical_json(document) + or document["schema_version"] != M49_COMPILED_RUNNER_BUILD_SCHEMA + or not isinstance(document["source_revision"], str) + or len(document["source_revision"]) != 40 + or any(character not in "0123456789abcdef" for character in document["source_revision"]) + or document["source_state"] != "committed-snapshot" + or document["build_image_sha256"] != M49_PORTABLE_TRAVEL_BUILD_IMAGE_SHA256 + or document["profile_sha256"] != M49_PORTABLE_PROFILE_SHA256 + or document["runner_source_sha256"] != M49_PORTABLE_RUNNER_SOURCE_SHA256 + or document["runner_wrapper_sha256"] != M49_PORTABLE_RUNNER_WRAPPER_SHA256 + or document["compiler_contract"] != M49_PORTABLE_COMPILER_CONTRACT + or document["authority"] != OBSERVATION_ONLY_AUTHORITY + ): + raise M49PortableExecutorError("portable M4.9 build seal identity changed") + return document + + +def _exact_file(path: Path, expected_sha256: str, label: str) -> Path: + candidate = _regular_file(path, label) + if _sha256_file(candidate) != expected_sha256: + raise M49PortableExecutorError(f"{label} digest changed") + return candidate + + +def _regular_file(path: Path, label: str) -> Path: + candidate = path.expanduser().absolute() + try: + metadata = candidate.lstat() + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise M49PortableExecutorError(f"{label} is unavailable") from exc + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or not os.path.samefile(candidate, resolved) + ): + raise M49PortableExecutorError(f"{label} is unsafe") + return resolved + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _is_elf(path: Path) -> bool: + try: + with path.open("rb") as stream: + return stream.read(4) == b"\x7fELF" + except OSError: + return False + + +def _object(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise M49PortableExecutorError(f"{label} must be an object") + return cast(dict[str, object], value) diff --git a/src/k1link/observatory/m49_portable_result.py b/src/k1link/observatory/m49_portable_result.py new file mode 100644 index 0000000..8a3efd7 --- /dev/null +++ b/src/k1link/observatory/m49_portable_result.py @@ -0,0 +1,1378 @@ +"""Deterministic M4.9 portable result assembly and exact v2 validation. + +This is the Worker-side evidence boundary for the algorithm-only M4.9 TRAVEL +TGS profile. It consumes a validated, source-derived stage and the exact +generic runner outputs. It never reads a server-supplied path or command and +never produces navigation or actuation authority. + +The assembler reconstructs the complete eligible input multiset, retains the +TGS complement as ``UNKNOWN_REJECTED``, emits missing LiDAR frames as entirely +``UNOBSERVED``, and seals the result with the shared portable package envelope. +The validator independently checks the exact schema, every accounting row and +the mmap-friendly costmap arrays before Observatory publication. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import math +import os +import shutil +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Final, cast + +import numpy as np +import numpy.typing as npt + +from k1link.observatory.m49_portable_source import ( + M49_PORTABLE_PROFILE_SCHEMA, + M49_PORTABLE_STAGE_INDEX, + M49_PORTABLE_STAGE_MANIFEST, + read_m49_source_index, + validate_m49_portable_source_stage, +) +from k1link.observatory.portable_result_contract import ( + OBSERVATION_ONLY_AUTHORITY, + RESULT_DOCUMENT_ROLE, + PortableResultArtifact, + PortableResultPackageIntegrityError, + PortableResultPackageManifest, + PortableResultValidationContext, + canonical_json, + job_identity_document, + result_identity_document, + run_definition_document, + source_identity_document, +) +from k1link.observatory.portable_run_definitions import PortableRunDefinition +from k1link.observatory.recorded_jobs import ObservatoryRecordedJob +from k1link.observatory.worker_agent import SealedObservatoryRecordedJob + +type _M49PortableAssemblyJob = ObservatoryRecordedJob | SealedObservatoryRecordedJob + +M49_PORTABLE_RESULT_SCHEMA: Final = "missioncore.recorded-tgs-costmap-review/v2" +M49_PORTABLE_FRAME_SCHEMA: Final = "missioncore.recorded-tgs-costmap-review-frame/v2" +M49_PORTABLE_PROFILE_SHA256: Final = ( + "6128d6af7e6137f9a9473db045e3b155e2105319159f17c32f344b4aedf823a9" +) +M49_PORTABLE_RESULT_CONTRACT_SHA256: Final = ( + "9dd80c8e2504559d2156fca933de6eb27901e35305e6853aeb84707e1cb13892" +) +M49_PORTABLE_RESULT_PREFIX: Final = "m49-tgs-portable-review-" + +M49_ROLE_SOURCE_STAGE_MANIFEST: Final = "source-stage-manifest" +M49_ROLE_SOURCE_STAGE_INDEX: Final = "source-stage-index" +M49_ROLE_TGS_TIMING: Final = "tgs-timing" +M49_ROLE_FRAME_INDEX: Final = "frame-index" +M49_ROLE_COSTMAP_CELL_INDICES: Final = "costmap-cell-indices" +M49_ROLE_COSTMAP_CELL_CENTERS: Final = "costmap-cell-centers" +M49_ROLE_COSTMAP_STATES: Final = "costmap-states" +M49_ROLE_COSTMAP_Z_BOUNDS: Final = "costmap-z-bounds" + +M49_PORTABLE_RESULT_ROLES: Final = frozenset( + { + RESULT_DOCUMENT_ROLE, + M49_ROLE_SOURCE_STAGE_MANIFEST, + M49_ROLE_SOURCE_STAGE_INDEX, + M49_ROLE_TGS_TIMING, + M49_ROLE_FRAME_INDEX, + M49_ROLE_COSTMAP_CELL_INDICES, + M49_ROLE_COSTMAP_CELL_CENTERS, + M49_ROLE_COSTMAP_STATES, + M49_ROLE_COSTMAP_Z_BOUNDS, + } +) + +M49_PORTABLE_TIMING_HEADER: Final = ( + "timeline_frame_index\tsource_frame_index\tsession_seconds\tsample_available" + "\tavailable_slot\tinput_points\tground_points\tnonground_points\ttgs_ms" + "\tstage_wall_ms" +) +M49_PORTABLE_STATE_CODES: Final = { + "UNOBSERVED": 0, + "GROUND_SUPPORT": 1, + "NONGROUND_OCCUPIED": 2, + "UNKNOWN_REJECTED": 3, +} +M49_PORTABLE_STATE_PRIORITY: Final = [ + "NONGROUND_OCCUPIED", + "UNKNOWN_REJECTED", + "GROUND_SUPPORT", + "UNOBSERVED", +] +_MAX_TIMING_LINE_BYTES: Final = 4096 +_MAX_FRAME_LINE_BYTES: Final = 16 * 1024 +_HASH_CHUNK_BYTES: Final = 1024 * 1024 + + +class M49PortableResultError(PortableResultPackageIntegrityError): + """The portable M4.9 runner result is incomplete or inconsistent.""" + + +@dataclass(frozen=True, slots=True) +class M49PortableResultPackage: + root: Path + result_id: str + manifest: PortableResultPackageManifest + + +@dataclass(frozen=True, slots=True) +class _Profile: + sha256: str + min_range_m: float + max_range_m: float + cell_size_m: float + radius_m: float + + +@dataclass(frozen=True, slots=True) +class _TimingRow: + timeline_frame_index: int + source_frame_index: int + session_seconds: float + sample_available: bool + available_slot: int | None + input_points: int + ground_points: int + nonground_points: int + tgs_ms: float + stage_wall_ms: float + + +def assemble_m49_portable_result( + *, + source_stage_root: Path, + runner_output_root: Path, + runner_timing_path: Path, + profile_path: Path, + output_parent: Path, + job: _M49PortableAssemblyJob, + definition: PortableRunDefinition, + created_at_utc: str, +) -> M49PortableResultPackage: + """Assemble one exact content-addressed portable result package. + + The input ``job`` is either the running durable queue record or its exact + path-free Worker projection. The returned manifest digest and result ID + are the two values the Worker later supplies to the queue's success + transition. + """ + + _verify_running_job_definition(job, definition) + source_stage = validate_m49_portable_source_stage(source_stage_root) + _verify_source_stage_job_binding(source_stage.root, job) + profile = _read_profile(profile_path, definition) + source_index_path = source_stage.root / M49_PORTABLE_STAGE_INDEX + source_rows = read_m49_source_index( + source_index_path, + expected_frame_count=source_stage.timeline_frame_count, + ) + output_root = _safe_directory(runner_output_root, "portable TGS output root") + timing_path = _safe_file(runner_timing_path, "portable TGS timing") + timing_rows = _read_timing(timing_path, source_rows) + _verify_runner_output_set(output_root, source_rows) + + parent = output_parent.expanduser().absolute() + parent.mkdir(mode=0o700, parents=True, exist_ok=True) + if parent.is_symlink() or not parent.is_dir(): + raise M49PortableResultError("portable result parent is unsafe") + staging = Path(tempfile.mkdtemp(prefix=".m49-result-", dir=parent)) + published = False + try: + artifacts_root = staging / "artifacts" + artifacts_root.mkdir() + grid = _costmap_grid(profile.radius_m, profile.cell_size_m) + indices_path = artifacts_root / "costmap-cell-indices-xy.npy" + centers_path = artifacts_root / "costmap-cell-centers-xy-m.npy" + states_path = artifacts_root / "costmap-states.npy" + z_bounds_path = artifacts_root / "costmap-z-bounds-m.npy" + frames_path = artifacts_root / "frames.ndjson" + copied_timing_path = artifacts_root / "tgs-timing.tsv" + copied_stage_manifest_path = artifacts_root / "source-stage-manifest.json" + copied_stage_index_path = artifacts_root / "source-stage-index.ndjson" + + np.save(indices_path, grid[:, :2].astype(" None: + """Exact publisher validator for ``recorded-tgs-costmap-review/v2``.""" + + manifest = context.manifest + job = context.job + definition = context.definition + result = _object(context.result_document, "portable M4.9 result document") + _verify_terminal_job_definition(job, definition) + if set(context.artifact_paths) != M49_PORTABLE_RESULT_ROLES: + raise M49PortableResultError("portable M4.9 result artifact roles changed") + if ( + manifest.job != job_identity_document(job) + or manifest.source != source_identity_document(job) + or manifest.run_definition != run_definition_document(definition) + or manifest.result != result_identity_document(definition, cast(str, job.result_id)) + or manifest.authority != OBSERVATION_ONLY_AUTHORITY + ): + raise M49PortableResultError("portable M4.9 package envelope changed") + _exact_keys( + result, + { + "schema_version", + "result_id", + "result_kind", + "created_at_utc", + "profile", + "source_stage", + "timeline", + "costmap", + "point_accounting", + "performance", + "run_output_sha256", + "invariants", + "authority", + }, + "portable M4.9 result document", + ) + if ( + result["schema_version"] != M49_PORTABLE_RESULT_SCHEMA + or result["result_id"] != job.result_id + or result["result_kind"] != definition.result_contract.result_kind + or result["created_at_utc"] != manifest.created_at_utc + or result["authority"] != OBSERVATION_ONLY_AUTHORITY + ): + raise M49PortableResultError("portable M4.9 result identity changed") + profile = _object(result["profile"], "portable M4.9 result profile") + _exact_keys(profile, {"profile_id", "profile_sha256"}, "result profile") + if profile != { + "profile_id": "m49-tgs-portable-v2", + "profile_sha256": M49_PORTABLE_PROFILE_SHA256, + }: + raise M49PortableResultError("portable M4.9 result profile changed") + + stage_manifest_path = context.artifact_paths[M49_ROLE_SOURCE_STAGE_MANIFEST] + stage_manifest_payload = _canonical_document_bytes( + stage_manifest_path, "portable M4.9 source-stage manifest" + ) + stage_manifest = _object( + json.loads(stage_manifest_payload), "portable M4.9 source-stage manifest" + ) + source_stage = _object(result["source_stage"], "portable result source stage") + _exact_keys( + source_stage, + {"identity_sha256", "manifest_sha256"}, + "portable result source stage", + ) + if ( + source_stage["manifest_sha256"] != hashlib.sha256(stage_manifest_payload).hexdigest() + or source_stage["identity_sha256"] != stage_manifest.get("identity_sha256") + or stage_manifest.get("authority") != OBSERVATION_ONLY_AUTHORITY + ): + raise M49PortableResultError("portable result source-stage identity changed") + stage_identity = _object(stage_manifest.get("identity"), "portable M4.9 source-stage identity") + stage_source = _object(stage_identity.get("source"), "portable M4.9 source-stage source") + _exact_keys( + stage_source, + { + "source_session_id", + "source_catalog_sha256", + "source_bundle_sha256", + "source_capability_manifest_sha256", + "source_adapter_sha256", + "raw_capture_sha256", + "metadata_sha256", + }, + "portable M4.9 source-stage source", + ) + if ( + stage_source["source_session_id"] != job.source_session_id + or stage_source["source_catalog_sha256"] != job.source_catalog_sha256 + or stage_source["source_bundle_sha256"] != job.source_bundle_sha256 + or stage_source["source_capability_manifest_sha256"] + != job.source_capability_manifest_sha256 + or stage_source["source_adapter_sha256"] != job.source_adapter_sha256 + or stage_identity.get("authority") != OBSERVATION_ONLY_AUTHORITY + ): + raise M49PortableResultError("portable result source binding changed") + stage_profile = _object(stage_identity.get("profile"), "portable M4.9 source-stage profile") + if stage_profile != { + "profile_id": "m49-tgs-portable-v2", + "profile_sha256": M49_PORTABLE_PROFILE_SHA256, + }: + raise M49PortableResultError("portable result source-stage profile changed") + stage_artifacts = _object( + stage_manifest.get("artifacts"), "portable M4.9 source-stage artifacts" + ) + stage_index_descriptor = _object( + stage_artifacts.get("sequence-index"), "portable source-stage index descriptor" + ) + packaged_source_index = context.artifact_paths[M49_ROLE_SOURCE_STAGE_INDEX] + if ( + stage_index_descriptor.get("sha256") != _sha256_file(packaged_source_index) + or stage_index_descriptor.get("byte_length") != packaged_source_index.stat().st_size + ): + raise M49PortableResultError("portable result source-stage index changed") + + timeline = _object(result["timeline"], "portable result timeline") + _exact_keys( + timeline, + { + "frame_count", + "available_lidar_frame_count", + "missing_lidar_frame_count", + "duration_seconds", + "effective_fps", + }, + "portable result timeline", + ) + frame_count = _positive_int(timeline["frame_count"], "timeline frame count") + available_count = _positive_int( + timeline["available_lidar_frame_count"], "available LiDAR frame count" + ) + missing_count = _non_negative_int( + timeline["missing_lidar_frame_count"], "missing LiDAR frame count" + ) + if available_count + missing_count != frame_count: + raise M49PortableResultError("portable result timeline accounting changed") + source_rows = read_m49_source_index( + context.artifact_paths[M49_ROLE_SOURCE_STAGE_INDEX], + expected_frame_count=frame_count, + ) + if sum(row["sample_available"] is True for row in source_rows) != available_count: + raise M49PortableResultError("portable result source availability changed") + duration = _finite_number(source_rows[-1]["session_seconds"], "timeline end") - _finite_number( + source_rows[0]["session_seconds"], "timeline start" + ) + if not _float_equal(timeline["duration_seconds"], duration) or not _float_equal( + timeline["effective_fps"], (frame_count - 1) / duration + ): + raise M49PortableResultError("portable result timeline rate changed") + timing_rows = _read_timing(context.artifact_paths[M49_ROLE_TGS_TIMING], source_rows) + frame_rows = _read_frame_rows(context.artifact_paths[M49_ROLE_FRAME_INDEX], frame_count) + + costmap = _object(result["costmap"], "portable result costmap") + _exact_keys( + costmap, + { + "coordinate_frame", + "cell_size_m", + "radius_m", + "cell_count", + "state_codes", + "state_priority", + }, + "portable result costmap", + ) + if ( + costmap["coordinate_frame"] != "map-gravity-local" + or not _float_equal(costmap["cell_size_m"], 0.45) + or not _float_equal(costmap["radius_m"], 12.0) + or costmap["state_codes"] != M49_PORTABLE_STATE_CODES + or costmap["state_priority"] != M49_PORTABLE_STATE_PRIORITY + ): + raise M49PortableResultError("portable result costmap contract changed") + cell_count = _positive_int(costmap["cell_count"], "costmap cell count") + indices = cast( + npt.NDArray[np.int32], + _load_npy( + context.artifact_paths[M49_ROLE_COSTMAP_CELL_INDICES], + dtype=np.dtype(" z_bounds[..., 1][observed]) + ): + raise M49PortableResultError("observed portable costmap Z bounds are invalid") + + totals = _verify_frame_rows( + frame_rows, + source_rows=source_rows, + timing_rows=timing_rows, + states=states, + z_bounds=z_bounds, + ) + point_accounting = _object(result["point_accounting"], "portable result point accounting") + _exact_keys( + point_accounting, + {"eligible", "ground", "nonground", "rejected", "unaccounted"}, + "portable result point accounting", + ) + if point_accounting != { + **totals, + "unaccounted": 0, + }: + raise M49PortableResultError("portable result point accounting changed") + performance = _object(result["performance"], "portable result performance") + _exact_keys( + performance, + {"candidate_tgs_ms", "stage_wall_ms"}, + "portable result performance", + ) + if performance != { + "candidate_tgs_ms": _statistics( + [row.tgs_ms for row in timing_rows if row.sample_available] + ), + "stage_wall_ms": _statistics([row.stage_wall_ms for row in timing_rows]), + }: + raise M49PortableResultError("portable result timing statistics changed") + invariants = _object(result["invariants"], "portable result invariants") + if invariants != { + "all_timeline_frames_accounted": True, + "all_eligible_points_accounted": True, + "future_frames_used": False, + "missing_lidar_means_unobserved": True, + "missing_support_means_free": False, + "gpu_used": False, + "aos_used": False, + }: + raise M49PortableResultError("portable result invariants changed") + evidence_artifacts = tuple( + artifact for artifact in manifest.artifacts if artifact.role != RESULT_DOCUMENT_ROLE + ) + expected_output_identity = _run_output_identity( + source_stage_identity_sha256=cast(str, source_stage["identity_sha256"]), + profile_sha256=M49_PORTABLE_PROFILE_SHA256, + evidence_artifacts=evidence_artifacts, + ) + if ( + result["run_output_sha256"] != expected_output_identity + or result["result_id"] != f"{M49_PORTABLE_RESULT_PREFIX}{expected_output_identity}" + ): + raise M49PortableResultError("portable result output identity changed") + + +def _verify_running_job_definition( + job: _M49PortableAssemblyJob, definition: PortableRunDefinition +) -> None: + if isinstance(job, ObservatoryRecordedJob): + if job.state != "running" or job.result_id is not None or job.result_sha256 is not None: + raise M49PortableResultError("portable M4.9 assembly requires a running job") + elif ( + not isinstance(job.claim_generation, int) + or isinstance(job.claim_generation, bool) + or job.claim_generation < 1 + or job.claim_claimed_at_utc is None + or job.claim_expires_at_utc is None + or len(job.submission_receipt_sha256) != 64 + or any(character not in "0123456789abcdef" for character in job.submission_receipt_sha256) + ): + raise M49PortableResultError( + "portable M4.9 assembly requires a claim-bound sealed Worker job" + ) + _verify_job_definition_identity(job, definition) + + +def _verify_source_stage_job_binding(stage_root: Path, job: _M49PortableAssemblyJob) -> None: + payload = _canonical_document_bytes( + stage_root / M49_PORTABLE_STAGE_MANIFEST, + "portable M4.9 source-stage manifest", + ) + manifest = _object(json.loads(payload), "portable M4.9 source-stage manifest") + identity = _object(manifest.get("identity"), "portable M4.9 source-stage identity") + source = _object(identity.get("source"), "portable M4.9 source-stage source") + if ( + source.get("source_session_id") != job.source_session_id + or source.get("source_catalog_sha256") != job.source_catalog_sha256 + or source.get("source_bundle_sha256") != job.source_bundle_sha256 + or source.get("source_capability_manifest_sha256") != job.source_capability_manifest_sha256 + or source.get("source_adapter_sha256") != job.source_adapter_sha256 + or identity.get("profile") + != { + "profile_id": "m49-tgs-portable-v2", + "profile_sha256": M49_PORTABLE_PROFILE_SHA256, + } + or identity.get("authority") != OBSERVATION_ONLY_AUTHORITY + ): + raise M49PortableResultError("portable M4.9 source stage and recorded job disagree") + + +def _verify_terminal_job_definition( + job: ObservatoryRecordedJob, definition: PortableRunDefinition +) -> None: + if job.state != "succeeded" or job.result_id is None or job.result_sha256 is None: + raise M49PortableResultError("portable M4.9 validation requires a succeeded job") + _verify_job_definition_identity(job, definition) + + +def _verify_job_definition_identity( + job: _M49PortableAssemblyJob, definition: PortableRunDefinition +) -> None: + if ( + definition.result_contract.result_schema != M49_PORTABLE_RESULT_SCHEMA + or definition.result_contract.contract_sha256 != M49_PORTABLE_RESULT_CONTRACT_SHA256 + or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY + ): + raise M49PortableResultError("portable M4.9 result contract changed") + try: + expected = definition.to_recorded_run_definition() + except Exception as exc: + raise M49PortableResultError("portable M4.9 executor definition is not installed") from exc + if isinstance(job, ObservatoryRecordedJob): + executor_release_sha256 = job.executor_release_sha256 + executor_image_sha256 = job.executor_image_sha256 + model_manifest_sha256 = job.model_manifest_sha256 + resource_profile_sha256 = job.resource_profile_sha256 + else: + executor_release_sha256 = job.executor_identity.release_sha256 + executor_image_sha256 = job.executor_identity.image_sha256 + model_manifest_sha256 = job.executor_identity.model_manifest_sha256 + resource_profile_sha256 = job.executor_identity.resource_profile_sha256 + actual_identity = ( + job.setup_id, + job.definition_id, + job.definition_version, + job.definition_sha256, + job.source_adapter_id, + job.source_adapter_version, + job.source_adapter_sha256, + job.executor_release_id, + executor_release_sha256, + executor_image_sha256, + job.model_release_ids, + model_manifest_sha256, + job.resource_profile_id, + resource_profile_sha256, + job.checkpoint_policy, + job.allowed_checkpoints, + ) + expected_identity = ( + expected.setup_id, + expected.definition_id, + expected.definition_version, + expected.definition_sha256, + expected.source_adapter_id, + expected.source_adapter_version, + expected.source_adapter_sha256, + expected.executor_release_id, + expected.executor_release_sha256, + expected.executor_image_sha256, + expected.model_release_ids, + expected.model_manifest_sha256, + expected.resource_profile_id, + expected.resource_profile_sha256, + expected.checkpoint_policy, + expected.allowed_checkpoints, + ) + if actual_identity != expected_identity: + raise M49PortableResultError("portable M4.9 job definition identity changed") + components = {component.component_id: component for component in definition.components} + profile = components.get("m49-tgs-portable-profile-v2") + if profile is None or profile.sha256 != M49_PORTABLE_PROFILE_SHA256: + raise M49PortableResultError("portable M4.9 profile component changed") + + +def _read_profile(path: Path, definition: PortableRunDefinition) -> _Profile: + candidate = _safe_file(path, "portable M4.9 profile") + payload = candidate.read_bytes() + if hashlib.sha256(payload).hexdigest() != M49_PORTABLE_PROFILE_SHA256: + raise M49PortableResultError("portable M4.9 profile digest changed") + try: + document = _object(json.loads(payload), "portable M4.9 profile") + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise M49PortableResultError("portable M4.9 profile is invalid JSON") from exc + if document.get("schema_version") != M49_PORTABLE_PROFILE_SCHEMA: + raise M49PortableResultError("portable M4.9 profile schema changed") + if definition.result_contract.contract_sha256 != M49_PORTABLE_RESULT_CONTRACT_SHA256: + raise M49PortableResultError("portable M4.9 result contract changed") + tgs = _object(document.get("tgs"), "portable M4.9 TGS profile") + costmap = _object(document.get("costmap"), "portable M4.9 costmap profile") + invariants = _object(document.get("invariants"), "portable M4.9 invariants") + if ( + document.get("profile_id") != "m49-tgs-portable-v2" + or costmap.get("coordinate_frame") != "map-gravity-local" + or costmap.get("state_priority") != M49_PORTABLE_STATE_PRIORITY + or invariants.get("aos_allowed") is not False + or invariants.get("gpu_allowed") is not False + or invariants.get("missing_support_means_free") is not False + or invariants.get("missing_lidar_means_unobserved") is not True + or invariants.get("navigation_or_actuation_allowed") is not False + ): + raise M49PortableResultError("portable M4.9 profile invariants changed") + result = _Profile( + sha256=M49_PORTABLE_PROFILE_SHA256, + min_range_m=_positive_float(tgs.get("min_range_m"), "TGS minimum range"), + max_range_m=_positive_float(tgs.get("max_range_m"), "TGS maximum range"), + cell_size_m=_positive_float(costmap.get("cell_size_m"), "costmap cell size"), + radius_m=_positive_float(costmap.get("radius_m"), "costmap radius"), + ) + if ( + not math.isclose(result.min_range_m, 1.0) + or not math.isclose(result.max_range_m, 80.0) + or not math.isclose(result.cell_size_m, 0.45) + or not math.isclose(result.radius_m, 12.0) + ): + raise M49PortableResultError("portable M4.9 numeric profile changed") + return result + + +def _read_timing(path: Path, source_rows: Sequence[Mapping[str, object]]) -> tuple[_TimingRow, ...]: + rows: list[_TimingRow] = [] + try: + with path.open("r", encoding="utf-8", newline="") as stream: + header = stream.readline() + if header != M49_PORTABLE_TIMING_HEADER + "\n": + raise M49PortableResultError("portable TGS timing header changed") + reader = csv.reader(stream, delimiter="\t", strict=True) + for index, values in enumerate(reader): + if len("\t".join(values).encode("utf-8")) > _MAX_TIMING_LINE_BYTES: + raise M49PortableResultError("portable TGS timing row is too large") + if len(values) != 10 or index >= len(source_rows): + raise M49PortableResultError("portable TGS timing row changed") + source = source_rows[index] + available_raw = _parse_int(values[3], "timing availability") + if available_raw not in (0, 1): + raise M49PortableResultError("portable TGS timing availability changed") + slot_raw = _parse_int(values[4], "timing available slot") + row = _TimingRow( + timeline_frame_index=_parse_non_negative_int(values[0], "timing index"), + source_frame_index=_parse_non_negative_int(values[1], "timing source index"), + session_seconds=_parse_float(values[2], "timing session time"), + sample_available=bool(available_raw), + available_slot=None if slot_raw == -1 else slot_raw, + input_points=_parse_non_negative_int(values[5], "timing input points"), + ground_points=_parse_non_negative_int(values[6], "timing ground points"), + nonground_points=_parse_non_negative_int(values[7], "timing nonground points"), + tgs_ms=_parse_non_negative_float(values[8], "timing TGS milliseconds"), + stage_wall_ms=_parse_non_negative_float(values[9], "timing wall milliseconds"), + ) + expected_slot = source["available_slot"] + if ( + row.timeline_frame_index != index + or row.source_frame_index != source["source_frame_index"] + or not math.isclose( + row.session_seconds, + _finite_number(source["session_seconds"], "source session time"), + rel_tol=0.0, + abs_tol=5e-7, + ) + or row.sample_available != source["sample_available"] + or row.available_slot != expected_slot + or row.input_points != source["point_count"] + or ( + not row.sample_available + and any( + value != 0 + for value in ( + row.input_points, + row.ground_points, + row.nonground_points, + row.tgs_ms, + ) + ) + ) + ): + raise M49PortableResultError("portable TGS timing and source index disagree") + rows.append(row) + except M49PortableResultError: + raise + except (OSError, csv.Error) as exc: + raise M49PortableResultError("portable TGS timing is invalid") from exc + if len(rows) != len(source_rows): + raise M49PortableResultError("portable TGS timing is incomplete") + return tuple(rows) + + +def _verify_runner_output_set(root: Path, source_rows: Sequence[Mapping[str, object]]) -> None: + expected = { + f"{row['timeline_frame_index']}_{kind}.bin" + for row in source_rows + if row["sample_available"] is True + for kind in ("ground", "nonground") + } + actual: set[str] = set() + try: + for path in root.iterdir(): + if path.is_symlink() or not path.is_file(): + raise M49PortableResultError("portable TGS output root is unsafe") + actual.add(path.name) + except OSError as exc: + raise M49PortableResultError("portable TGS output root is unavailable") from exc + if actual != expected: + raise M49PortableResultError("portable TGS output file set changed") + + +def _classify_exact_input( + native: npt.NDArray[np.float32], + ground: npt.NDArray[np.float32], + nonground: npt.NDArray[np.float32], + *, + min_range_m: float, + max_range_m: float, +) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.uint8]]: + ranges = np.linalg.norm(native[:, :2].astype(np.float64), axis=1) + points = np.ascontiguousarray(native[(ranges > min_range_m) & (ranges < max_range_m), :3]) + output = np.ascontiguousarray(np.concatenate((ground[:, :3], nonground[:, :3]), axis=0)) + output_states = np.concatenate( + ( + np.ones(ground.shape[0], dtype=np.uint8), + np.full(nonground.shape[0], 2, dtype=np.uint8), + ) + ) + key_dtype = np.dtype((np.void, 12)) + input_keys = points.view(key_dtype).reshape(-1) + output_keys = output.view(key_dtype).reshape(-1) + input_order = np.argsort(input_keys, kind="stable") + output_order = np.argsort(output_keys, kind="stable") + sorted_input = input_keys[input_order] + sorted_output = output_keys[output_order] + positions = np.searchsorted(sorted_input, sorted_output, side="left") + if sorted_output.size: + group_starts = np.r_[0, np.flatnonzero(sorted_output[1:] != sorted_output[:-1]) + 1] + group_lengths = np.diff(np.r_[group_starts, sorted_output.size]) + occurrence = np.arange(sorted_output.size) - np.repeat(group_starts, group_lengths) + targets = positions + occurrence + if ( + np.any(targets >= sorted_input.size) + or np.any(sorted_input[targets] != sorted_output) + or np.unique(targets).size != targets.size + ): + raise M49PortableResultError( + "portable TGS output is not an exact input multiset subset" + ) + else: + targets = np.empty(0, dtype=np.int64) + sorted_states = np.full(points.shape[0], 3, dtype=np.uint8) + sorted_states[targets] = output_states[output_order] + states = np.empty_like(sorted_states) + states[input_order] = sorted_states + return points, states + + +def _costmap_grid(radius_m: float, cell_size_m: float) -> npt.NDArray[np.float64]: + minimum = math.floor(-radius_m / cell_size_m) + maximum = math.ceil(radius_m / cell_size_m) + cells = [ + (ix, iy, (ix + 0.5) * cell_size_m, (iy + 0.5) * cell_size_m) + for ix in range(minimum, maximum) + for iy in range(minimum, maximum) + if math.hypot((ix + 0.5) * cell_size_m, (iy + 0.5) * cell_size_m) <= radius_m + ] + if not cells: + raise M49PortableResultError("portable M4.9 costmap grid is empty") + return np.asarray(cells, dtype=np.float64) + + +def _rasterize( + points: npt.NDArray[np.float32], + states: npt.NDArray[np.uint8], + grid: npt.NDArray[np.float64], + *, + cell_size_m: float, +) -> tuple[npt.NDArray[np.uint8], npt.NDArray[np.float32]]: + minimum_ix = int(np.min(grid[:, 0])) + maximum_ix = int(np.max(grid[:, 0])) + minimum_iy = int(np.min(grid[:, 1])) + maximum_iy = int(np.max(grid[:, 1])) + lookup = np.full( + (maximum_ix - minimum_ix + 1, maximum_iy - minimum_iy + 1), + -1, + dtype=np.int32, + ) + lookup[ + grid[:, 0].astype(np.int32) - minimum_ix, + grid[:, 1].astype(np.int32) - minimum_iy, + ] = np.arange(grid.shape[0], dtype=np.int32) + cell_xy = np.floor(points[:, :2] / cell_size_m).astype(np.int32) + inside = ( + (cell_xy[:, 0] >= minimum_ix) + & (cell_xy[:, 0] <= maximum_ix) + & (cell_xy[:, 1] >= minimum_iy) + & (cell_xy[:, 1] <= maximum_iy) + ) + point_indices = np.flatnonzero(inside) + cell_indices = lookup[ + cell_xy[inside, 0] - minimum_ix, + cell_xy[inside, 1] - minimum_iy, + ] + valid = cell_indices >= 0 + point_indices = point_indices[valid] + cell_indices = cell_indices[valid] + cell_states = np.zeros(grid.shape[0], dtype=np.uint8) + selected_states = states[point_indices] + ground = np.zeros(grid.shape[0], dtype=np.uint8) + rejected = np.zeros(grid.shape[0], dtype=np.uint8) + nonground = np.zeros(grid.shape[0], dtype=np.uint8) + np.maximum.at(ground, cell_indices, (selected_states == 1).astype(np.uint8)) + np.maximum.at(rejected, cell_indices, (selected_states == 3).astype(np.uint8)) + np.maximum.at(nonground, cell_indices, (selected_states == 2).astype(np.uint8)) + cell_states[ground > 0] = 1 + cell_states[rejected > 0] = 3 + cell_states[nonground > 0] = 2 + minimum_z = np.full(grid.shape[0], np.inf, dtype=np.float32) + maximum_z = np.full(grid.shape[0], -np.inf, dtype=np.float32) + np.minimum.at(minimum_z, cell_indices, points[point_indices, 2]) + np.maximum.at(maximum_z, cell_indices, points[point_indices, 2]) + z_bounds = np.column_stack((minimum_z, maximum_z)).astype(np.float32, copy=False) + z_bounds[~np.isfinite(z_bounds)] = np.nan + return cell_states, z_bounds + + +def _missing_frame_row(source: Mapping[str, object]) -> dict[str, object]: + return { + "schema_version": M49_PORTABLE_FRAME_SCHEMA, + "timeline_frame_index": source["timeline_frame_index"], + "source_frame_index": source["source_frame_index"], + "session_seconds": source["session_seconds"], + "sample_available": False, + "available_slot": None, + "input_point_count": 0, + "eligible_point_count": 0, + "ground_point_count": 0, + "nonground_point_count": 0, + "rejected_point_count": 0, + "occupied_cell_count": 0, + } + + +def _read_frame_rows(path: Path, expected_count: int) -> tuple[dict[str, object], ...]: + rows: list[dict[str, object]] = [] + try: + with path.open("rb") as stream: + for line in stream: + if len(line) > _MAX_FRAME_LINE_BYTES or not line.endswith(b"\n"): + raise M49PortableResultError("portable frame-index line is invalid") + value = _object(json.loads(line), "portable frame-index row") + if canonical_json(value) != line[:-1]: + raise M49PortableResultError("portable frame-index is not canonical JSONL") + rows.append(value) + if len(rows) > expected_count: + raise M49PortableResultError("portable frame-index has extra rows") + except M49PortableResultError: + raise + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise M49PortableResultError("portable frame-index is invalid") from exc + if len(rows) != expected_count: + raise M49PortableResultError("portable frame-index is incomplete") + return tuple(rows) + + +def _verify_frame_rows( + rows: Sequence[Mapping[str, object]], + *, + source_rows: Sequence[Mapping[str, object]], + timing_rows: Sequence[_TimingRow], + states: npt.NDArray[np.uint8], + z_bounds: npt.NDArray[np.float32], +) -> dict[str, int]: + expected_keys = set(_missing_frame_row(source_rows[0])) + totals = {"eligible": 0, "ground": 0, "nonground": 0, "rejected": 0} + for index, (row, source, timing) in enumerate(zip(rows, source_rows, timing_rows, strict=True)): + _exact_keys(row, expected_keys, "portable frame-index row") + if ( + row["schema_version"] != M49_PORTABLE_FRAME_SCHEMA + or row["timeline_frame_index"] != index + or row["source_frame_index"] != source["source_frame_index"] + or not _float_equal(row["session_seconds"], source["session_seconds"]) + or row["sample_available"] != source["sample_available"] + or row["available_slot"] != source["available_slot"] + or row["input_point_count"] != timing.input_points + or row["ground_point_count"] != timing.ground_points + or row["nonground_point_count"] != timing.nonground_points + ): + raise M49PortableResultError("portable frame-index identity changed") + counts = { + key: _non_negative_int(row[key], f"frame {key}") + for key in ( + "eligible_point_count", + "ground_point_count", + "nonground_point_count", + "rejected_point_count", + "occupied_cell_count", + ) + } + if counts["eligible_point_count"] != ( + counts["ground_point_count"] + + counts["nonground_point_count"] + + counts["rejected_point_count"] + ) or counts["occupied_cell_count"] != int(np.count_nonzero(states[index] == 2)): + raise M49PortableResultError("portable frame point accounting changed") + if source["sample_available"] is not True and ( + any(counts.values()) or np.any(states[index]) or not np.isnan(z_bounds[index]).all() + ): + raise M49PortableResultError("missing LiDAR frame invents evidence") + totals["eligible"] += counts["eligible_point_count"] + totals["ground"] += counts["ground_point_count"] + totals["nonground"] += counts["nonground_point_count"] + totals["rejected"] += counts["rejected_point_count"] + return totals + + +def _run_output_identity( + *, + source_stage_identity_sha256: str, + profile_sha256: str, + evidence_artifacts: Sequence[PortableResultArtifact], +) -> str: + return hashlib.sha256( + canonical_json( + { + "schema_version": "missioncore.m49-tgs-portable-run-output/v1", + "source_stage_identity_sha256": source_stage_identity_sha256, + "profile_sha256": profile_sha256, + "artifacts": [ + artifact.as_dict() + for artifact in sorted(evidence_artifacts, key=lambda item: item.role) + ], + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + ) + ).hexdigest() + + +def _artifact(role: str, path: Path, media_type: str) -> PortableResultArtifact: + return PortableResultArtifact( + role=role, + relative_path=f"artifacts/{path.name}", + media_type=media_type, + byte_length=path.stat().st_size, + sha256=_sha256_file(path), + ) + + +def _verify_staged_package(root: Path, package: PortableResultPackageManifest) -> None: + if (root / "manifest.json").read_bytes() != package.canonical_bytes: + raise M49PortableResultError("portable result manifest changed during assembly") + for artifact in package.artifacts: + path = root.joinpath(*PurePosixPath(artifact.relative_path).parts) + if ( + path.is_symlink() + or not path.is_file() + or path.stat().st_size != artifact.byte_length + or _sha256_file(path) != artifact.sha256 + ): + raise M49PortableResultError("portable result artifact changed during assembly") + + +def _read_existing_package(root: Path) -> PortableResultPackageManifest: + candidate = _safe_directory(root, "existing portable result package") + payload = _safe_file(candidate / "manifest.json", "portable result manifest").read_bytes() + package = PortableResultPackageManifest.from_bytes(payload) + if candidate.name != package.manifest_sha256: + raise M49PortableResultError("existing portable result directory changed") + _verify_staged_package(candidate, package) + return package + + +def _load_xyzi(path: Path, *, label: str) -> npt.NDArray[np.float32]: + candidate = _safe_file(path, label) + if candidate.stat().st_size % 16: + raise M49PortableResultError(f"{label} byte shape changed") + values = np.fromfile(candidate, dtype=" npt.NDArray[np.generic]: + candidate = _safe_file(path, "portable M4.9 numpy artifact") + try: + value = np.load(candidate, mmap_mode="r", allow_pickle=False) + except (OSError, ValueError) as exc: + raise M49PortableResultError("portable M4.9 numpy artifact is invalid") from exc + if value.dtype != dtype or value.shape != shape: + raise M49PortableResultError("portable M4.9 numpy artifact shape changed") + return cast(npt.NDArray[np.generic], value) + + +def _statistics(values: Sequence[float]) -> dict[str, float]: + data = np.asarray(values, dtype=np.float64) + if data.size < 1 or not np.isfinite(data).all() or np.any(data < 0): + raise M49PortableResultError("portable M4.9 performance series is invalid") + return { + "p50": float(np.percentile(data, 50)), + "p95": float(np.percentile(data, 95)), + "p99": float(np.percentile(data, 99)), + "max": float(np.max(data)), + } + + +def _canonical_document_bytes(path: Path, label: str) -> bytes: + payload = _safe_file(path, label).read_bytes() + try: + value = _object(json.loads(payload), label) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise M49PortableResultError(f"{label} is invalid JSON") from exc + if canonical_json(value) != payload: + raise M49PortableResultError(f"{label} is not canonical JSON") + return payload + + +def _safe_directory(path: Path, label: str) -> Path: + candidate = path.expanduser().absolute() + try: + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise M49PortableResultError(f"{label} is unavailable") from exc + if candidate.is_symlink() or not resolved.is_dir() or not os.path.samefile(candidate, resolved): + raise M49PortableResultError(f"{label} is unsafe") + return resolved + + +def _safe_file(path: Path, label: str) -> Path: + candidate = path.expanduser().absolute() + try: + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise M49PortableResultError(f"{label} is unavailable") from exc + if ( + candidate.is_symlink() + or not resolved.is_file() + or not os.path.samefile(candidate, resolved) + ): + raise M49PortableResultError(f"{label} is unsafe") + return resolved + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(_HASH_CHUNK_BYTES), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _exact_keys(value: Mapping[str, object], expected: set[str], label: str) -> None: + if set(value) != expected: + raise M49PortableResultError(f"{label} fields changed") + + +def _object(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise M49PortableResultError(f"{label} must be an object") + return cast(dict[str, object], value) + + +def _non_negative_int(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise M49PortableResultError(f"{label} is invalid") + return value + + +def _positive_int(value: object, label: str) -> int: + result = _non_negative_int(value, label) + if result < 1: + raise M49PortableResultError(f"{label} is invalid") + return result + + +def _positive_float(value: object, label: str) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + or float(value) <= 0 + ): + raise M49PortableResultError(f"{label} is invalid") + return float(value) + + +def _finite_number(value: object, label: str) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + ): + raise M49PortableResultError(f"{label} is invalid") + return float(value) + + +def _parse_int(value: str, label: str) -> int: + try: + return int(value) + except ValueError as exc: + raise M49PortableResultError(f"{label} is invalid") from exc + + +def _parse_non_negative_int(value: str, label: str) -> int: + result = _parse_int(value, label) + if result < 0: + raise M49PortableResultError(f"{label} is invalid") + return result + + +def _parse_float(value: str, label: str) -> float: + try: + result = float(value) + except ValueError as exc: + raise M49PortableResultError(f"{label} is invalid") from exc + if not math.isfinite(result): + raise M49PortableResultError(f"{label} is invalid") + return result + + +def _parse_non_negative_float(value: str, label: str) -> float: + result = _parse_float(value, label) + if result < 0: + raise M49PortableResultError(f"{label} is invalid") + return result + + +def _float_equal(left: object, right: object) -> bool: + if ( + isinstance(left, bool) + or isinstance(right, bool) + or not isinstance(left, (int, float)) + or not isinstance(right, (int, float)) + ): + return False + return math.isclose(float(left), float(right), rel_tol=1e-12, abs_tol=1e-12) diff --git a/src/k1link/observatory/m49_portable_source.py b/src/k1link/observatory/m49_portable_source.py new file mode 100644 index 0000000..1e1ee1d --- /dev/null +++ b/src/k1link/observatory/m49_portable_source.py @@ -0,0 +1,1245 @@ +"""Dynamic, fail-closed K1 source materialization for portable M4.9 TGS. + +The historical M4.9 experiment proved the TRAVEL/TGS algorithm on one exact +RAVNOVES00 pack. This module preserves that algorithmic profile while +removing the session name, filesystem path, and 4489/3928 frame-count binding. + +It consumes only: + +* canonical admitted portable source/capability documents; +* a field-retaining ``LidarReplayPackV2`` built from digest-bound raw capture + and host-arrival metadata; and +* the exact portable M4.9 profile. + +The output is a content-addressed local Worker stage. It contains no command +authority and is not a navigation or safety acceptance. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import shutil +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Final, cast + +import numpy as np +import numpy.typing as npt + +from k1link.compute.lidar_replay import LidarReplayPackV2, build_lidar_replay_pack_v2 +from k1link.observatory.portable_result_contract import canonical_json +from k1link.observatory.source_admission import ( + PORTABLE_SOURCE_BUNDLE_SCHEMA, + PORTABLE_SOURCE_CAPABILITY_SCHEMA, + PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID, + PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE, +) + +if TYPE_CHECKING: + from k1link.observatory.portable_worker_runtime import PortableWorkerSourceStage + from k1link.observatory.worker_agent import SealedObservatoryRecordedJob + +M49_PORTABLE_SOURCE_STAGE_SCHEMA: Final = "missioncore.m49-tgs-portable-source-stage/v1" +M49_PORTABLE_SOURCE_INDEX_SCHEMA: Final = "missioncore.m49-tgs-portable-source-index-row/v1" +M49_PORTABLE_PROFILE_SCHEMA: Final = "missioncore.m49-tgs-portable-profile/v2" +M49_PORTABLE_STAGE_PREFIX: Final = "m49-tgs-source-stage-" +M49_PORTABLE_STAGE_MANIFEST: Final = "manifest.json" +M49_PORTABLE_STAGE_INDEX: Final = "sequence-index.ndjson" +M49_PORTABLE_STAGE_SCHEDULE: Final = "schedule.tsv" +M49_PORTABLE_TGS_SEQUENCE: Final = "tgs/sequence/velodyne" + +M49_PORTABLE_MAX_TIMELINE_FRAMES: Final = 250_000 +M49_PORTABLE_MAX_SOURCE_DURATION_SECONDS: Final = 8 * 60 * 60 +M49_PORTABLE_MAX_POINTS_PER_MATERIALIZED_FRAME: Final = 20_000_000 +M49_PORTABLE_MAX_INDEX_LINE_BYTES: Final = 64 * 1024 +PORTABLE_SOURCE_MATERIALIZATION_SCHEMA: Final = ( + "missioncore.observatory-portable-source-materialization/v1" +) + +_SHA256: Final = re.compile(r"^[a-f0-9]{64}$") +_SESSION_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +_PACK_ID: Final = re.compile(r"^lidar-replay-pack-[a-f0-9]{64}$") +_AUTHORITY: Final = { + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, + "production_accepted": False, +} + + +class M49PortableSourceError(RuntimeError): + """The delivered K1 evidence cannot produce an exact portable TGS stage.""" + + +@dataclass(frozen=True, slots=True) +class M49PortableSourceIdentity: + source_session_id: str + source_catalog_sha256: str + source_bundle_sha256: str + source_capability_manifest_sha256: str + source_adapter_sha256: str + raw_capture_sha256: str + metadata_sha256: str + + def __post_init__(self) -> None: + if _SESSION_ID.fullmatch(self.source_session_id) is None: + raise ValueError("M4.9 source session id is invalid") + for value, label in ( + (self.source_catalog_sha256, "source catalog sha256"), + (self.source_bundle_sha256, "source bundle sha256"), + ( + self.source_capability_manifest_sha256, + "source capability manifest sha256", + ), + (self.source_adapter_sha256, "source adapter sha256"), + (self.raw_capture_sha256, "raw capture sha256"), + (self.metadata_sha256, "host metadata sha256"), + ): + _digest(value, label) + + def as_dict(self) -> dict[str, str]: + return { + "source_session_id": self.source_session_id, + "source_catalog_sha256": self.source_catalog_sha256, + "source_bundle_sha256": self.source_bundle_sha256, + "source_capability_manifest_sha256": (self.source_capability_manifest_sha256), + "source_adapter_sha256": self.source_adapter_sha256, + "raw_capture_sha256": self.raw_capture_sha256, + "metadata_sha256": self.metadata_sha256, + } + + +@dataclass(frozen=True, slots=True) +class M49PortableSourceStage: + root: Path + identity_sha256: str + manifest_sha256: str + timeline_frame_count: int + available_lidar_frame_count: int + + def __post_init__(self) -> None: + _digest(self.identity_sha256, "M4.9 source stage identity sha256") + _digest(self.manifest_sha256, "M4.9 source stage manifest sha256") + if ( + self.root.is_symlink() + or not self.root.is_dir() + or self.root.name != f"{M49_PORTABLE_STAGE_PREFIX}{self.identity_sha256}" + or not 1 <= self.timeline_frame_count <= M49_PORTABLE_MAX_TIMELINE_FRAMES + or not 0 < self.available_lidar_frame_count <= self.timeline_frame_count + ): + raise ValueError("M4.9 source stage identity is invalid") + + +@dataclass(frozen=True, slots=True) +class _Profile: + profile_id: str + profile_sha256: str + history_seconds: float + local_radius_m: float + maximum_lidar_age_seconds: float + + +@dataclass(frozen=True, slots=True) +class _CameraAnchor: + timeline_frame_index: int + source_frame_index: int + session_seconds: float + + +def materialize_m49_portable_source_from_worker_stage( + *, + worker_stage: PortableWorkerSourceStage, + job: SealedObservatoryRecordedJob, + profile_path: Path, + output_parent: Path, +) -> M49PortableSourceStage: + """Consume only the fixed, manifest-bound Worker source layout. + + The transport chooses no executor paths: it materializes a server-owned + member plan into fixed local names. This adapter independently verifies + that plan, including the separately admitted ``raw-transport-index``, then + builds the exact field-retaining LiDAR replay pack and the portable TGS + source stage. An adjacent metadata file absent from the manifest is an + error even if it exists on disk. + """ + + if ( + worker_stage.source_bundle_sha256 != job.source_bundle_sha256 + or worker_stage.source_capability_manifest_sha256 != job.source_capability_manifest_sha256 + or worker_stage.source_adapter_sha256 != job.source_adapter_sha256 + ): + raise M49PortableSourceError("Worker source stage belongs to another job") + root = _safe_directory(worker_stage.root, "Worker source stage") + manifest_payload, manifest = _read_canonical_document( + root / "materialization-manifest.json", + expected_sha256=None, + label="Worker source materialization manifest", + ) + del manifest_payload + members = _verify_worker_materialization_manifest( + root, + manifest, + job=job, + ) + raw = next( + ( + member + for member in members + if member["kind"] == "spatial-replay" + and member["artifact_id"] == "raw-transport-primary" + ), + None, + ) + metadata = next( + ( + member + for member in members + if member["kind"] == "spatial-replay-metadata" + and member["artifact_id"] == PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID + ), + None, + ) + if raw is None or metadata is None: + raise M49PortableSourceError( + "portable M4.9 requires admitted raw and host-time replay members" + ) + expected = M49PortableSourceIdentity( + source_session_id=job.source_session_id, + source_catalog_sha256=job.source_catalog_sha256, + source_bundle_sha256=job.source_bundle_sha256, + source_capability_manifest_sha256=job.source_capability_manifest_sha256, + source_adapter_sha256=job.source_adapter_sha256, + raw_capture_sha256=_string(raw["sha256"], "raw replay member sha256"), + metadata_sha256=_string(metadata["sha256"], "metadata replay member sha256"), + ) + parent = output_parent.expanduser().absolute() + parent.mkdir(mode=0o700, parents=True, exist_ok=True) + if parent.is_symlink() or not parent.is_dir(): + raise M49PortableSourceError("portable M4.9 output parent is unsafe") + try: + lidar_pack_root = build_lidar_replay_pack_v2( + root / "mqtt.raw.k1mqtt", + parent / "lidar-replay-packs", + session_id=job.source_session_id, + ) + except Exception as exc: + raise M49PortableSourceError("exact admitted K1 replay could not be materialized") from exc + return materialize_m49_portable_source( + source_bundle_path=root / "source-bundle.json", + source_capability_path=root / "source-capability.json", + lidar_pack_root=lidar_pack_root, + profile_path=profile_path, + output_parent=parent / "tgs-source-stages", + expected=expected, + ) + + +def materialize_m49_portable_source( + *, + source_bundle_path: Path, + source_capability_path: Path, + lidar_pack_root: Path, + profile_path: Path, + output_parent: Path, + expected: M49PortableSourceIdentity, +) -> M49PortableSourceStage: + """Build one deterministic source-derived TRAVEL/TGS input stage. + + ``metadata_sha256`` is intentionally mandatory even though portable source + admission did not historically expose that sidecar. The LiDAR replay pack + requires exact host time; accepting an adjacent unsealed file would silently + repeat the legacy binding mistake. + """ + + bundle_payload, bundle = _read_canonical_document( + source_bundle_path, + expected_sha256=expected.source_bundle_sha256, + label="portable source bundle", + ) + capability_payload, capability = _read_canonical_document( + source_capability_path, + expected_sha256=expected.source_capability_manifest_sha256, + label="portable source capability", + ) + _verify_source_documents(bundle, capability, expected=expected) + profile = _read_profile(profile_path) + pack = LidarReplayPackV2(lidar_pack_root) + try: + _verify_lidar_pack(pack, bundle=bundle, expected=expected) + anchors = _camera_anchors(bundle) + if len(anchors) > M49_PORTABLE_MAX_TIMELINE_FRAMES: + raise M49PortableSourceError("portable camera timeline exceeds the release bound") + duration = anchors[-1].session_seconds - anchors[0].session_seconds + if not 0 < duration <= M49_PORTABLE_MAX_SOURCE_DURATION_SECONDS: + raise M49PortableSourceError("portable camera duration exceeds the release bound") + return _materialize_stage( + pack=pack, + anchors=anchors, + profile=profile, + expected=expected, + bundle_payload=bundle_payload, + capability_payload=capability_payload, + output_parent=output_parent, + ) + finally: + pack.close() + + +def validate_m49_portable_source_stage(root: Path) -> M49PortableSourceStage: + """Validate a materialized stage without trusting its directory label.""" + + candidate = root.expanduser().absolute() + try: + candidate.lstat() + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise M49PortableSourceError("portable source stage is unavailable") from exc + if candidate.is_symlink() or not resolved.is_dir() or not os.path.samefile(candidate, resolved): + raise M49PortableSourceError("portable source stage root is unsafe") + manifest_path = resolved / M49_PORTABLE_STAGE_MANIFEST + payload, manifest = _read_canonical_document( + manifest_path, + expected_sha256=None, + label="M4.9 source stage manifest", + ) + if manifest.get("schema_version") != M49_PORTABLE_SOURCE_STAGE_SCHEMA: + raise M49PortableSourceError("M4.9 source stage schema changed") + identity_sha256 = _string(manifest.get("identity_sha256"), "M4.9 source stage identity sha256") + _digest(identity_sha256, "M4.9 source stage identity sha256") + identity = _object(manifest.get("identity"), "M4.9 source stage identity") + if ( + hashlib.sha256(canonical_json(identity)).hexdigest() != identity_sha256 + or resolved.name != f"{M49_PORTABLE_STAGE_PREFIX}{identity_sha256}" + or manifest.get("authority") != _AUTHORITY + ): + raise M49PortableSourceError("M4.9 source stage identity changed") + timeline = _object(identity.get("timeline"), "M4.9 source stage timeline") + frame_count = _positive_int(timeline.get("frame_count"), "timeline frame count") + available_count = _positive_int( + timeline.get("available_lidar_frame_count"), + "available LiDAR frame count", + ) + if frame_count > M49_PORTABLE_MAX_TIMELINE_FRAMES or available_count > frame_count: + raise M49PortableSourceError("M4.9 source stage frame accounting is invalid") + artifacts = _object(manifest.get("artifacts"), "M4.9 source stage artifacts") + if set(artifacts) != {"schedule", "sequence-index"}: + raise M49PortableSourceError("M4.9 source stage artifacts changed") + schedule = _verify_stage_artifact(resolved, artifacts["schedule"], "schedule") + index = _verify_stage_artifact(resolved, artifacts["sequence-index"], "sequence-index") + records = read_m49_source_index(index, expected_frame_count=frame_count) + _verify_schedule(schedule, records) + if sum(bool(row["sample_available"]) for row in records) != available_count: + raise M49PortableSourceError("M4.9 source stage availability changed") + _verify_sequence_files(resolved, records) + return M49PortableSourceStage( + root=resolved, + identity_sha256=identity_sha256, + manifest_sha256=hashlib.sha256(payload).hexdigest(), + timeline_frame_count=frame_count, + available_lidar_frame_count=available_count, + ) + + +def read_m49_source_index( + path: Path, + *, + expected_frame_count: int, +) -> tuple[dict[str, object], ...]: + if not 1 <= expected_frame_count <= M49_PORTABLE_MAX_TIMELINE_FRAMES: + raise M49PortableSourceError("M4.9 source index frame bound is invalid") + rows: list[dict[str, object]] = [] + previous_seconds = -math.inf + try: + with path.open("rb") as stream: + for expected_index, line in enumerate(stream): + if len(line) > M49_PORTABLE_MAX_INDEX_LINE_BYTES or not line.endswith(b"\n"): + raise M49PortableSourceError("M4.9 source index line is invalid") + try: + decoded: object = json.loads(line) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise M49PortableSourceError("M4.9 source index is invalid JSON") from exc + row = _object(decoded, "M4.9 source index row") + if line[:-1] != canonical_json(row): + raise M49PortableSourceError("M4.9 source index is not canonical JSONL") + _verify_index_row(row, expected_index=expected_index) + seconds = _finite_float(row["session_seconds"], "source index time") + if seconds <= previous_seconds: + raise M49PortableSourceError("M4.9 source index time is not increasing") + rows.append(row) + previous_seconds = seconds + if len(rows) > expected_frame_count: + raise M49PortableSourceError("M4.9 source index has extra rows") + except M49PortableSourceError: + raise + except OSError as exc: + raise M49PortableSourceError("M4.9 source index is unavailable") from exc + if len(rows) != expected_frame_count: + raise M49PortableSourceError("M4.9 source index is incomplete") + return tuple(rows) + + +def _materialize_stage( + *, + pack: LidarReplayPackV2, + anchors: Sequence[_CameraAnchor], + profile: _Profile, + expected: M49PortableSourceIdentity, + bundle_payload: bytes, + capability_payload: bytes, + output_parent: Path, +) -> M49PortableSourceStage: + parent = output_parent.expanduser().absolute() + parent.mkdir(mode=0o700, parents=True, exist_ok=True) + if parent.is_symlink() or not parent.is_dir(): + raise M49PortableSourceError("portable source stage parent is unsafe") + staging = Path(tempfile.mkdtemp(prefix=".m49-source-stage-", dir=parent)) + published = False + try: + sequence_root = staging / M49_PORTABLE_TGS_SEQUENCE + sequence_root.mkdir(parents=True) + point_times = ( + np.asarray(pack.arrays["point_received_monotonic_ns"], dtype=np.int64) + - _timeline_origin_monotonic_ns(bundle_payload) + ).astype(np.float64) / 1e9 + pose_times = ( + np.asarray(pack.arrays["pose_received_monotonic_ns"], dtype=np.int64) + - _timeline_origin_monotonic_ns(bundle_payload) + ).astype(np.float64) / 1e9 + if ( + point_times.size < 1 + or pose_times.size < 1 + or np.any(np.diff(point_times) < 0) + or np.any(np.diff(pose_times) < 0) + ): + raise M49PortableSourceError("portable LiDAR host timeline is invalid") + + schedule_rows = [ + "timeline_frame_index\tsource_frame_index\tsession_seconds\tavailable_slot\tpoint_count" + ] + index_rows: list[dict[str, object]] = [] + available_slot = 0 + sequence_logical = hashlib.sha256() + for anchor in anchors: + point_index = int( + np.searchsorted(point_times, anchor.session_seconds, side="right") - 1 + ) + pose_index = -1 + if point_index >= 0: + pose_index = int( + np.searchsorted( + pose_times, + point_times[point_index], + side="right", + ) + - 1 + ) + available = ( + point_index >= 0 + and pose_index >= 0 + and anchor.session_seconds - float(point_times[point_index]) + <= profile.maximum_lidar_age_seconds + and float(point_times[point_index]) - float(pose_times[pose_index]) + <= profile.maximum_lidar_age_seconds + ) + base: dict[str, object] = { + "schema_version": M49_PORTABLE_SOURCE_INDEX_SCHEMA, + "timeline_frame_index": anchor.timeline_frame_index, + "source_frame_index": anchor.source_frame_index, + "session_seconds": anchor.session_seconds, + "sample_available": available, + } + if not available: + row = { + **base, + "available_slot": None, + "selected_lidar_frame_index": None, + "selected_pose_frame_index": None, + "contributing_lidar_frame_indices": [], + "position_map_m": None, + "point_count": 0, + "relative_path": None, + "byte_length": 0, + "sha256": None, + } + index_rows.append(row) + schedule_rows.append( + f"{anchor.timeline_frame_index}\t{anchor.source_frame_index}" + f"\t{anchor.session_seconds:.9f}\t-1\t0" + ) + continue + + start_seconds = anchor.session_seconds - profile.history_seconds + first = int(np.searchsorted(point_times, start_seconds, side="left")) + contributors = tuple(range(first, point_index + 1)) + if not contributors or contributors[-1] != point_index: + raise M49PortableSourceError("portable TGS causal window is invalid") + position = np.asarray(pack.arrays["pose_positions_map"][pose_index], dtype=np.float64) + if position.shape != (3,) or not np.isfinite(position).all(): + raise M49PortableSourceError("portable TGS pose is invalid") + clouds: list[npt.NDArray[np.float32]] = [] + for contributor in contributors: + frame = pack.point_frame(contributor) + xyz_map = np.asarray(frame.xyz_map, dtype=np.float64) + intensity = np.asarray(frame.intensity, dtype=np.uint8) + if xyz_map.shape != (intensity.shape[0], 3) or not np.isfinite(xyz_map).all(): + raise M49PortableSourceError("portable TGS source cloud is invalid") + relative = xyz_map - position + inside = np.linalg.norm(relative[:, :2], axis=1) <= profile.local_radius_m + selected = relative[inside] + selected_intensity = intensity[inside] + if selected.size: + xyzi = np.empty((selected.shape[0], 4), dtype=np.float32) + xyzi[:, :3] = selected.astype(np.float32) + xyzi[:, 3] = selected_intensity.astype(np.float32) + clouds.append(xyzi) + if not clouds: + raise M49PortableSourceError( + "available portable TGS frame produced no bounded local points" + ) + native = np.concatenate(clouds, axis=0) + if native.shape[0] > M49_PORTABLE_MAX_POINTS_PER_MATERIALIZED_FRAME: + raise M49PortableSourceError("portable TGS frame exceeds the point bound") + payload = np.ascontiguousarray(native).tobytes() + relative_path = ( + PurePosixPath(M49_PORTABLE_TGS_SEQUENCE) / f"{available_slot:06d}.bin" + ).as_posix() + target = staging.joinpath(*PurePosixPath(relative_path).parts) + target.write_bytes(payload) + payload_sha256 = hashlib.sha256(payload).hexdigest() + sequence_logical.update(bytes.fromhex(payload_sha256)) + row = { + **base, + "available_slot": available_slot, + "selected_lidar_frame_index": point_index, + "selected_pose_frame_index": pose_index, + "contributing_lidar_frame_indices": list(contributors), + "position_map_m": [float(value) for value in position], + "point_count": int(native.shape[0]), + "relative_path": relative_path, + "byte_length": len(payload), + "sha256": payload_sha256, + } + index_rows.append(row) + schedule_rows.append( + f"{anchor.timeline_frame_index}\t{anchor.source_frame_index}" + f"\t{anchor.session_seconds:.9f}\t{available_slot}\t{native.shape[0]}" + ) + available_slot += 1 + + if available_slot < 1: + raise M49PortableSourceError("portable K1 source has no admissible LiDAR frames") + schedule_path = staging / M49_PORTABLE_STAGE_SCHEDULE + schedule_path.write_text("\n".join(schedule_rows) + "\n", encoding="utf-8") + index_path = staging / M49_PORTABLE_STAGE_INDEX + index_path.write_bytes(b"".join(canonical_json(row) + b"\n" for row in index_rows)) + timeline = { + "frame_count": len(anchors), + "available_lidar_frame_count": available_slot, + "missing_lidar_frame_count": len(anchors) - available_slot, + "timeline_start_seconds": anchors[0].session_seconds, + "timeline_end_seconds": anchors[-1].session_seconds, + } + identity = { + "schema_version": M49_PORTABLE_SOURCE_STAGE_SCHEMA, + "source": expected.as_dict(), + "profile": { + "profile_id": profile.profile_id, + "profile_sha256": profile.profile_sha256, + }, + "lidar_replay": { + "pack_id": pack.pack_id, + "identity_sha256": _string( + pack.manifest.get("identity_sha256"), + "LiDAR replay identity sha256", + ), + "logical_content_sha256": _string( + pack.identity.get("logical_content_sha256"), + "LiDAR replay logical content sha256", + ), + "sequence_logical_sha256": sequence_logical.hexdigest(), + }, + "timeline": timeline, + "alignment": { + "camera_anchor": "recorded-fragment-host-arrival-start", + "lidar_selection": "latest-not-newer-than-camera-frame", + "pose_selection": "latest-not-newer-than-selected-lidar-frame", + "maximum_age_seconds": profile.maximum_lidar_age_seconds, + "future_frames_used": False, + }, + "rolling_profile": { + "history_seconds": profile.history_seconds, + "local_radius_m": profile.local_radius_m, + "missing_lidar_policy": "all-cells-unobserved", + }, + "authority": dict(_AUTHORITY), + } + identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest() + manifest = { + "schema_version": M49_PORTABLE_SOURCE_STAGE_SCHEMA, + "identity_sha256": identity_sha256, + "identity": identity, + "artifacts": { + "schedule": _artifact_descriptor( + schedule_path, + relative_path=M49_PORTABLE_STAGE_SCHEDULE, + media_type="text/tab-separated-values", + ), + "sequence-index": _artifact_descriptor( + index_path, + relative_path=M49_PORTABLE_STAGE_INDEX, + media_type="application/x-ndjson", + ), + }, + "authority": dict(_AUTHORITY), + } + manifest_path = staging / M49_PORTABLE_STAGE_MANIFEST + manifest_path.write_bytes(canonical_json(manifest)) + final = parent / f"{M49_PORTABLE_STAGE_PREFIX}{identity_sha256}" + if final.exists(): + existing = validate_m49_portable_source_stage(final) + if existing.manifest_sha256 != hashlib.sha256(manifest_path.read_bytes()).hexdigest(): + raise M49PortableSourceError("existing portable source stage has another manifest") + return existing + os.replace(staging, final) + published = True + return validate_m49_portable_source_stage(final) + finally: + if not published: + shutil.rmtree(staging, ignore_errors=True) + + +def _read_profile(path: Path) -> _Profile: + payload, profile = _read_canonical_or_pretty_document(path, label="portable M4.9 profile") + if profile.get("schema_version") != M49_PORTABLE_PROFILE_SCHEMA: + raise M49PortableSourceError("portable M4.9 profile schema changed") + alignment = _object(profile.get("alignment"), "portable M4.9 alignment") + rolling = _object(profile.get("rolling_profile"), "portable M4.9 rolling profile") + if ( + profile.get("profile_id") != "m49-tgs-portable-v2" + or alignment.get("timeline") != "recorded-camera-host-arrival" + or alignment.get("lidar_selection") != "latest-not-newer-than-camera-frame" + or alignment.get("future_frames_allowed") is not False + or rolling.get("missing_lidar_policy") != "all-cells-unobserved" + or _object(profile.get("invariants"), "portable M4.9 invariants").get( + "navigation_or_actuation_allowed" + ) + is not False + ): + raise M49PortableSourceError("portable M4.9 profile invariants changed") + history = _positive_float(rolling.get("history_seconds"), "history seconds") + radius = _positive_float(rolling.get("local_radius_m"), "local radius") + maximum_age = _positive_float(alignment.get("maximum_lidar_age_seconds"), "maximum LiDAR age") + if history > 10 or radius > 100 or maximum_age > 10: + raise M49PortableSourceError("portable M4.9 source profile exceeds release bounds") + return _Profile( + profile_id="m49-tgs-portable-v2", + profile_sha256=hashlib.sha256(payload).hexdigest(), + history_seconds=history, + local_radius_m=radius, + maximum_lidar_age_seconds=maximum_age, + ) + + +def _verify_source_documents( + bundle: Mapping[str, object], + capability: Mapping[str, object], + *, + expected: M49PortableSourceIdentity, +) -> None: + adapter = _object(bundle.get("source_adapter"), "portable source adapter") + camera = _object(bundle.get("camera"), "portable source camera") + epoch = _object(camera.get("epoch"), "portable source camera epoch") + camera_profile = _object(capability.get("camera_profile"), "portable source camera profile") + if ( + bundle.get("schema_version") != PORTABLE_SOURCE_BUNDLE_SCHEMA + or bundle.get("source_session_id") != expected.source_session_id + or bundle.get("source_catalog_sha256") != expected.source_catalog_sha256 + or adapter.get("sha256") != expected.source_adapter_sha256 + or bundle.get("authority") != _AUTHORITY + or capability.get("schema_version") != PORTABLE_SOURCE_CAPABILITY_SCHEMA + or capability.get("source_session_id") != expected.source_session_id + or capability.get("source_catalog_sha256") != expected.source_catalog_sha256 + or capability.get("source_bundle_sha256") != expected.source_bundle_sha256 + or capability.get("source_adapter_sha256") != expected.source_adapter_sha256 + or capability.get("authority") != _AUTHORITY + or camera_profile.get("generation_sha256") != camera.get("generation_sha256") + or camera_profile.get("frame_count") + != len(_array(epoch.get("segments"), "portable source camera segments")) + ): + raise M49PortableSourceError("portable K1 source documents disagree") + + +def _verify_lidar_pack( + pack: LidarReplayPackV2, + *, + bundle: Mapping[str, object], + expected: M49PortableSourceIdentity, +) -> None: + if _PACK_ID.fullmatch(pack.pack_id) is None or pack.identity.get("session_id") != ( + expected.source_session_id + ): + raise M49PortableSourceError("LiDAR replay pack belongs to another session") + evidence = _object(pack.identity.get("source_evidence"), "LiDAR source evidence") + raw = _object(evidence.get("raw"), "LiDAR raw source evidence") + metadata = _object(evidence.get("metadata"), "LiDAR metadata source evidence") + replay = _object(bundle.get("spatial_replay"), "portable spatial replay") + members = _array(replay.get("members"), "portable spatial replay members") + raw_members = [ + _object(member, "portable replay member") + for member in members + if ( + _object(member, "portable replay member").get("artifact_id") == "raw-transport-primary" + and _object(member, "portable replay member").get("media_type") + == "application/x-nodedc-k1mqtt" + ) + ] + metadata_members = [ + _object(member, "portable replay member") + for member in members + if ( + _object(member, "portable replay member").get("artifact_id") + == PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID + and _object(member, "portable replay member").get("media_type") + == PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE + ) + ] + if ( + len(raw_members) != 1 + or len(metadata_members) != 1 + or raw_members[0].get("sha256") != expected.raw_capture_sha256 + or metadata_members[0].get("sha256") != expected.metadata_sha256 + or raw.get("sha256") != expected.raw_capture_sha256 + or metadata.get("sha256") != expected.metadata_sha256 + or raw.get("byte_length") != raw_members[0].get("byte_length") + or metadata.get("byte_length") != metadata_members[0].get("byte_length") + ): + raise M49PortableSourceError( + "LiDAR replay pack is not bound to admitted raw and host-time evidence" + ) + + +def _camera_anchors(bundle: Mapping[str, object]) -> tuple[_CameraAnchor, ...]: + camera = _object(bundle.get("camera"), "portable source camera") + epoch = _object(camera.get("epoch"), "portable source camera epoch") + start = _finite_float(epoch.get("timeline_start_seconds"), "camera timeline start") + end = _finite_float(epoch.get("timeline_end_seconds"), "camera timeline end") + segments = _array(epoch.get("segments"), "portable source camera segments") + if not segments or end <= start: + raise M49PortableSourceError("portable camera timeline is incomplete") + anchors: list[_CameraAnchor] = [] + previous_end = 0.0 + for index, value in enumerate(segments): + segment = _object(value, "portable source camera segment") + sequence = _positive_int(segment.get("sequence"), "camera segment sequence") + segment_end = _positive_float(segment.get("end_time_seconds"), "camera segment end time") + if sequence != index + 1 or segment_end <= previous_end: + raise M49PortableSourceError("portable camera segment order changed") + anchors.append( + _CameraAnchor( + timeline_frame_index=index, + source_frame_index=sequence - 1, + session_seconds=start + previous_end, + ) + ) + previous_end = segment_end + if not math.isclose(start + previous_end, end, rel_tol=0.0, abs_tol=0.05): + raise M49PortableSourceError("portable camera segment duration changed") + return tuple(anchors) + + +def _timeline_origin_monotonic_ns(bundle_payload: bytes) -> int: + try: + document = cast(dict[str, object], json.loads(bundle_payload)) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise M49PortableSourceError("portable source bundle is invalid") from exc + replay = _object(document.get("spatial_replay"), "portable spatial replay") + return _non_negative_int( + replay.get("timeline_origin_monotonic_ns"), + "spatial replay monotonic origin", + ) + + +def _verify_index_row(row: Mapping[str, object], *, expected_index: int) -> None: + expected_keys = { + "schema_version", + "timeline_frame_index", + "source_frame_index", + "session_seconds", + "sample_available", + "available_slot", + "selected_lidar_frame_index", + "selected_pose_frame_index", + "contributing_lidar_frame_indices", + "position_map_m", + "point_count", + "relative_path", + "byte_length", + "sha256", + } + if set(row) != expected_keys or row.get("schema_version") != (M49_PORTABLE_SOURCE_INDEX_SCHEMA): + raise M49PortableSourceError("M4.9 source index row fields changed") + if _non_negative_int(row.get("timeline_frame_index"), "timeline frame index") != ( + expected_index + ): + raise M49PortableSourceError("M4.9 source index order changed") + _non_negative_int(row.get("source_frame_index"), "source frame index") + _finite_float(row.get("session_seconds"), "source index session time") + available = row.get("sample_available") + if not isinstance(available, bool): + raise M49PortableSourceError("M4.9 source index availability is invalid") + point_count = _non_negative_int(row.get("point_count"), "source index point count") + byte_length = _non_negative_int(row.get("byte_length"), "source index byte length") + contributors = _array(row.get("contributing_lidar_frame_indices"), "contributing LiDAR frames") + if any(_non_negative_int(value, "contributing LiDAR frame") < 0 for value in contributors): + raise M49PortableSourceError("contributing LiDAR frame is invalid") + if available: + slot = _non_negative_int(row.get("available_slot"), "available slot") + _non_negative_int(row.get("selected_lidar_frame_index"), "selected LiDAR frame index") + _non_negative_int(row.get("selected_pose_frame_index"), "selected pose frame index") + position = _array(row.get("position_map_m"), "source index pose") + if len(position) != 3 or any( + not math.isfinite(_finite_float(value, "source index pose value")) for value in position + ): + raise M49PortableSourceError("source index pose is invalid") + relative = _relative_path(row.get("relative_path")) + if ( + relative.as_posix() != f"{M49_PORTABLE_TGS_SEQUENCE}/{slot:06d}.bin" + or point_count < 1 + or byte_length != point_count * 16 + or not contributors + ): + raise M49PortableSourceError("available M4.9 source index row is invalid") + _digest(row.get("sha256"), "source index artifact sha256") + elif ( + any( + row.get(key) is not None + for key in ( + "available_slot", + "selected_lidar_frame_index", + "selected_pose_frame_index", + "position_map_m", + "relative_path", + "sha256", + ) + ) + or point_count != 0 + or byte_length != 0 + or contributors + ): + raise M49PortableSourceError("missing M4.9 source row invents evidence") + + +def _verify_sequence_files(root: Path, rows: Sequence[Mapping[str, object]]) -> None: + expected_paths: set[Path] = set() + for row in rows: + if row["sample_available"] is not True: + continue + relative = _relative_path(row["relative_path"]) + path = root.joinpath(*relative.parts) + _verify_exact_file( + path, + parent=root, + expected_sha256=_string(row["sha256"], "sequence file sha256"), + expected_bytes=_positive_int(row["byte_length"], "sequence file bytes"), + ) + expected_paths.add(path.resolve()) + sequence_root = root.joinpath(*PurePosixPath(M49_PORTABLE_TGS_SEQUENCE).parts) + actual = { + path.resolve() + for path in sequence_root.iterdir() + if path.is_file() and not path.is_symlink() + } + if actual != expected_paths: + raise M49PortableSourceError("portable TGS sequence contains unexpected files") + + +def _verify_schedule(path: Path, records: Sequence[Mapping[str, object]]) -> None: + expected = [ + "timeline_frame_index\tsource_frame_index\tsession_seconds\tavailable_slot\tpoint_count" + ] + for row in records: + slot = row["available_slot"] if row["available_slot"] is not None else -1 + expected.append( + f"{row['timeline_frame_index']}\t{row['source_frame_index']}" + f"\t{_finite_float(row['session_seconds'], 'schedule session time'):.9f}" + f"\t{slot}\t{row['point_count']}" + ) + try: + payload = path.read_text(encoding="utf-8") + except OSError as exc: + raise M49PortableSourceError("portable TGS schedule is unavailable") from exc + if payload != "\n".join(expected) + "\n": + raise M49PortableSourceError("portable TGS schedule changed") + + +def _verify_worker_materialization_manifest( + root: Path, + manifest: Mapping[str, object], + *, + job: SealedObservatoryRecordedJob, +) -> tuple[dict[str, object], ...]: + expected_top = { + "schema_version", + "job_id", + "job_identity_sha256", + "claim_generation", + "source", + "members", + "authority", + } + if ( + set(manifest) != expected_top + or manifest.get("schema_version") != PORTABLE_SOURCE_MATERIALIZATION_SCHEMA + or manifest.get("job_id") != job.job_id + or manifest.get("job_identity_sha256") != job.identity_sha256 + or manifest.get("claim_generation") != job.claim_generation + or manifest.get("authority") != _AUTHORITY + ): + raise M49PortableSourceError("Worker source materialization identity changed") + source = _object(manifest.get("source"), "Worker source materialization source") + if set(source) != { + "session_id", + "bundle_sha256", + "capability_manifest_sha256", + } or source != { + "session_id": job.source_session_id, + "bundle_sha256": job.source_bundle_sha256, + "capability_manifest_sha256": job.source_capability_manifest_sha256, + }: + raise M49PortableSourceError("Worker source materialization source changed") + values = _array(manifest.get("members"), "Worker source materialization members") + if not 3 <= len(values) <= 100_000: + raise M49PortableSourceError("Worker source materialization member count changed") + members: list[dict[str, object]] = [] + expected_paths: set[Path] = {(root / "materialization-manifest.json").resolve()} + for value in values: + member = _object(value, "Worker source materialization member") + if set(member) != { + "member_id", + "kind", + "media_type", + "byte_length", + "sha256", + "artifact_id", + "primary", + "camera_epoch", + "camera_sequence", + }: + raise M49PortableSourceError("Worker source materialization member fields changed") + member_id = _digest(member.get("member_id"), "source member id") + kind = _string(member.get("kind"), "source member kind") + if kind not in { + "source-bundle", + "source-capability", + "spatial-replay", + "spatial-replay-metadata", + "camera-init", + "camera-segment", + }: + raise M49PortableSourceError("Worker source materialization kind changed") + media_type = _string(member.get("media_type"), "source member media type") + byte_length = _positive_int(member.get("byte_length"), "source member bytes") + sha256 = _digest(member.get("sha256"), "source member sha256") + artifact_id = member.get("artifact_id") + primary = member.get("primary") + camera_epoch = member.get("camera_epoch") + camera_sequence = member.get("camera_sequence") + if not isinstance(primary, bool) or ( + artifact_id is not None + and (not isinstance(artifact_id, str) or _SESSION_ID.fullmatch(artifact_id) is None) + ): + raise M49PortableSourceError("Worker source member metadata is invalid") + for ordinal in (camera_epoch, camera_sequence): + if ordinal is not None: + _positive_int(ordinal, "source camera ordinal") + if kind in {"source-bundle", "source-capability"}: + if ( + artifact_id is not None + or primary + or camera_epoch is not None + or camera_sequence is not None + ): + raise M49PortableSourceError("source document member metadata changed") + elif kind in {"spatial-replay", "spatial-replay-metadata"}: + if ( + artifact_id is None + or camera_epoch is not None + or camera_sequence is not None + or (kind == "spatial-replay-metadata" and primary) + ): + raise M49PortableSourceError("spatial source member metadata changed") + elif kind == "camera-init": + if ( + artifact_id is None + or primary + or camera_epoch is None + or camera_sequence is not None + ): + raise M49PortableSourceError("camera init member metadata changed") + elif artifact_id is None or primary or camera_epoch is None or camera_sequence is None: + raise M49PortableSourceError("camera segment member metadata changed") + expected_member_id = hashlib.sha256( + canonical_json( + { + "job_identity_sha256": job.identity_sha256, + "source_bundle_sha256": job.source_bundle_sha256, + "kind": kind, + "artifact_id": artifact_id, + "primary": primary, + "camera_epoch": camera_epoch, + "camera_sequence": camera_sequence, + "media_type": media_type, + "byte_length": byte_length, + "sha256": sha256, + } + ) + ).hexdigest() + if member_id != expected_member_id: + raise M49PortableSourceError("Worker source member identity changed") + path = _worker_member_path(root, member) + _verify_exact_file( + path, + parent=root, + expected_sha256=sha256, + expected_bytes=byte_length, + ) + expected_paths.add(path.resolve()) + members.append(member) + member_ids = [cast(str, member["member_id"]) for member in members] + if member_ids != sorted(member_ids) or len(member_ids) != len(set(member_ids)): + raise M49PortableSourceError("Worker source members are not canonical") + if ( + sum(member["kind"] == "source-bundle" for member in members) != 1 + or sum(member["kind"] == "source-capability" for member in members) != 1 + ): + raise M49PortableSourceError("Worker source documents are incomplete") + raw_members = [ + member + for member in members + if member["kind"] == "spatial-replay" + and member["artifact_id"] == "raw-transport-primary" + and member["primary"] is True + and member["media_type"] == "application/x-nodedc-k1mqtt" + ] + metadata_members = [ + member + for member in members + if member["kind"] == "spatial-replay-metadata" + and member["artifact_id"] == PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID + and member["primary"] is False + and member["media_type"] == PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE + ] + if len(raw_members) != 1 or len(metadata_members) != 1: + raise M49PortableSourceError( + "portable M4.9 requires exact admitted raw and metadata members" + ) + try: + actual_paths = set() + for path in root.rglob("*"): + if path.is_symlink(): + raise M49PortableSourceError("Worker source stage contains a symlink") + if path.is_file(): + actual_paths.add(path.resolve()) + except OSError as exc: + raise M49PortableSourceError("Worker source stage is unreadable") from exc + if actual_paths != expected_paths: + raise M49PortableSourceError("Worker source stage contains unmanifested files") + return tuple(members) + + +def _worker_member_path(root: Path, member: Mapping[str, object]) -> Path: + kind = member["kind"] + if kind == "source-bundle": + return root / "source-bundle.json" + if kind == "source-capability": + return root / "source-capability.json" + if kind == "spatial-replay-metadata": + return root / "mqtt.metadata.jsonl" + if kind == "spatial-replay": + if member["primary"] is True: + return root / "mqtt.raw.k1mqtt" + return root / "spatial" / cast(str, member["member_id"]) + epoch = _positive_int(member["camera_epoch"], "source camera epoch") + epoch_root = root / "camera" / f"epoch-{epoch}" + if kind == "camera-init": + return epoch_root / "init.mp4" + sequence = _positive_int(member["camera_sequence"], "source camera sequence") + return epoch_root / "segments" / f"{sequence}.m4s" + + +def _safe_directory(path: Path, label: str) -> Path: + candidate = path.expanduser().absolute() + try: + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise M49PortableSourceError(f"{label} is unavailable") from exc + if candidate.is_symlink() or not resolved.is_dir() or not os.path.samefile(candidate, resolved): + raise M49PortableSourceError(f"{label} is unsafe") + return resolved + + +def _artifact_descriptor(path: Path, *, relative_path: str, media_type: str) -> dict[str, object]: + return { + "relative_path": relative_path, + "media_type": media_type, + "byte_length": path.stat().st_size, + "sha256": _sha256_file(path), + } + + +def _verify_stage_artifact(root: Path, value: object, role: str) -> Path: + descriptor = _object(value, f"M4.9 {role} descriptor") + if set(descriptor) != {"relative_path", "media_type", "byte_length", "sha256"}: + raise M49PortableSourceError(f"M4.9 {role} descriptor fields changed") + relative = _relative_path(descriptor["relative_path"]) + if len(relative.parts) != 1: + raise M49PortableSourceError(f"M4.9 {role} path changed") + path = root.joinpath(*relative.parts) + _verify_exact_file( + path, + parent=root, + expected_sha256=_string(descriptor["sha256"], f"M4.9 {role} sha256"), + expected_bytes=_positive_int(descriptor["byte_length"], f"M4.9 {role} byte length"), + ) + return path + + +def _verify_exact_file( + path: Path, + *, + parent: Path, + expected_sha256: str, + expected_bytes: int, +) -> None: + try: + metadata = path.lstat() + resolved = path.resolve(strict=True) + except OSError as exc: + raise M49PortableSourceError("portable M4.9 artifact is unavailable") from exc + if ( + path.is_symlink() + or not path.is_file() + or not resolved.is_relative_to(parent.resolve()) + or metadata.st_size != expected_bytes + or _sha256_file(path) != expected_sha256 + ): + raise M49PortableSourceError("portable M4.9 artifact identity changed") + + +def _read_canonical_document( + path: Path, + *, + expected_sha256: str | None, + label: str, +) -> tuple[bytes, dict[str, object]]: + try: + path.lstat() + payload = path.read_bytes() + except OSError as exc: + raise M49PortableSourceError(f"{label} is unavailable") from exc + if path.is_symlink() or not path.is_file() or not payload: + raise M49PortableSourceError(f"{label} is not a regular non-empty file") + if expected_sha256 is not None and hashlib.sha256(payload).hexdigest() != expected_sha256: + raise M49PortableSourceError(f"{label} digest changed") + try: + decoded: object = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise M49PortableSourceError(f"{label} is invalid JSON") from exc + document = _object(decoded, label) + if payload != canonical_json(document): + raise M49PortableSourceError(f"{label} is not canonical JSON") + return payload, document + + +def _read_canonical_or_pretty_document( + path: Path, + *, + label: str, +) -> tuple[bytes, dict[str, object]]: + try: + payload = path.read_bytes() + decoded: object = json.loads(payload) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise M49PortableSourceError(f"{label} is invalid") from exc + return payload, _object(decoded, label) + + +def _relative_path(value: object) -> PurePosixPath: + if not isinstance(value, str) or not 1 <= len(value) <= 512: + raise M49PortableSourceError("portable M4.9 relative path is invalid") + path = PurePosixPath(value) + if ( + path.is_absolute() + or path.as_posix() != value + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise M49PortableSourceError("portable M4.9 relative path is unsafe") + return path + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _object(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise M49PortableSourceError(f"{label} must be an object") + return cast(dict[str, object], value) + + +def _array(value: object, label: str) -> list[object]: + if not isinstance(value, list): + raise M49PortableSourceError(f"{label} must be an array") + return value + + +def _string(value: object, label: str) -> str: + if not isinstance(value, str): + raise M49PortableSourceError(f"{label} must be text") + return value + + +def _digest(value: object, label: str) -> str: + if not isinstance(value, str) or _SHA256.fullmatch(value) is None: + raise M49PortableSourceError(f"{label} is invalid") + return value + + +def _non_negative_int(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise M49PortableSourceError(f"{label} is invalid") + return value + + +def _positive_int(value: object, label: str) -> int: + result = _non_negative_int(value, label) + if result < 1: + raise M49PortableSourceError(f"{label} is invalid") + return result + + +def _finite_float(value: object, label: str) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + ): + raise M49PortableSourceError(f"{label} is invalid") + return float(value) + + +def _positive_float(value: object, label: str) -> float: + result = _finite_float(value, label) + if result <= 0: + raise M49PortableSourceError(f"{label} is invalid") + return result diff --git a/src/k1link/observatory/portable_artifact_transport.py b/src/k1link/observatory/portable_artifact_transport.py new file mode 100644 index 0000000..db3ab41 --- /dev/null +++ b/src/k1link/observatory/portable_artifact_transport.py @@ -0,0 +1,1714 @@ +"""Claim-bound source delivery and result staging for portable Observatory jobs. + +The public contract contains content identities and opaque member ids only. +Host paths are resolved from the already admitted SessionStore snapshot and +never accepted from a Worker request. Source files are copied into a local +content-addressed store before they are exposed. Result files use a +manifest-first, atomic-per-member upload protocol whose completed receipt is +bound to the exact job, claim token and claim generation. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import secrets +import stat +from collections.abc import AsyncIterable, Mapping, Sequence +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal + +from k1link.observatory.portable_result_contract import ( + OBSERVATION_ONLY_AUTHORITY, + PortableResultArtifact, + PortableResultPackageIntegrityError, + PortableResultPackageManifest, + canonical_json, + job_identity_document, + object_document, + relative_artifact_path, + result_identity_document, + run_definition_document, + source_identity_document, + string, +) +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinition, + PortableRunDefinitionRegistry, + PortableRunDefinitionRegistryError, +) +from k1link.observatory.recorded_jobs import ( + ObservatoryRecordedJob, + ObservatoryRecordedJobQueue, +) +from k1link.observatory.source_admission import ( + PORTABLE_SOURCE_BUNDLE_SCHEMA, + PORTABLE_SOURCE_CAPABILITY_SCHEMA, + PORTABLE_SOURCE_DOCUMENT_DIRECTORY, + PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID, + PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE, +) +from k1link.sessions.media import RecordedMediaInspector, RecordedMediaManifest +from k1link.sessions.models import ReplayCommand +from k1link.sessions.store import SessionStore + +PORTABLE_SOURCE_MATERIALIZATION_SCHEMA: Final = ( + "missioncore.observatory-portable-source-materialization/v1" +) +PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA: Final = ( + "missioncore.observatory-portable-result-upload-plan/v1" +) +PORTABLE_RESULT_UPLOAD_BINDING_SCHEMA: Final = ( + "missioncore.observatory-portable-result-upload-binding/v1" +) +PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA: Final = ( + "missioncore.observatory-portable-result-upload-receipt/v1" +) +PORTABLE_SOURCE_CAS_DIRECTORY: Final = "observatory-worker-source-cas" +PORTABLE_RESULT_STAGING_DIRECTORY: Final = "observatory-worker-result-staging" + +MAX_SOURCE_MEMBERS: Final = 100_000 +MAX_SOURCE_BYTES: Final = 2 * 1024 * 1024 * 1024 * 1024 +MAX_RESULT_MEMBER_BYTES: Final = 64 * 1024 * 1024 * 1024 +MAX_RESULT_PACKAGE_BYTES: Final = 256 * 1024 * 1024 * 1024 +MAX_RESULT_MANIFEST_BYTES: Final = 1024 * 1024 +COPY_CHUNK_BYTES: Final = 1024 * 1024 + +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_JOB_ID = re.compile(r"^observatory-run-[a-f0-9]{32}$") +_MEMBER_ID = re.compile(r"^[a-f0-9]{64}$") +_SESSION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +_CLAIMANT_ID = re.compile(r"^[a-z][a-z0-9-]{2,95}$") + +type SourceMemberKind = Literal[ + "source-bundle", + "source-capability", + "spatial-replay", + "spatial-replay-metadata", + "camera-init", + "camera-segment", +] + + +class PortableArtifactTransportError(RuntimeError): + """Base failure at the portable Worker artifact boundary.""" + + +class PortableArtifactTransportIntegrityError(PortableArtifactTransportError): + """A persisted contract, source member, or uploaded package changed.""" + + +class PortableArtifactTransportUnavailableError(PortableArtifactTransportError): + """A required admitted member or completed upload is unavailable.""" + + +@dataclass(frozen=True, slots=True) +class PortableSourceMember: + member_id: str + kind: SourceMemberKind + media_type: str + byte_length: int + sha256: str + artifact_id: str | None = None + primary: bool = False + camera_epoch: int | None = None + camera_sequence: int | None = None + _source_path: Path | None = None + + def __post_init__(self) -> None: + _digest(self.member_id, "source member id") + _digest(self.sha256, "source member sha256") + if ( + not isinstance(self.byte_length, int) + or isinstance(self.byte_length, bool) + or not 0 <= self.byte_length <= MAX_SOURCE_BYTES + ): + raise ValueError("source member byte length is invalid") + _media_type(self.media_type) + if self.kind in {"source-bundle", "source-capability"}: + if any( + value is not None + for value in (self.artifact_id, self.camera_epoch, self.camera_sequence) + ) or self.primary: + raise ValueError("source document member metadata is invalid") + elif self.kind in {"spatial-replay", "spatial-replay-metadata"}: + if ( + self.artifact_id is None + or self.camera_epoch is not None + or self.camera_sequence is not None + or (self.kind == "spatial-replay-metadata" and self.primary) + ): + raise ValueError("spatial source member metadata is invalid") + elif self.kind == "camera-init": + if ( + self.artifact_id is None + or self.camera_epoch is None + or self.camera_sequence is not None + or self.primary + ): + raise ValueError("camera init member metadata is invalid") + elif ( + self.artifact_id is None + or self.camera_epoch is None + or self.camera_sequence is None + or self.primary + ): + raise ValueError("camera segment member metadata is invalid") + if self._source_path is None: + raise ValueError("resolved source member has no internal source") + + def as_dict(self) -> dict[str, object]: + return { + "member_id": self.member_id, + "kind": self.kind, + "media_type": self.media_type, + "byte_length": self.byte_length, + "sha256": self.sha256, + "artifact_id": self.artifact_id, + "primary": self.primary, + "camera_epoch": self.camera_epoch, + "camera_sequence": self.camera_sequence, + } + + +@dataclass(frozen=True, slots=True) +class PortableSourceMaterializationManifest: + job_id: str + job_identity_sha256: str + claim_generation: int + source_session_id: str + source_bundle_sha256: str + source_capability_manifest_sha256: str + members: tuple[PortableSourceMember, ...] + + def __post_init__(self) -> None: + _job_id(self.job_id) + _digest(self.job_identity_sha256, "source materialization job identity") + _positive_int(self.claim_generation, "source materialization claim generation") + _session_id(self.source_session_id) + _digest(self.source_bundle_sha256, "source bundle sha256") + _digest( + self.source_capability_manifest_sha256, + "source capability manifest sha256", + ) + if ( + not 2 <= len(self.members) <= MAX_SOURCE_MEMBERS + or len({member.member_id for member in self.members}) != len(self.members) + or sum(member.byte_length for member in self.members) > MAX_SOURCE_BYTES + ): + raise PortableArtifactTransportIntegrityError( + "source materialization member bounds are invalid" + ) + if tuple(member.member_id for member in self.members) != tuple( + sorted(member.member_id for member in self.members) + ): + raise PortableArtifactTransportIntegrityError( + "source materialization members are not canonical" + ) + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": PORTABLE_SOURCE_MATERIALIZATION_SCHEMA, + "job_id": self.job_id, + "job_identity_sha256": self.job_identity_sha256, + "claim_generation": self.claim_generation, + "source": { + "session_id": self.source_session_id, + "bundle_sha256": self.source_bundle_sha256, + "capability_manifest_sha256": ( + self.source_capability_manifest_sha256 + ), + }, + "members": [member.as_dict() for member in self.members], + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + + +@dataclass(frozen=True, slots=True) +class PortableResultUploadMember: + member_id: str + role: str + media_type: str + byte_length: int + sha256: str + uploaded: bool + + def as_dict(self) -> dict[str, object]: + return { + "member_id": self.member_id, + "role": self.role, + "media_type": self.media_type, + "byte_length": self.byte_length, + "sha256": self.sha256, + "uploaded": self.uploaded, + } + + +@dataclass(frozen=True, slots=True) +class PortableResultUploadPlan: + job_id: str + claim_generation: int + result_id: str + result_sha256: str + package_identity_sha256: str + members: tuple[PortableResultUploadMember, ...] + complete: bool + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA, + "job_id": self.job_id, + "claim_generation": self.claim_generation, + "result_id": self.result_id, + "result_sha256": self.result_sha256, + "package_identity_sha256": self.package_identity_sha256, + "members": [member.as_dict() for member in self.members], + "complete": self.complete, + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + + +@dataclass(frozen=True, slots=True) +class PortableResultUploadReceipt: + job_id: str + job_identity_sha256: str + claim_generation: int + claim_token_sha256: str + result_id: str + result_sha256: str + package_identity_sha256: str + member_count: int + total_bytes: int + + def __post_init__(self) -> None: + _job_id(self.job_id) + _digest(self.job_identity_sha256, "receipt job identity") + _positive_int(self.claim_generation, "receipt claim generation") + _digest(self.claim_token_sha256, "receipt claim token") + _session_id(self.result_id) + _digest(self.result_sha256, "receipt result sha256") + _digest(self.package_identity_sha256, "receipt package identity") + if ( + not isinstance(self.member_count, int) + or isinstance(self.member_count, bool) + or not 1 <= self.member_count <= 128 + or not isinstance(self.total_bytes, int) + or isinstance(self.total_bytes, bool) + or not 0 <= self.total_bytes <= MAX_RESULT_PACKAGE_BYTES + ): + raise ValueError("portable result receipt bounds are invalid") + + def identity_document(self) -> dict[str, object]: + return { + "schema_version": PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA, + "job_id": self.job_id, + "job_identity_sha256": self.job_identity_sha256, + "claim_generation": self.claim_generation, + "claim_token_sha256": self.claim_token_sha256, + "result_id": self.result_id, + "result_sha256": self.result_sha256, + "package_identity_sha256": self.package_identity_sha256, + "member_count": self.member_count, + "total_bytes": self.total_bytes, + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + + @property + def receipt_sha256(self) -> str: + return hashlib.sha256(canonical_json(self.identity_document())).hexdigest() + + def as_dict(self) -> dict[str, object]: + return {**self.identity_document(), "receipt_sha256": self.receipt_sha256} + + +@dataclass(frozen=True, slots=True) +class _ResultUploadBinding: + job_id: str + job_identity_sha256: str + claim_generation: int + claim_token_sha256: str + result_id: str + result_sha256: str + package_identity_sha256: str + + def __post_init__(self) -> None: + _job_id(self.job_id) + _digest(self.job_identity_sha256, "upload job identity") + _positive_int(self.claim_generation, "upload claim generation") + _digest(self.claim_token_sha256, "upload claim token") + _session_id(self.result_id) + _digest(self.result_sha256, "upload result sha256") + _digest(self.package_identity_sha256, "upload package identity") + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": PORTABLE_RESULT_UPLOAD_BINDING_SCHEMA, + "job_id": self.job_id, + "job_identity_sha256": self.job_identity_sha256, + "claim_generation": self.claim_generation, + "claim_token_sha256": self.claim_token_sha256, + "result_id": self.result_id, + "result_sha256": self.result_sha256, + "package_identity_sha256": self.package_identity_sha256, + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + + +class PortableObservatoryArtifactTransport: + """Resolve, stage and verify artifacts without widening execution authority.""" + + def __init__( + self, + *, + queue: ObservatoryRecordedJobQueue, + session_store: SessionStore, + media_inspector: RecordedMediaInspector, + definitions: PortableRunDefinitionRegistry, + source_cas_root: Path | None = None, + result_staging_root: Path | None = None, + ) -> None: + self._queue = queue + self._session_store = session_store + self._media_inspector = media_inspector + self._definitions = definitions + data_dir = session_store.data_dir + self._source_documents = data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY + self._source_cas = _prepare_secure_root( + source_cas_root or data_dir / PORTABLE_SOURCE_CAS_DIRECTORY, + create=source_cas_root is None, + ) + self._result_staging = _prepare_secure_root( + result_staging_root or data_dir / PORTABLE_RESULT_STAGING_DIRECTORY, + create=result_staging_root is None, + ) + + def source_manifest( + self, + *, + job_id: str, + claim_token: str, + claim_generation: int, + claimant_id: str, + ) -> PortableSourceMaterializationManifest: + job = self._authorize( + job_id=job_id, + claim_token=claim_token, + claim_generation=claim_generation, + claimant_id=claimant_id, + allowed_states=("claimed", "running"), + ) + return self._resolve_source_manifest(job) + + def materialize_source_member( + self, + *, + job_id: str, + member_id: str, + claim_token: str, + claim_generation: int, + claimant_id: str, + ) -> tuple[PortableSourceMember, Path]: + _digest(member_id, "source member id") + job = self._authorize( + job_id=job_id, + claim_token=claim_token, + claim_generation=claim_generation, + claimant_id=claimant_id, + allowed_states=("claimed", "running"), + ) + manifest = self._resolve_source_manifest(job) + matches = tuple(member for member in manifest.members if member.member_id == member_id) + if len(matches) != 1: + raise PortableArtifactTransportUnavailableError( + "source materialization member is not admitted" + ) + member = matches[0] + assert member._source_path is not None + destination = _publish_exact_cas_member( + self._source_cas, + member._source_path, + expected_sha256=member.sha256, + expected_byte_length=member.byte_length, + ) + self._authorize( + job_id=job_id, + claim_token=claim_token, + claim_generation=claim_generation, + claimant_id=claimant_id, + allowed_states=("claimed", "running"), + ) + return member, destination + + def stage_result_manifest( + self, + *, + job_id: str, + result_sha256: str, + manifest_payload: bytes, + claim_token: str, + claim_generation: int, + claimant_id: str, + ) -> PortableResultUploadPlan: + _digest(result_sha256, "portable result sha256") + if not 0 < len(manifest_payload) <= MAX_RESULT_MANIFEST_BYTES: + raise PortableArtifactTransportIntegrityError( + "portable result manifest is outside transport bounds" + ) + job = self._authorize( + job_id=job_id, + claim_token=claim_token, + claim_generation=claim_generation, + claimant_id=claimant_id, + allowed_states=("running",), + ) + try: + package = PortableResultPackageManifest.from_bytes(manifest_payload) + definition = self._definition(job) + result_id = string(package.result.get("result_id"), "portable result id") + except (PortableResultPackageIntegrityError, ValueError) as exc: + raise PortableArtifactTransportIntegrityError( + "portable result manifest is invalid" + ) from exc + _session_id(result_id) + if ( + hashlib.sha256(manifest_payload).hexdigest() != result_sha256 + or package.manifest_sha256 != result_sha256 + or package.job != job_identity_document(job) + or package.source != source_identity_document(job) + or package.run_definition != run_definition_document(definition) + or package.result != result_identity_document(definition, result_id) + or package.authority != OBSERVATION_ONLY_AUTHORITY + ): + raise PortableArtifactTransportIntegrityError( + "portable result manifest belongs to another sealed job" + ) + total_bytes = sum(artifact.byte_length for artifact in package.artifacts) + if ( + total_bytes > MAX_RESULT_PACKAGE_BYTES + or any( + artifact.byte_length > MAX_RESULT_MEMBER_BYTES + for artifact in package.artifacts + ) + ): + raise PortableArtifactTransportIntegrityError( + "portable result package exceeds transport bounds" + ) + root = self._package_root(job, result_sha256) + _prepare_secure_root(root) + binding = _ResultUploadBinding( + job_id=job.job_id, + job_identity_sha256=job.identity_sha256, + claim_generation=job.claim_generation, + claim_token_sha256=_token_sha256(claim_token), + result_id=result_id, + result_sha256=result_sha256, + package_identity_sha256=package.identity_sha256, + ) + _write_immutable_file( + self._binding_path(job, result_sha256), + canonical_json(binding.as_dict()), + ) + _write_immutable_file(root / "manifest.json", manifest_payload) + return self._result_plan(job, binding, package) + + async def upload_result_member( + self, + *, + job_id: str, + result_sha256: str, + member_id: str, + chunks: AsyncIterable[bytes], + claim_token: str, + claim_generation: int, + claimant_id: str, + ) -> PortableResultUploadPlan: + _digest(result_sha256, "portable result sha256") + _digest(member_id, "portable result member id") + job = self._authorize( + job_id=job_id, + claim_token=claim_token, + claim_generation=claim_generation, + claimant_id=claimant_id, + allowed_states=("running",), + ) + binding, package = self._load_result_upload( + job, + result_sha256, + claim_token=claim_token, + ) + expected = _result_upload_members(package) + matches = tuple(item for item in expected if item[0] == member_id) + if len(matches) != 1: + raise PortableArtifactTransportUnavailableError( + "portable result member is not declared by the manifest" + ) + _member_id, artifact = matches[0] + temporary_root = _prepare_secure_root(self._generation_root(job) / ".incoming") + temporary = temporary_root / f"upload-{secrets.token_hex(16)}" + descriptor = os.open( + temporary, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + digest = hashlib.sha256() + byte_length = 0 + try: + with os.fdopen(descriptor, "wb") as stream: + async for chunk in chunks: + if not isinstance(chunk, bytes): + raise PortableArtifactTransportIntegrityError( + "portable result upload emitted a non-byte chunk" + ) + byte_length += len(chunk) + if byte_length > artifact.byte_length: + raise PortableArtifactTransportIntegrityError( + "portable result upload exceeds its declared length" + ) + digest.update(chunk) + stream.write(chunk) + stream.flush() + os.fsync(stream.fileno()) + if ( + byte_length != artifact.byte_length + or digest.hexdigest() != artifact.sha256 + ): + raise PortableArtifactTransportIntegrityError( + "portable result upload differs from its manifest" + ) + self._authorize( + job_id=job_id, + claim_token=claim_token, + claim_generation=claim_generation, + claimant_id=claimant_id, + allowed_states=("running",), + ) + destination = _result_artifact_path( + self._package_root(job, result_sha256), artifact + ) + _publish_uploaded_file( + temporary, + destination, + expected_sha256=artifact.sha256, + expected_byte_length=artifact.byte_length, + ) + finally: + with suppress(FileNotFoundError): + temporary.unlink() + return self._result_plan(job, binding, package) + + def complete_result_upload( + self, + *, + job_id: str, + result_sha256: str, + claim_token: str, + claim_generation: int, + claimant_id: str, + ) -> PortableResultUploadReceipt: + _digest(result_sha256, "portable result sha256") + job = self._authorize( + job_id=job_id, + claim_token=claim_token, + claim_generation=claim_generation, + claimant_id=claimant_id, + allowed_states=("running",), + ) + binding, package = self._load_result_upload( + job, + result_sha256, + claim_token=claim_token, + ) + plan = self._result_plan(job, binding, package) + if not plan.complete: + raise PortableArtifactTransportUnavailableError( + "portable result package is incomplete" + ) + receipt = PortableResultUploadReceipt( + job_id=job.job_id, + job_identity_sha256=job.identity_sha256, + claim_generation=job.claim_generation, + claim_token_sha256=_token_sha256(claim_token), + result_id=binding.result_id, + result_sha256=result_sha256, + package_identity_sha256=package.identity_sha256, + member_count=len(package.artifacts), + total_bytes=sum(artifact.byte_length for artifact in package.artifacts), + ) + _write_immutable_file( + self._receipt_path(job, result_sha256), + canonical_json(receipt.as_dict()), + ) + return receipt + + def require_completed_for_success( + self, + *, + job_id: str, + result_id: str, + result_sha256: str, + claim_token: str, + claimant_id: str, + ) -> Path: + job = self._queue.get(job_id) + if job.active_claim_token is None: + if ( + job.state != "succeeded" + or job.result_id != result_id + or job.result_sha256 != result_sha256 + or job.terminal_claim_token_sha256 != _token_sha256(claim_token) + ): + raise PortableArtifactTransportUnavailableError( + "portable result success does not match a completed upload" + ) + else: + self._authorize( + job_id=job_id, + claim_token=claim_token, + claim_generation=job.claim_generation, + claimant_id=claimant_id, + allowed_states=("running",), + ) + receipt = self._read_receipt(job, result_sha256) + if ( + receipt.result_id != result_id + or receipt.result_sha256 != result_sha256 + or receipt.claim_token_sha256 != _token_sha256(claim_token) + ): + raise PortableArtifactTransportIntegrityError( + "portable result completion receipt changed" + ) + return self._package_root(job, result_sha256) + + def package_root_for_terminal(self, job: ObservatoryRecordedJob) -> Path: + if ( + job.state != "succeeded" + or job.result_id is None + or job.result_sha256 is None + or job.terminal_claim_token_sha256 is None + ): + raise PortableArtifactTransportUnavailableError( + "portable result job is not terminal" + ) + receipt = self._read_receipt(job, job.result_sha256) + if ( + receipt.result_id != job.result_id + or receipt.claim_token_sha256 != job.terminal_claim_token_sha256 + ): + raise PortableArtifactTransportIntegrityError( + "portable result terminal receipt is inconsistent" + ) + return self._package_root(job, job.result_sha256) + + def _authorize( + self, + *, + job_id: str, + claim_token: str, + claim_generation: int, + claimant_id: str, + allowed_states: tuple[Literal["claimed", "running"], ...], + ) -> ObservatoryRecordedJob: + _job_id(job_id) + _positive_int(claim_generation, "claim generation") + if _CLAIMANT_ID.fullmatch(claimant_id) is None: + raise ValueError("claimant id is invalid") + return self._queue.authorize_claim_access( + job_id, + claim_token=claim_token, + claim_generation=claim_generation, + claimant_id=claimant_id, + allowed_states=allowed_states, + ) + + def _definition(self, job: ObservatoryRecordedJob) -> PortableRunDefinition: + try: + definition = self._definitions.resolve(job.setup_id, job.definition_sha256) + recorded = definition.to_recorded_run_definition() + except (PortableRunDefinitionRegistryError, ValueError) as exc: + raise PortableArtifactTransportIntegrityError( + "portable job RunDefinition is unavailable" + ) from exc + if recorded != self._queue.resolve_definition( + job.setup_id, job.definition_sha256 + ): + raise PortableArtifactTransportIntegrityError( + "portable RunDefinition differs from the queue allowlist" + ) + return definition + + def _resolve_source_manifest( + self, + job: ObservatoryRecordedJob, + ) -> PortableSourceMaterializationManifest: + self._definition(job) + bundle_path, bundle = _read_source_document( + self._source_documents, job.source_bundle_sha256 + ) + capability_path, capability = _read_source_document( + self._source_documents, + job.source_capability_manifest_sha256, + ) + if ( + bundle.get("schema_version") != PORTABLE_SOURCE_BUNDLE_SCHEMA + or bundle.get("source_session_id") != job.source_session_id + or bundle.get("source_catalog_sha256") != job.source_catalog_sha256 + or bundle.get("source_adapter") + != { + "id": job.source_adapter_id, + "version": job.source_adapter_version, + "sha256": job.source_adapter_sha256, + } + or capability.get("schema_version") != PORTABLE_SOURCE_CAPABILITY_SCHEMA + or capability.get("source_session_id") != job.source_session_id + or capability.get("source_catalog_sha256") != job.source_catalog_sha256 + or capability.get("source_bundle_sha256") != job.source_bundle_sha256 + or capability.get("source_adapter_sha256") != job.source_adapter_sha256 + or bundle.get("authority") != OBSERVATION_ONLY_AUTHORITY + or capability.get("authority") != OBSERVATION_ONLY_AUTHORITY + ): + raise PortableArtifactTransportIntegrityError( + "portable source documents disagree with the sealed job" + ) + try: + detail, catalog_sha256 = ( + self._session_store.get_session_with_catalog_snapshot( + job.source_session_id + ) + ) + replay = self._session_store.prepare_replay(job.source_session_id) + recorded_media = self._session_store.list_recorded_media( + job.source_session_id + ) + except Exception as exc: + raise PortableArtifactTransportUnavailableError( + "portable source session is unavailable" + ) from exc + if ( + detail.summary.session_id != job.source_session_id + or detail.summary.lab is not None + or catalog_sha256 != job.source_catalog_sha256 + or replay.session_id != job.source_session_id + ): + raise PortableArtifactTransportIntegrityError( + "portable source catalog changed after admission" + ) + members: list[PortableSourceMember] = [ + _source_member( + job, + kind="source-bundle", + path=bundle_path, + media_type="application/json", + byte_length=bundle_path.stat().st_size, + sha256=job.source_bundle_sha256, + ), + _source_member( + job, + kind="source-capability", + path=capability_path, + media_type="application/json", + byte_length=capability_path.stat().st_size, + sha256=job.source_capability_manifest_sha256, + ), + ] + members.extend(_spatial_members(job, replay, bundle)) + members.extend( + self._camera_members( + job=job, + replay=replay, + bundle=bundle, + recorded_media=recorded_media, + ) + ) + return PortableSourceMaterializationManifest( + job_id=job.job_id, + job_identity_sha256=job.identity_sha256, + claim_generation=job.claim_generation, + source_session_id=job.source_session_id, + source_bundle_sha256=job.source_bundle_sha256, + source_capability_manifest_sha256=( + job.source_capability_manifest_sha256 + ), + members=tuple(sorted(members, key=lambda member: member.member_id)), + ) + + def _camera_members( + self, + *, + job: ObservatoryRecordedJob, + replay: ReplayCommand, + bundle: Mapping[str, object], + recorded_media: Sequence[object], + ) -> tuple[PortableSourceMember, ...]: + camera = object_document(bundle.get("camera"), "portable source camera") + artifact_id = string(camera.get("artifact_id"), "camera artifact id") + matches = tuple( + item + for item in recorded_media + if getattr(item, "artifact_id", None) == artifact_id + and getattr(item, "session_id", None) == job.source_session_id + ) + if len(matches) != 1: + raise PortableArtifactTransportIntegrityError( + "portable camera artifact is not unique" + ) + media_artifact = matches[0] + try: + manifest = self._media_inspector.restore_prepared( + media_artifact, # type: ignore[arg-type] + replay, + ) + except Exception as exc: + raise PortableArtifactTransportUnavailableError( + "portable camera preparation is unavailable" + ) from exc + if manifest is None: + raise PortableArtifactTransportUnavailableError( + "portable camera preparation is unavailable" + ) + return _camera_source_members(job, manifest, camera) + + def _generation_root(self, job: ObservatoryRecordedJob) -> Path: + return ( + self._result_staging + / job.job_id + / f"generation-{job.claim_generation}" + ) + + def _package_root(self, job: ObservatoryRecordedJob, result_sha256: str) -> Path: + _digest(result_sha256, "portable result sha256") + return self._generation_root(job) / "packages" / result_sha256 + + def _binding_path(self, job: ObservatoryRecordedJob, result_sha256: str) -> Path: + root = _prepare_secure_root(self._generation_root(job) / "bindings") + return root / f"{result_sha256}.json" + + def _receipt_path(self, job: ObservatoryRecordedJob, result_sha256: str) -> Path: + root = _prepare_secure_root(self._generation_root(job) / "receipts") + return root / f"{result_sha256}.json" + + def _load_result_upload( + self, + job: ObservatoryRecordedJob, + result_sha256: str, + *, + claim_token: str, + ) -> tuple[_ResultUploadBinding, PortableResultPackageManifest]: + binding = _read_upload_binding(self._binding_path(job, result_sha256)) + if ( + binding.job_id != job.job_id + or binding.job_identity_sha256 != job.identity_sha256 + or binding.claim_generation != job.claim_generation + or binding.claim_token_sha256 != _token_sha256(claim_token) + or binding.result_sha256 != result_sha256 + ): + raise PortableArtifactTransportIntegrityError( + "portable result upload binding is stale" + ) + manifest_path = self._package_root(job, result_sha256) / "manifest.json" + try: + payload = _read_exact_regular_file( + manifest_path, + maximum_bytes=MAX_RESULT_MANIFEST_BYTES, + ) + package = PortableResultPackageManifest.from_bytes(payload) + except (OSError, PortableResultPackageIntegrityError) as exc: + raise PortableArtifactTransportIntegrityError( + "portable result staged manifest changed" + ) from exc + if ( + hashlib.sha256(payload).hexdigest() != result_sha256 + or package.identity_sha256 != binding.package_identity_sha256 + or package.job != job_identity_document(job) + ): + raise PortableArtifactTransportIntegrityError( + "portable result staged manifest identity changed" + ) + return binding, package + + def _result_plan( + self, + job: ObservatoryRecordedJob, + binding: _ResultUploadBinding, + package: PortableResultPackageManifest, + ) -> PortableResultUploadPlan: + root = self._package_root(job, binding.result_sha256) + members = tuple( + PortableResultUploadMember( + member_id=member_id, + role=artifact.role, + media_type=artifact.media_type, + byte_length=artifact.byte_length, + sha256=artifact.sha256, + uploaded=_matches_exact_result_artifact(root, artifact), + ) + for member_id, artifact in _result_upload_members(package) + ) + return PortableResultUploadPlan( + job_id=job.job_id, + claim_generation=job.claim_generation, + result_id=binding.result_id, + result_sha256=binding.result_sha256, + package_identity_sha256=binding.package_identity_sha256, + members=members, + complete=all(member.uploaded for member in members), + ) + + def _read_receipt( + self, + job: ObservatoryRecordedJob, + result_sha256: str, + ) -> PortableResultUploadReceipt: + path = self._receipt_path(job, result_sha256) + try: + payload = _read_exact_regular_file(path, maximum_bytes=16 * 1024) + persisted = object_document( + json.loads(payload.decode("utf-8")), + "portable result completion receipt", + ) + if canonical_json(persisted) != payload or set(persisted) != { + "schema_version", + "job_id", + "job_identity_sha256", + "claim_generation", + "claim_token_sha256", + "result_id", + "result_sha256", + "package_identity_sha256", + "member_count", + "total_bytes", + "authority", + "receipt_sha256", + }: + raise ValueError("receipt document is not exact canonical JSON") + document = dict(persisted) + receipt_sha256 = string( + document.pop("receipt_sha256", None), + "portable result completion receipt sha256", + ) + if document.get("schema_version") != PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA: + raise ValueError("receipt schema is invalid") + receipt = PortableResultUploadReceipt( + job_id=string(document.get("job_id"), "receipt job id"), + job_identity_sha256=string( + document.get("job_identity_sha256"), "receipt job identity" + ), + claim_generation=_integer( + document.get("claim_generation"), "receipt claim generation" + ), + claim_token_sha256=string( + document.get("claim_token_sha256"), "receipt claim token" + ), + result_id=string(document.get("result_id"), "receipt result id"), + result_sha256=string( + document.get("result_sha256"), "receipt result sha256" + ), + package_identity_sha256=string( + document.get("package_identity_sha256"), + "receipt package identity", + ), + member_count=_integer( + document.get("member_count"), "receipt member count" + ), + total_bytes=_integer(document.get("total_bytes"), "receipt total bytes"), + ) + except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError) as exc: + raise PortableArtifactTransportUnavailableError( + "portable result completion receipt is unavailable" + ) from exc + if ( + receipt.receipt_sha256 != receipt_sha256 + or receipt.job_id != job.job_id + or receipt.job_identity_sha256 != job.identity_sha256 + or receipt.claim_generation != job.claim_generation + or receipt.result_sha256 != result_sha256 + or document.get("authority") != OBSERVATION_ONLY_AUTHORITY + ): + raise PortableArtifactTransportIntegrityError( + "portable result completion receipt changed" + ) + return receipt + + +def _source_member( + job: ObservatoryRecordedJob, + *, + kind: SourceMemberKind, + path: Path, + media_type: str, + byte_length: int, + sha256: str, + artifact_id: str | None = None, + primary: bool = False, + camera_epoch: int | None = None, + camera_sequence: int | None = None, +) -> PortableSourceMember: + identity = { + "job_identity_sha256": job.identity_sha256, + "source_bundle_sha256": job.source_bundle_sha256, + "kind": kind, + "artifact_id": artifact_id, + "primary": primary, + "camera_epoch": camera_epoch, + "camera_sequence": camera_sequence, + "media_type": media_type, + "byte_length": byte_length, + "sha256": sha256, + } + return PortableSourceMember( + member_id=hashlib.sha256(canonical_json(identity)).hexdigest(), + kind=kind, + media_type=media_type, + byte_length=byte_length, + sha256=sha256, + artifact_id=artifact_id, + primary=primary, + camera_epoch=camera_epoch, + camera_sequence=camera_sequence, + _source_path=path, + ) + + +def _spatial_members( + job: ObservatoryRecordedJob, + replay: ReplayCommand, + bundle: Mapping[str, object], +) -> tuple[PortableSourceMember, ...]: + spatial = object_document(bundle.get("spatial_replay"), "portable spatial replay") + rows = spatial.get("members") + if not isinstance(rows, list): + raise PortableArtifactTransportIntegrityError( + "portable spatial replay members are invalid" + ) + replay_by_id = {artifact.artifact_id: artifact for artifact in replay.artifacts} + if len(replay_by_id) != len(replay.artifacts): + raise PortableArtifactTransportIntegrityError( + "portable replay artifact identities are not unique" + ) + if spatial.get("primary_artifact_id") != replay.primary_artifact_id: + raise PortableArtifactTransportIntegrityError( + "portable replay primary artifact changed" + ) + result: list[PortableSourceMember] = [] + metadata_member_count = 0 + for value in rows: + row = object_document(value, "portable spatial replay member") + artifact_id = string(row.get("artifact_id"), "spatial replay artifact id") + artifact = replay_by_id.get(artifact_id) + if artifact is None: + raise PortableArtifactTransportIntegrityError( + "portable replay member is absent from SessionStore" + ) + sha256 = string(row.get("sha256"), "spatial replay artifact sha256") + _digest(sha256, "spatial replay artifact sha256") + is_metadata_member = ( + artifact_id == PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID + and artifact.media_type == PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE + ) + if artifact_id == PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID and not ( + is_metadata_member + ): + raise PortableArtifactTransportIntegrityError( + "portable replay metadata media type changed after admission" + ) + exact_digest = artifact.expected_sha256 + if is_metadata_member and exact_digest is None: + try: + actual_sha256, actual_bytes = _hash_regular_file( + artifact.path, + maximum_bytes=artifact.file_byte_length, + ) + except (OSError, PortableArtifactTransportError) as exc: + raise PortableArtifactTransportIntegrityError( + "portable replay metadata changed after admission" + ) from exc + if actual_bytes != artifact.file_byte_length: + raise PortableArtifactTransportIntegrityError( + "portable replay metadata changed after admission" + ) + exact_digest = actual_sha256 + if ( + row.get("media_type") != artifact.media_type + or row.get("byte_length") != artifact.file_byte_length + or row.get("replay_byte_length") != artifact.replay_byte_length + or sha256 != exact_digest + ): + raise PortableArtifactTransportIntegrityError( + "portable replay member changed after admission" + ) + kind: SourceMemberKind = "spatial-replay" + if is_metadata_member: + metadata_member_count += 1 + kind = "spatial-replay-metadata" + result.append( + _source_member( + job, + kind=kind, + path=artifact.path, + media_type=artifact.media_type, + byte_length=artifact.file_byte_length, + sha256=sha256, + artifact_id=artifact_id, + primary=artifact_id == replay.primary_artifact_id, + ) + ) + if {member.artifact_id for member in result} != set(replay_by_id): + raise PortableArtifactTransportIntegrityError( + "portable replay member inventory changed after admission" + ) + if metadata_member_count > 1: + raise PortableArtifactTransportIntegrityError( + "portable replay metadata member is not unique" + ) + return tuple(result) + + +def _camera_source_members( + job: ObservatoryRecordedJob, + manifest: RecordedMediaManifest, + camera: Mapping[str, object], +) -> tuple[PortableSourceMember, ...]: + epoch_document = object_document(camera.get("epoch"), "portable camera epoch") + if ( + manifest.session_id != job.source_session_id + or camera.get("artifact_id") != manifest.artifact_id + or camera.get("public_source_id") != manifest.public_source_id + or camera.get("generation_sha256") != manifest.generation_sha256 + or len(manifest.epochs) != 1 + ): + raise PortableArtifactTransportIntegrityError( + "portable camera manifest changed after admission" + ) + epoch = manifest.epochs[0] + init = object_document(epoch_document.get("init"), "portable camera init") + segments = epoch_document.get("segments") + if ( + epoch_document.get("ordinal") != epoch.ordinal + or epoch_document.get("media_type") != epoch.media_type + or init.get("byte_length") != epoch.init_byte_length + or init.get("sha256") != epoch.init_sha256 + or not isinstance(segments, list) + or len(segments) != len(epoch.segments) + ): + raise PortableArtifactTransportIntegrityError( + "portable camera epoch changed after admission" + ) + members = [ + _source_member( + job, + kind="camera-init", + path=epoch.init_path, + media_type=epoch.media_type, + byte_length=epoch.init_byte_length, + sha256=epoch.init_sha256, + artifact_id=manifest.artifact_id, + camera_epoch=epoch.ordinal, + ) + ] + for row_value, segment in zip(segments, epoch.segments, strict=True): + row = object_document(row_value, "portable camera segment") + if ( + row.get("sequence") != segment.sequence + or row.get("byte_length") != segment.byte_length + or row.get("sha256") != segment.sha256 + or row.get("random_access") != segment.random_access + or row.get("end_time_seconds") != segment.end_time_seconds + ): + raise PortableArtifactTransportIntegrityError( + "portable camera segment changed after admission" + ) + members.append( + _source_member( + job, + kind="camera-segment", + path=segment.path, + media_type="video/iso.segment", + byte_length=segment.byte_length, + sha256=segment.sha256, + artifact_id=manifest.artifact_id, + camera_epoch=epoch.ordinal, + camera_sequence=segment.sequence, + ) + ) + return tuple(members) + + +def _read_source_document( + root: Path, + sha256: str, +) -> tuple[Path, dict[str, object]]: + _digest(sha256, "portable source document sha256") + path = root / f"{sha256}.json" + try: + payload = _read_exact_regular_file(path, maximum_bytes=8 * 1024 * 1024) + if hashlib.sha256(payload).hexdigest() != sha256: + raise PortableArtifactTransportIntegrityError( + "portable source document digest changed" + ) + document = object_document( + json.loads(payload.decode("utf-8")), "portable source document" + ) + if canonical_json(document) != payload: + raise PortableArtifactTransportIntegrityError( + "portable source document is not canonical JSON" + ) + except PortableArtifactTransportError: + raise + except (OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc: + raise PortableArtifactTransportUnavailableError( + "portable source document is unavailable" + ) from exc + return path, document + + +def _result_upload_members( + package: PortableResultPackageManifest, +) -> tuple[tuple[str, PortableResultArtifact], ...]: + rows = tuple( + ( + hashlib.sha256( + canonical_json( + { + "package_identity_sha256": package.identity_sha256, + "artifact": artifact.as_dict(), + } + ) + ).hexdigest(), + artifact, + ) + for artifact in package.artifacts + ) + return tuple(sorted(rows, key=lambda row: row[0])) + + +def _result_artifact_path(root: Path, artifact: PortableResultArtifact) -> Path: + relative = relative_artifact_path(artifact.relative_path) + parent = root + for part in relative.parts[:-1]: + parent = _prepare_secure_root(parent / part) + return parent / relative.parts[-1] + + +def _matches_exact_result_artifact( + root: Path, + artifact: PortableResultArtifact, +) -> bool: + try: + path = _result_artifact_path(root, artifact) + digest, byte_length = _hash_regular_file( + path, + maximum_bytes=artifact.byte_length, + ) + return digest == artifact.sha256 and byte_length == artifact.byte_length + except (OSError, PortableArtifactTransportError, ValueError): + return False + + +def _read_upload_binding(path: Path) -> _ResultUploadBinding: + try: + payload = _read_exact_regular_file(path, maximum_bytes=16 * 1024) + document = object_document( + json.loads(payload.decode("utf-8")), "portable result upload binding" + ) + if canonical_json(document) != payload or set(document) != { + "schema_version", + "job_id", + "job_identity_sha256", + "claim_generation", + "claim_token_sha256", + "result_id", + "result_sha256", + "package_identity_sha256", + "authority", + }: + raise ValueError("upload binding is not canonical") + if ( + document.get("schema_version") != PORTABLE_RESULT_UPLOAD_BINDING_SCHEMA + or document.get("authority") != OBSERVATION_ONLY_AUTHORITY + ): + raise ValueError("upload binding schema is invalid") + binding = _ResultUploadBinding( + job_id=string(document.get("job_id"), "upload job id"), + job_identity_sha256=string( + document.get("job_identity_sha256"), "upload job identity" + ), + claim_generation=_integer( + document.get("claim_generation"), "upload claim generation" + ), + claim_token_sha256=string( + document.get("claim_token_sha256"), "upload claim token sha256" + ), + result_id=string(document.get("result_id"), "upload result id"), + result_sha256=string( + document.get("result_sha256"), "upload result sha256" + ), + package_identity_sha256=string( + document.get("package_identity_sha256"), + "upload package identity", + ), + ) + except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError) as exc: + raise PortableArtifactTransportUnavailableError( + "portable result upload has no manifest binding" + ) from exc + return binding + + +def _publish_exact_cas_member( + root: Path, + source: Path, + *, + expected_sha256: str, + expected_byte_length: int, +) -> Path: + _digest(expected_sha256, "CAS member sha256") + destination_root = _prepare_secure_root(root / expected_sha256[:2]) + destination = destination_root / expected_sha256 + if _matches_exact_file( + destination, + expected_sha256=expected_sha256, + expected_byte_length=expected_byte_length, + ): + return destination + temporary = destination_root / f".tmp-{secrets.token_hex(16)}" + _copy_exact_file( + source, + temporary, + expected_sha256=expected_sha256, + expected_byte_length=expected_byte_length, + ) + try: + _publish_uploaded_file( + temporary, + destination, + expected_sha256=expected_sha256, + expected_byte_length=expected_byte_length, + ) + finally: + with suppress(FileNotFoundError): + temporary.unlink() + return destination + + +def _copy_exact_file( + source: Path, + destination: Path, + *, + expected_sha256: str, + expected_byte_length: int, +) -> None: + source_descriptor = -1 + destination_descriptor = -1 + try: + source_descriptor = os.open( + source, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + before = os.fstat(source_descriptor) + if not stat.S_ISREG(before.st_mode) or before.st_size != expected_byte_length: + raise PortableArtifactTransportIntegrityError( + "source member is not the admitted regular file" + ) + destination_descriptor = os.open( + destination, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + digest = hashlib.sha256() + copied = 0 + while chunk := os.read(source_descriptor, COPY_CHUNK_BYTES): + copied += len(chunk) + if copied > expected_byte_length: + raise PortableArtifactTransportIntegrityError( + "source member grew during materialization" + ) + digest.update(chunk) + view = memoryview(chunk) + while view: + written = os.write(destination_descriptor, view) + view = view[written:] + os.fsync(destination_descriptor) + after = os.fstat(source_descriptor) + stable = ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + ) == ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ) + if ( + copied != expected_byte_length + or digest.hexdigest() != expected_sha256 + or not stable + ): + raise PortableArtifactTransportIntegrityError( + "source member changed during materialization" + ) + except PortableArtifactTransportError: + raise + except OSError as exc: + raise PortableArtifactTransportUnavailableError( + "source member could not be materialized" + ) from exc + finally: + if source_descriptor >= 0: + os.close(source_descriptor) + if destination_descriptor >= 0: + os.close(destination_descriptor) + if destination.exists() and not _matches_exact_file( + destination, + expected_sha256=expected_sha256, + expected_byte_length=expected_byte_length, + ): + with suppress(OSError): + destination.unlink() + + +def _publish_uploaded_file( + temporary: Path, + destination: Path, + *, + expected_sha256: str, + expected_byte_length: int, +) -> None: + if _matches_exact_file( + destination, + expected_sha256=expected_sha256, + expected_byte_length=expected_byte_length, + ): + return + if destination.exists(): + raise PortableArtifactTransportIntegrityError( + "content-addressed destination contains another payload" + ) + try: + os.link(temporary, destination, follow_symlinks=False) + os.chmod(destination, 0o400, follow_symlinks=False) + _fsync_directory(destination.parent) + except FileExistsError: + if not _matches_exact_file( + destination, + expected_sha256=expected_sha256, + expected_byte_length=expected_byte_length, + ): + raise PortableArtifactTransportIntegrityError( + "content-addressed publication collided" + ) from None + except OSError as exc: + raise PortableArtifactTransportUnavailableError( + "content-addressed member could not be published" + ) from exc + + +def _write_immutable_file(path: Path, payload: bytes) -> None: + parent = _prepare_secure_root(path.parent) + if path.exists(): + try: + if _read_exact_regular_file(path, maximum_bytes=max(1, len(payload))) == payload: + return + except OSError as exc: + raise PortableArtifactTransportIntegrityError( + "immutable transport document is unreadable" + ) from exc + raise PortableArtifactTransportIntegrityError( + "immutable transport document identity collided" + ) + temporary = parent / f".tmp-{secrets.token_hex(16)}" + descriptor = os.open( + temporary, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + try: + os.link(temporary, path, follow_symlinks=False) + except FileExistsError: + if _read_exact_regular_file(path, maximum_bytes=max(1, len(payload))) != payload: + raise PortableArtifactTransportIntegrityError( + "immutable transport document identity collided" + ) from None + os.chmod(path, 0o400, follow_symlinks=False) + _fsync_directory(parent) + finally: + with suppress(FileNotFoundError): + temporary.unlink() + + +def _prepare_secure_root(path: Path, *, create: bool = True) -> Path: + candidate = path.expanduser().absolute() + if create: + candidate.mkdir(mode=0o700, parents=True, exist_ok=True) + try: + metadata = candidate.lstat() + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise PortableArtifactTransportUnavailableError( + "artifact transport storage is unavailable" + ) from exc + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise PortableArtifactTransportIntegrityError( + "artifact transport storage root is unsafe" + ) + return resolved + + +def _read_exact_regular_file(path: Path, *, maximum_bytes: int) -> bytes: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size < 0 + or before.st_size > maximum_bytes + ): + raise PortableArtifactTransportIntegrityError( + "artifact transport file is outside bounds" + ) + payload = bytearray() + while len(payload) < before.st_size: + chunk = os.read(descriptor, min(COPY_CHUNK_BYTES, before.st_size - len(payload))) + if not chunk: + break + payload.extend(chunk) + after = os.fstat(descriptor) + if len(payload) != before.st_size or ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + ) != ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ): + raise PortableArtifactTransportIntegrityError( + "artifact transport file changed while read" + ) + return bytes(payload) + finally: + os.close(descriptor) + + +def _hash_regular_file(path: Path, *, maximum_bytes: int) -> tuple[str, int]: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size < 0 + or before.st_size > maximum_bytes + ): + raise PortableArtifactTransportIntegrityError( + "artifact transport member is outside bounds" + ) + digest = hashlib.sha256() + byte_length = 0 + while chunk := os.read(descriptor, COPY_CHUNK_BYTES): + byte_length += len(chunk) + digest.update(chunk) + after = os.fstat(descriptor) + if ( + byte_length != before.st_size + or ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + ) + != ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ) + ): + raise PortableArtifactTransportIntegrityError( + "artifact transport member changed while read" + ) + return digest.hexdigest(), byte_length + finally: + os.close(descriptor) + + +def _matches_exact_file( + path: Path, + *, + expected_sha256: str, + expected_byte_length: int, +) -> bool: + try: + sha256, byte_length = _hash_regular_file( + path, + maximum_bytes=expected_byte_length, + ) + return sha256 == expected_sha256 and byte_length == expected_byte_length + except (OSError, PortableArtifactTransportError): + return False + + +def _token_sha256(token: str) -> str: + if not isinstance(token, str) or re.fullmatch(r"^[a-f0-9]{64}$", token) is None: + raise ValueError("claim token is invalid") + return hashlib.sha256(token.encode("ascii")).hexdigest() + + +def _media_type(value: object) -> str: + if ( + not isinstance(value, str) + or not 3 <= len(value) <= 255 + or "/" not in value + or value != value.strip() + or any(ord(character) < 32 or ord(character) > 126 for character in value) + ): + raise ValueError("artifact media type is invalid") + return value + + +def _job_id(value: object) -> str: + if not isinstance(value, str) or _JOB_ID.fullmatch(value) is None: + raise ValueError("recorded job id is invalid") + return value + + +def _session_id(value: object) -> str: + if not isinstance(value, str) or _SESSION_ID.fullmatch(value) is None: + raise ValueError("session id is invalid") + return value + + +def _digest(value: object, label: str) -> str: + if not isinstance(value, str) or _SHA256.fullmatch(value) is None: + raise ValueError(f"{label} is invalid") + return value + + +def _positive_int(value: object, label: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise ValueError(f"{label} is invalid") + return value + + +def _integer(value: object, label: str) -> int: + if not isinstance(value, int) or isinstance(value, bool): + raise ValueError(f"{label} is invalid") + return value + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) diff --git a/src/k1link/observatory/portable_lab_v1_executor.py b/src/k1link/observatory/portable_lab_v1_executor.py new file mode 100644 index 0000000..5dd561f --- /dev/null +++ b/src/k1link/observatory/portable_lab_v1_executor.py @@ -0,0 +1,2790 @@ +"""Fail-closed portable LAB V1 executor contracts. + +The historical EoMT and DDRNet runners are useful implementation assets, but +their old wrappers are not a portable executor release. This module provides +the source-independent boundary around those assets: + +* an admitted K1 camera compute job is bound to the exact Observatory claim; +* a path-free, digest-fenced sequential orchestration plan is produced; +* legacy EoMT and DDRNet outputs are assembled into the v2 portable result; +* the exact v2 result validator can be registered with the portable publisher; +* a release candidate can be inspected without pretending that missing model, + image, calibration, or dependency artifacts are installed. + +Nothing in this module grants command, navigation, safety, or actuation +authority. It deliberately contains no default executable command and cannot +turn a blocked release candidate into a Worker registration. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import secrets +import shutil +import stat +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Final, Literal, cast + +from k1link.compute.jobs import CameraComputeJob, validate_camera_compute_job +from k1link.observatory.portable_result_contract import ( + OBSERVATION_ONLY_AUTHORITY, + PortableResultArtifact, + PortableResultPackageIntegrityError, + PortableResultPackageManifest, + PortableResultValidationContext, + canonical_json, +) +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinition, + canonical_sha256, +) +from k1link.observatory.portable_worker_runtime import PortableWorkerResultDraft +from k1link.observatory.recorded_jobs import ObservatoryRecordedJob +from k1link.observatory.source_admission import ( + PORTABLE_SOURCE_BUNDLE_SCHEMA, + PORTABLE_SOURCE_CAPABILITY_SCHEMA, +) +from k1link.observatory.worker_agent import SealedObservatoryRecordedJob + +PORTABLE_LAB_V1_SOURCE_SCHEMA: Final = ( + "missioncore.observatory-portable-lab-v1-source/v1" +) +PORTABLE_LAB_V1_PLAN_SCHEMA: Final = ( + "missioncore.observatory-portable-lab-v1-orchestration-plan/v1" +) +PORTABLE_LAB_V1_PLAN_IDENTITY_SCHEMA: Final = ( + "missioncore.observatory-portable-lab-v1-orchestration-plan-identity/v1" +) +PORTABLE_LAB_V1_RESULT_SCHEMA: Final = "missioncore.recorded-eomt-ddrnet-review/v2" +PORTABLE_LAB_V1_RESULT_IDENTITY_SCHEMA: Final = ( + "missioncore.recorded-eomt-ddrnet-review-identity/v2" +) +PORTABLE_LAB_V1_RELEASE_SCHEMA: Final = ( + "missioncore.observatory-portable-lab-v1-executor-candidate/v1" +) +PORTABLE_LAB_V1_RELEASE_IDENTITY_SCHEMA: Final = ( + "missioncore.observatory-portable-lab-v1-executor-candidate-identity/v1" +) +PORTABLE_LAB_V1_EXECUTOR_SEAL_SCHEMA: Final = ( + "missioncore.observatory-portable-lab-v1-executor-seal/v1" +) +PORTABLE_LAB_V1_DDRNET_PROFILE_SCHEMA: Final = ( + "missioncore.lab-v1-eomt-ddrnet-portable-profile/v2" +) +PORTABLE_LAB_V1_DDRNET_EFFECTIVE_CONFIG_SCHEMA: Final = ( + "missioncore.lab-v1-goose-vegetation-benchmark/v1" +) +EOMT_RESULT_SCHEMA: Final = "missioncore.recorded-perception-result/v2" +DDRNET_RESULT_SCHEMA: Final = "missioncore.lab-v1-goose-vegetation-run/v1" +DECODE_REPAIR_SCHEMA: Final = "missioncore.recorded-video-decode-repair/v1" + +_EXPECTED_SETUP_ID: Final = "lab-v1-eomt-ddrnet-portable-v1" +_EXPECTED_DEFINITION_ID: Final = "lab-v1-eomt-ddrnet-portable" +_EXPECTED_RESULT_CONTRACT_SHA256: Final = ( + "b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a" +) +_EOMT_RELEASE_ID: Final = "eomt-cityscapes-large-1024-v1" +_DDRNET_RELEASE_ID: Final = "lab-v1-ddrnet-39-goose-fine-64-v1" +_EOMT_PIPELINE: Final = "recorded-semantic-eomt-fisheye-mask/v1" +_DDRNET_CANDIDATE_ID: Final = "goose-ddrnet-class-512" +_DDRNET_CANDIDATE_KEY: Final = "ddrnet" +_DDRNET_CHECKPOINT_SHA256: Final = ( + "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6" +) +_GOOSE_MAPPING_SHA256: Final = ( + "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f" +) +_OBSERVATORY_JOB_ID = re.compile(r"^observatory-run-[a-f0-9]{32}$") +_CAMERA_JOB_ID = re.compile(r"^recorded-camera-[a-f0-9]{24}$") +_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$") +_ASSET_ID = re.compile(r"^[a-z][a-z0-9.-]{2,127}$") +_ROLE = re.compile(r"^[a-z][a-z0-9-]{2,95}$") +_SESSION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_MAX_DOCUMENT_BYTES: Final = 8 * 1024 * 1024 +_MAX_RELEASE_BYTES: Final = 1024 * 1024 +_MAX_ARTIFACTS: Final = 64 + +type ReleaseAssetKind = Literal[ + "container-image", + "definition-component", + "model-artifact", + "repository-file", + "runtime-artifact", +] +type ReleaseAssetBinding = Path | str + + +class PortableLabV1Error(RuntimeError): + """Base error for the portable LAB V1 boundary.""" + + +class PortableLabV1SourceError(PortableLabV1Error): + """The local source stage differs from its sealed K1 identity.""" + + +class PortableLabV1PlanError(PortableLabV1Error): + """The combined orchestration plan is malformed or not executable.""" + + +class PortableLabV1ResultError(PortableResultPackageIntegrityError): + """A component or assembled LAB V1 result violates the v2 contract.""" + + +class PortableLabV1ReleaseError(PortableLabV1Error): + """The exact portable executor release is unavailable or malformed.""" + + +@dataclass(frozen=True, slots=True) +class PortableLabV1SourceInput: + """Path-free identity of one materialized, validated camera compute job.""" + + observatory_job_id: str + observatory_request_sha256: str + observatory_identity_sha256: str + source_session_id: str + source_catalog_sha256: str + source_bundle_sha256: str + source_capability_manifest_sha256: str + source_adapter_sha256: str + camera_job_id: str + camera_input_sha256: str + camera_source_id: str + codec_epoch: int + input_byte_length: int + frame_count: int + timeline_start_seconds: float + timeline_end_seconds: float + camera_generation_sha256: str + calibration_sha256: str + + def __post_init__(self) -> None: + _pattern(self.observatory_job_id, _OBSERVATORY_JOB_ID, "Observatory job id") + _pattern(self.camera_job_id, _CAMERA_JOB_ID, "camera compute job id") + _pattern(self.source_session_id, _SESSION_ID, "source session id") + _pattern(self.camera_source_id, _SESSION_ID, "camera source id") + for value, label in ( + (self.observatory_request_sha256, "Observatory request sha256"), + (self.observatory_identity_sha256, "Observatory identity sha256"), + (self.source_catalog_sha256, "source catalog sha256"), + (self.source_bundle_sha256, "source bundle sha256"), + ( + self.source_capability_manifest_sha256, + "source capability manifest sha256", + ), + (self.source_adapter_sha256, "source adapter sha256"), + (self.camera_input_sha256, "camera input sha256"), + (self.camera_generation_sha256, "camera generation sha256"), + (self.calibration_sha256, "calibration sha256"), + ): + _digest(value, label) + for integer_value, label in ( + (self.codec_epoch, "codec epoch"), + (self.input_byte_length, "camera input byte length"), + (self.frame_count, "camera frame count"), + ): + if ( + isinstance(integer_value, bool) + or not isinstance(integer_value, int) + or integer_value < 1 + ): + raise PortableLabV1SourceError(f"{label} is invalid") + if ( + not _finite_number(self.timeline_start_seconds) + or not _finite_number(self.timeline_end_seconds) + or self.timeline_end_seconds <= self.timeline_start_seconds + ): + raise PortableLabV1SourceError("camera timeline is invalid") + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": PORTABLE_LAB_V1_SOURCE_SCHEMA, + "observatory_job": { + "job_id": self.observatory_job_id, + "request_sha256": self.observatory_request_sha256, + "identity_sha256": self.observatory_identity_sha256, + }, + "source": { + "session_id": self.source_session_id, + "catalog_sha256": self.source_catalog_sha256, + "bundle_sha256": self.source_bundle_sha256, + "capability_manifest_sha256": ( + self.source_capability_manifest_sha256 + ), + "adapter_sha256": self.source_adapter_sha256, + }, + "camera_compute_job": { + "job_id": self.camera_job_id, + "input_sha256": self.camera_input_sha256, + "source_id": self.camera_source_id, + "codec_epoch": self.codec_epoch, + "input_byte_length": self.input_byte_length, + "frame_count": self.frame_count, + "timeline_start_seconds": self.timeline_start_seconds, + "timeline_end_seconds": self.timeline_end_seconds, + "generation_sha256": self.camera_generation_sha256, + "calibration_sha256": self.calibration_sha256, + }, + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + + @property + def identity_sha256(self) -> str: + return canonical_sha256(self.as_dict()) + + @property + def canonical_bytes(self) -> bytes: + return canonical_json(self.as_dict()) + + +@dataclass(frozen=True, slots=True) +class PortableLabV1MaterializedSource: + """Validated local paths plus the path-free source descriptor.""" + + root: Path + camera_job_root: Path + descriptor: PortableLabV1SourceInput + + def __post_init__(self) -> None: + root = _real_directory(self.root, "portable LAB V1 source root") + camera = _real_directory(self.camera_job_root, "portable LAB V1 camera job") + if not camera.is_relative_to(root): + raise PortableLabV1SourceError("camera job escapes its source stage") + + +@dataclass(frozen=True, slots=True) +class PortableLabV1SourceMaterializer: + """Resolve one server-sealed stage from a digest-owned local inbox. + + The transport owns the copy into ``staging_root``. This materializer never + accepts a path from the queued job. It selects only + ``/`` and validates every identity before + producing the source contract. + """ + + staging_root: Path + definition: PortableRunDefinition + + def materialize( + self, + job: SealedObservatoryRecordedJob, + ) -> PortableLabV1MaterializedSource: + _verify_definition_and_job(self.definition, job) + root = _real_directory(self.staging_root, "portable source staging root") + stage = _real_directory( + root / job.source_bundle_sha256, + "portable source digest stage", + ) + if stage.parent != root: + raise PortableLabV1SourceError("source stage is not a direct digest child") + source_bundle_bytes = _read_bounded_regular_file( + stage / "source-bundle.json", + stage, + "source bundle", + ) + capability_bytes = _read_bounded_regular_file( + stage / "source-capability.json", + stage, + "source capability", + ) + camera_jobs = _direct_real_directories(stage / "camera-job") + if len(camera_jobs) != 1: + raise PortableLabV1SourceError( + "portable source stage requires exactly one camera compute job" + ) + try: + camera_job = validate_camera_compute_job(camera_jobs[0]) + except Exception as exc: + raise PortableLabV1SourceError( + "portable camera compute job failed exact validation" + ) from exc + descriptor = materialize_lab_v1_source_input( + job=job, + definition=self.definition, + camera_job=camera_job, + source_bundle_bytes=source_bundle_bytes, + capability_bytes=capability_bytes, + ) + return PortableLabV1MaterializedSource( + root=stage, + camera_job_root=camera_job.job_root, + descriptor=descriptor, + ) + + +def materialize_lab_v1_source_input( + *, + job: SealedObservatoryRecordedJob, + definition: PortableRunDefinition, + camera_job: CameraComputeJob, + source_bundle_bytes: bytes, + capability_bytes: bytes, +) -> PortableLabV1SourceInput: + """Validate and bind persisted source documents to a camera compute job.""" + + _verify_definition_and_job(definition, job) + source_bundle = _canonical_document( + source_bundle_bytes, + "source bundle", + maximum=_MAX_DOCUMENT_BYTES, + ) + capability = _canonical_document( + capability_bytes, + "source capability", + maximum=_MAX_DOCUMENT_BYTES, + ) + if hashlib.sha256(source_bundle_bytes).hexdigest() != job.source_bundle_sha256: + raise PortableLabV1SourceError("source bundle digest differs from the sealed job") + if ( + hashlib.sha256(capability_bytes).hexdigest() + != job.source_capability_manifest_sha256 + ): + raise PortableLabV1SourceError( + "source capability digest differs from the sealed job" + ) + _validate_source_documents( + source_bundle=source_bundle, + capability=capability, + job=job, + definition=definition, + camera_job=camera_job, + ) + camera = _object(source_bundle.get("camera"), "source bundle camera") + calibration = _object( + capability.get("calibration"), + "source capability calibration", + ) + return PortableLabV1SourceInput( + observatory_job_id=job.job_id, + observatory_request_sha256=job.request_sha256, + observatory_identity_sha256=job.identity_sha256, + source_session_id=job.source_session_id, + source_catalog_sha256=job.source_catalog_sha256, + source_bundle_sha256=job.source_bundle_sha256, + source_capability_manifest_sha256=job.source_capability_manifest_sha256, + source_adapter_sha256=job.source_adapter_sha256, + camera_job_id=camera_job.job_id, + camera_input_sha256=camera_job.input_sha256, + camera_source_id=camera_job.source_id, + codec_epoch=camera_job.codec_epoch, + input_byte_length=camera_job.input_byte_length, + frame_count=camera_job.segment_count, + timeline_start_seconds=camera_job.timeline_start_seconds, + timeline_end_seconds=camera_job.timeline_end_seconds, + camera_generation_sha256=_string( + camera.get("generation_sha256"), + "source camera generation sha256", + ), + calibration_sha256=_string( + calibration.get("sha256"), + "source calibration sha256", + ), + ) + + +@dataclass(frozen=True, slots=True) +class PortableLabV1ReleaseAsset: + asset_id: str + kind: ReleaseAssetKind + sha256: str + byte_length: int | None + repository_path: str | None + + def __post_init__(self) -> None: + _pattern(self.asset_id, _ASSET_ID, "release asset id") + if self.kind not in ( + "container-image", + "definition-component", + "model-artifact", + "repository-file", + "runtime-artifact", + ): + raise PortableLabV1ReleaseError("release asset kind is invalid") + _digest(self.sha256, "release asset sha256") + if self.byte_length is not None and ( + isinstance(self.byte_length, bool) + or not isinstance(self.byte_length, int) + or self.byte_length < 1 + ): + raise PortableLabV1ReleaseError("release asset byte length is invalid") + if self.repository_path is not None: + _safe_relative_path(self.repository_path) + if self.kind != "repository-file": + raise PortableLabV1ReleaseError( + "only a repository file may carry a repository path" + ) + + def as_dict(self) -> dict[str, object]: + return { + "asset_id": self.asset_id, + "kind": self.kind, + "sha256": self.sha256, + "byte_length": self.byte_length, + "repository_path": self.repository_path, + } + + +@dataclass(frozen=True, slots=True) +class PortableLabV1ReleaseInspection: + candidate_sha256: str + matched_assets: tuple[str, ...] + blockers: tuple[str, ...] + ready: bool + + def __post_init__(self) -> None: + _digest(self.candidate_sha256, "release candidate sha256") + if self.matched_assets != tuple(sorted(self.matched_assets)): + raise ValueError("matched release assets are not canonical") + if self.blockers != tuple(sorted(self.blockers)): + raise ValueError("release blockers are not canonical") + if self.ready and self.blockers: + raise ValueError("ready release inspection contains blockers") + + +@dataclass(frozen=True, slots=True) +class PortableLabV1ReleaseCandidate: + release_id: str + setup_id: str + definition_id: str + definition_version: int + definition_sha256: str + result_contract_sha256: str + executor_image_sha256: str | None + assets: tuple[PortableLabV1ReleaseAsset, ...] + phases: tuple[str, ...] + declared_blockers: tuple[str, ...] + candidate_sha256: str + repository_root: Path + + def __post_init__(self) -> None: + for value, label in ( + (self.release_id, "release id"), + (self.setup_id, "release setup id"), + (self.definition_id, "release definition id"), + ): + _pattern(value, _IDENTIFIER, label) + if ( + isinstance(self.definition_version, bool) + or not isinstance(self.definition_version, int) + or self.definition_version < 1 + ): + raise PortableLabV1ReleaseError("release definition version is invalid") + _digest(self.definition_sha256, "release definition sha256") + _digest(self.result_contract_sha256, "release result contract sha256") + if self.executor_image_sha256 is not None: + _digest(self.executor_image_sha256, "release executor image sha256") + ids = tuple(asset.asset_id for asset in self.assets) + if ids != tuple(sorted(ids)) or len(ids) != len(set(ids)): + raise PortableLabV1ReleaseError( + "release assets must be unique and canonically ordered" + ) + if not self.phases or len(self.phases) != len(set(self.phases)): + raise PortableLabV1ReleaseError("release phases are invalid") + for phase in self.phases: + _pattern(phase, _IDENTIFIER, "release phase") + if self.declared_blockers != tuple(sorted(self.declared_blockers)) or len( + self.declared_blockers + ) != len(set(self.declared_blockers)): + raise PortableLabV1ReleaseError("declared blockers are not canonical") + for blocker in self.declared_blockers: + _pattern(blocker, _IDENTIFIER, "release blocker") + _digest(self.candidate_sha256, "release candidate sha256") + if canonical_sha256(self.identity_document()) != self.candidate_sha256: + raise PortableLabV1ReleaseError("release candidate identity digest changed") + + @classmethod + def from_file( + cls, + path: Path, + *, + repository_root: Path, + ) -> PortableLabV1ReleaseCandidate: + payload = _read_bounded_regular_file( + path, + path.parent, + "portable LAB V1 release candidate", + maximum=_MAX_RELEASE_BYTES, + ) + document = _decoded_document( + payload, + "portable LAB V1 release candidate", + maximum=_MAX_RELEASE_BYTES, + ) + _exact_keys( + document, + { + "schema_version", + "release_id", + "setup_id", + "definition_id", + "definition_version", + "definition_sha256", + "result_contract_sha256", + "executor_image_sha256", + "assets", + "phases", + "declared_blockers", + "authority", + "candidate_sha256", + }, + "portable LAB V1 release candidate", + ) + if ( + document["schema_version"] != PORTABLE_LAB_V1_RELEASE_SCHEMA + or document["authority"] != OBSERVATION_ONLY_AUTHORITY + ): + raise PortableLabV1ReleaseError("release candidate contract is incompatible") + rows = document["assets"] + phases_value = document["phases"] + blockers_value = document["declared_blockers"] + if ( + not isinstance(rows, list) + or not isinstance(phases_value, list) + or not isinstance(blockers_value, list) + ): + raise PortableLabV1ReleaseError("release candidate arrays are invalid") + assets = tuple(_release_asset(row) for row in rows) + phases = tuple(_string(row, "release phase") for row in phases_value) + blockers = tuple(_string(row, "release blocker") for row in blockers_value) + image_value = document["executor_image_sha256"] + if image_value is not None and not isinstance(image_value, str): + raise PortableLabV1ReleaseError("release image identity is invalid") + return cls( + release_id=_string(document["release_id"], "release id"), + setup_id=_string(document["setup_id"], "release setup id"), + definition_id=_string( + document["definition_id"], "release definition id" + ), + definition_version=_positive_int( + document["definition_version"], "release definition version" + ), + definition_sha256=_string( + document["definition_sha256"], "release definition sha256" + ), + result_contract_sha256=_string( + document["result_contract_sha256"], + "release result contract sha256", + ), + executor_image_sha256=image_value, + assets=assets, + phases=phases, + declared_blockers=blockers, + candidate_sha256=_string( + document["candidate_sha256"], "release candidate sha256" + ), + repository_root=_real_directory(repository_root, "repository root"), + ) + + def identity_document(self) -> dict[str, object]: + return { + "schema_version": PORTABLE_LAB_V1_RELEASE_IDENTITY_SCHEMA, + "release_id": self.release_id, + "setup_id": self.setup_id, + "definition_id": self.definition_id, + "definition_version": self.definition_version, + "definition_sha256": self.definition_sha256, + "result_contract_sha256": self.result_contract_sha256, + "executor_image_sha256": self.executor_image_sha256, + "assets": [asset.as_dict() for asset in self.assets], + "phases": list(self.phases), + "declared_blockers": list(self.declared_blockers), + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + + def bind_definition(self, definition: PortableRunDefinition) -> None: + if ( + definition.setup_id != self.setup_id + or definition.definition_id != self.definition_id + or definition.version != self.definition_version + or definition.definition_sha256 != self.definition_sha256 + or definition.result_contract.contract_sha256 + != self.result_contract_sha256 + ): + raise PortableLabV1ReleaseError( + "release candidate belongs to another RunDefinition" + ) + + def inspect( + self, + bindings: Mapping[str, ReleaseAssetBinding] | None = None, + ) -> PortableLabV1ReleaseInspection: + supplied = {} if bindings is None else dict(bindings) + unknown = set(supplied) - {asset.asset_id for asset in self.assets} + if unknown: + raise PortableLabV1ReleaseError("release inspection contains unknown assets") + matched: list[str] = [] + blockers = set(self.declared_blockers) + for asset in self.assets: + binding: ReleaseAssetBinding | None = supplied.get(asset.asset_id) + if binding is None and asset.repository_path is not None: + binding = self.repository_root / Path( + *_safe_relative_path(asset.repository_path).parts + ) + if binding is None: + blockers.add(f"asset-{asset.asset_id}-missing") + continue + if _release_binding_matches(asset, binding): + matched.append(asset.asset_id) + else: + blockers.add(f"asset-{asset.asset_id}-mismatched") + if self.executor_image_sha256 is None: + blockers.add("executor-image-unsealed") + return PortableLabV1ReleaseInspection( + candidate_sha256=self.candidate_sha256, + matched_assets=tuple(sorted(matched)), + blockers=tuple(sorted(blockers)), + ready=not blockers, + ) + + def seal( + self, + inspection: PortableLabV1ReleaseInspection, + ) -> PortableLabV1ExecutorSeal: + """Return an executor seal only for a fully admitted immutable image.""" + + if ( + inspection.candidate_sha256 != self.candidate_sha256 + or not inspection.ready + or inspection.blockers + or self.executor_image_sha256 is None + or len(inspection.matched_assets) != len(self.assets) + ): + raise PortableLabV1ReleaseError( + "portable LAB V1 executor candidate is not sealable" + ) + identity = { + "schema_version": PORTABLE_LAB_V1_EXECUTOR_SEAL_SCHEMA, + "release_id": self.release_id, + "candidate_sha256": self.candidate_sha256, + "definition_sha256": self.definition_sha256, + "executor_image_sha256": self.executor_image_sha256, + "asset_sha256s": [asset.sha256 for asset in self.assets], + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + return PortableLabV1ExecutorSeal( + release_id=self.release_id, + candidate_sha256=self.candidate_sha256, + definition_sha256=self.definition_sha256, + executor_image_sha256=self.executor_image_sha256, + release_sha256=canonical_sha256(identity), + ) + + +@dataclass(frozen=True, slots=True) +class PortableLabV1ExecutorSeal: + """Path-free content identity of a fully admitted executor image.""" + + release_id: str + candidate_sha256: str + definition_sha256: str + executor_image_sha256: str + release_sha256: str + + def __post_init__(self) -> None: + _pattern(self.release_id, _IDENTIFIER, "executor seal release id") + for value, label in ( + (self.candidate_sha256, "executor seal candidate sha256"), + (self.definition_sha256, "executor seal definition sha256"), + (self.executor_image_sha256, "executor seal image sha256"), + (self.release_sha256, "executor seal release sha256"), + ): + _digest(value, label) + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": PORTABLE_LAB_V1_EXECUTOR_SEAL_SCHEMA, + "release_id": self.release_id, + "candidate_sha256": self.candidate_sha256, + "definition_sha256": self.definition_sha256, + "executor_image_sha256": self.executor_image_sha256, + "release_sha256": self.release_sha256, + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + + +@dataclass(frozen=True, slots=True) +class PortableLabV1PlanPhase: + phase_id: str + component_sha256s: tuple[str, ...] + input_roles: tuple[str, ...] + output_roles: tuple[str, ...] + + def __post_init__(self) -> None: + _pattern(self.phase_id, _IDENTIFIER, "plan phase id") + if ( + not self.component_sha256s + or self.component_sha256s != tuple(sorted(self.component_sha256s)) + ): + raise PortableLabV1PlanError("plan component identities are not canonical") + for digest_value in self.component_sha256s: + _digest(digest_value, "plan component sha256") + for values, label in ( + (self.input_roles, "plan input role"), + (self.output_roles, "plan output role"), + ): + if not values or values != tuple(sorted(values)) or len(values) != len(set(values)): + raise PortableLabV1PlanError(f"{label}s are not canonical") + for value in values: + _pattern(value, _ROLE, label) + + def as_dict(self) -> dict[str, object]: + return { + "phase_id": self.phase_id, + "component_sha256s": list(self.component_sha256s), + "input_roles": list(self.input_roles), + "output_roles": list(self.output_roles), + } + + +@dataclass(frozen=True, slots=True) +class PortableLabV1OrchestrationPlan: + observatory_job_id: str + observatory_request_sha256: str + observatory_identity_sha256: str + setup_id: str + definition_id: str + definition_version: int + definition_sha256: str + result_contract_sha256: str + source_input: PortableLabV1SourceInput + release_candidate_sha256: str + effective_ddrnet_config: Mapping[str, object] + effective_ddrnet_config_sha256: str + phases: tuple[PortableLabV1PlanPhase, ...] + blockers: tuple[str, ...] + plan_sha256: str + + def __post_init__(self) -> None: + _pattern(self.observatory_job_id, _OBSERVATORY_JOB_ID, "plan job id") + for value, label in ( + (self.setup_id, "plan setup id"), + (self.definition_id, "plan definition id"), + ): + _pattern(value, _IDENTIFIER, label) + for value, label in ( + (self.observatory_request_sha256, "plan request sha256"), + (self.observatory_identity_sha256, "plan job identity sha256"), + (self.definition_sha256, "plan definition sha256"), + (self.result_contract_sha256, "plan result contract sha256"), + (self.release_candidate_sha256, "plan release candidate sha256"), + (self.effective_ddrnet_config_sha256, "effective DDRNet config sha256"), + (self.plan_sha256, "plan sha256"), + ): + _digest(value, label) + if ( + self.source_input.observatory_job_id != self.observatory_job_id + or self.source_input.observatory_request_sha256 + != self.observatory_request_sha256 + or self.source_input.observatory_identity_sha256 + != self.observatory_identity_sha256 + ): + raise PortableLabV1PlanError("plan source belongs to another job") + if self.blockers != tuple(sorted(self.blockers)) or len(self.blockers) != len( + set(self.blockers) + ): + raise PortableLabV1PlanError("plan blockers are not canonical") + if tuple(phase.phase_id for phase in self.phases) != ( + "source-materialization", + "eomt-full-session", + "ddrnet-full-session", + "result-v2-assembly", + ): + raise PortableLabV1PlanError("combined LAB V1 phase order changed") + if ( + canonical_sha256(self.effective_ddrnet_config) + != self.effective_ddrnet_config_sha256 + ): + raise PortableLabV1PlanError("effective DDRNet config digest changed") + if canonical_sha256(self.identity_document()) != self.plan_sha256: + raise PortableLabV1PlanError("orchestration plan identity changed") + + @classmethod + def create( + cls, + *, + job: SealedObservatoryRecordedJob, + definition: PortableRunDefinition, + source: PortableLabV1SourceInput, + release: PortableLabV1ReleaseCandidate, + release_inspection: PortableLabV1ReleaseInspection, + legacy_ddrnet_config: Mapping[str, object], + ) -> PortableLabV1OrchestrationPlan: + _verify_definition_and_job(definition, job) + _verify_source_and_job(source, job, definition) + release.bind_definition(definition) + if release_inspection.candidate_sha256 != release.candidate_sha256: + raise PortableLabV1PlanError("release inspection belongs to another candidate") + expected_asset_ids = tuple(asset.asset_id for asset in release.assets) + if release_inspection.ready and ( + release_inspection.matched_assets != expected_asset_ids + or release_inspection.blockers + ): + raise PortableLabV1PlanError( + "ready release inspection does not admit every exact asset" + ) + effective = build_portable_ddrnet_effective_config( + legacy_ddrnet_config, + source=source, + ) + components = {item.component_id: item.sha256 for item in definition.components} + release_assets = {asset.asset_id: asset.sha256 for asset in release.assets} + phases = ( + PortableLabV1PlanPhase( + phase_id="source-materialization", + component_sha256s=(definition.source_adapter.contract_sha256,), + input_roles=("camera-compute-job", "source-documents"), + output_roles=("source-input-manifest",), + ), + PortableLabV1PlanPhase( + phase_id="eomt-full-session", + component_sha256s=tuple( + sorted( + ( + components["eomt-recorded-orchestrator-v1"], + components["eomt-recorded-profile-v1"], + components["eomt-recorded-runner-v1"], + definition.model_manifest_sha256, + ) + ) + ), + input_roles=("camera-compute-job",), + output_roles=("eomt-component-result",), + ), + PortableLabV1PlanPhase( + phase_id="ddrnet-full-session", + component_sha256s=tuple( + sorted( + ( + components["ddrnet-portable-runtime-config-v2"], + components["vegetation-mission-policy-v1"], + components["vegetation-provider-label-map-v1"], + release_assets["ddrnet-goose-runner"], + definition.model_manifest_sha256, + ) + ) + ), + input_roles=("camera-compute-job", "ddrnet-effective-config"), + output_roles=("ddrnet-component-result",), + ), + PortableLabV1PlanPhase( + phase_id="result-v2-assembly", + component_sha256s=(release_assets["lab-v1-portable-contracts"],), + input_roles=("ddrnet-component-result", "eomt-component-result"), + output_roles=("portable-result-draft",), + ), + ) + effective_sha256 = canonical_sha256(effective) + identity = _plan_identity_document( + observatory_job_id=job.job_id, + observatory_request_sha256=job.request_sha256, + observatory_identity_sha256=job.identity_sha256, + setup_id=definition.setup_id, + definition_id=definition.definition_id, + definition_version=definition.version, + definition_sha256=definition.definition_sha256, + result_contract_sha256=definition.result_contract.contract_sha256, + source_input_sha256=source.identity_sha256, + release_candidate_sha256=release.candidate_sha256, + effective_ddrnet_config_sha256=effective_sha256, + phases=phases, + release_blockers=release_inspection.blockers, + ) + return cls( + observatory_job_id=job.job_id, + observatory_request_sha256=job.request_sha256, + observatory_identity_sha256=job.identity_sha256, + setup_id=definition.setup_id, + definition_id=definition.definition_id, + definition_version=definition.version, + definition_sha256=definition.definition_sha256, + result_contract_sha256=definition.result_contract.contract_sha256, + source_input=source, + release_candidate_sha256=release.candidate_sha256, + effective_ddrnet_config=effective, + effective_ddrnet_config_sha256=effective_sha256, + phases=phases, + blockers=release_inspection.blockers, + plan_sha256=canonical_sha256(identity), + ) + + @property + def executable(self) -> bool: + return not self.blockers + + def require_executable(self) -> None: + if self.blockers: + raise PortableLabV1PlanError( + "portable LAB V1 release is blocked: " + ", ".join(self.blockers) + ) + + def identity_document(self) -> dict[str, object]: + return _plan_identity_document( + observatory_job_id=self.observatory_job_id, + observatory_request_sha256=self.observatory_request_sha256, + observatory_identity_sha256=self.observatory_identity_sha256, + setup_id=self.setup_id, + definition_id=self.definition_id, + definition_version=self.definition_version, + definition_sha256=self.definition_sha256, + result_contract_sha256=self.result_contract_sha256, + source_input_sha256=self.source_input.identity_sha256, + release_candidate_sha256=self.release_candidate_sha256, + effective_ddrnet_config_sha256=self.effective_ddrnet_config_sha256, + phases=self.phases, + release_blockers=self.blockers, + ) + + def as_dict(self) -> dict[str, object]: + identity = self.identity_document() + identity.pop("schema_version") + return { + "schema_version": PORTABLE_LAB_V1_PLAN_SCHEMA, + "plan_sha256": self.plan_sha256, + **identity, + "effective_ddrnet_config": dict(self.effective_ddrnet_config), + "blockers": list(self.blockers), + "execution_admitted": self.executable, + } + + @property + def canonical_bytes(self) -> bytes: + return canonical_json(self.as_dict()) + + +def build_portable_ddrnet_effective_config( + portable_profile: Mapping[str, object], + *, + source: PortableLabV1SourceInput, +) -> dict[str, object]: + """Compile the source-independent v2 profile for one admitted recording. + + Model, preprocessing, policy, runtime, and fail-closed invariants come from + the exact repository-owned definition component. Only the legacy runner's + ``ravnoves`` input stanza is produced dynamically from the sealed source; + no session id, path, frame count, or source digest lives in the profile. + """ + + copied = cast(dict[str, object], json.loads(canonical_json(portable_profile))) + if copied.get("schema_version") != PORTABLE_LAB_V1_DDRNET_PROFILE_SCHEMA: + raise PortableLabV1PlanError("portable DDRNet profile schema is incompatible") + if copied.get("profile_id") != "lab-v1-eomt-ddrnet-portable-v2": + raise PortableLabV1PlanError("portable DDRNet profile identity changed") + source_binding = _object( + copied.pop("source_binding", None), + "portable DDRNet source binding", + ) + if source_binding != { + "mode": "admitted-k1-recording", + "camera_source_id": "sensor.camera.right", + "camera_timeline": "dynamic", + "filesystem_paths": "executor-resolved", + "expected_width": 800, + "expected_height": 600, + "expected_frame_count": "source-derived", + "frame_indices": "source-derived-representative", + "crop_contract": "center-600-square-to-512; outside-crop-is-undefined", + }: + raise PortableLabV1PlanError("portable DDRNet source contract changed") + effective_contract = _object( + copied.pop("effective_config_contract", None), + "portable DDRNet effective config contract", + ) + if effective_contract != { + "schema_version": PORTABLE_LAB_V1_DDRNET_EFFECTIVE_CONFIG_SCHEMA, + "dynamic_field": "ravnoves", + "source_id": "sealed-source-derived", + "source_sha256": "sealed-camera-input-derived", + "base_m4_result_id": None, + }: + raise PortableLabV1PlanError("portable DDRNet compilation contract changed") + copied.pop("profile_id", None) + copied["schema_version"] = PORTABLE_LAB_V1_DDRNET_EFFECTIVE_CONFIG_SCHEMA + candidates = _object(copied.get("candidates"), "DDRNet config candidates") + candidate = _object(candidates.get(_DDRNET_CANDIDATE_KEY), "DDRNet candidate") + if ( + candidate.get("candidate_id") != _DDRNET_CANDIDATE_ID + or candidate.get("checkpoint_sha256") != _DDRNET_CHECKPOINT_SHA256 + ): + raise PortableLabV1PlanError("legacy DDRNet candidate identity changed") + dataset = _object(copied.get("dataset"), "DDRNet dataset contract") + if dataset.get("mapping_sha256") != _GOOSE_MAPPING_SHA256: + raise PortableLabV1PlanError("DDRNet taxonomy mapping identity changed") + invariants = _object(copied.get("invariants"), "DDRNet invariants") + required_false = ( + "actuation_authority", + "camera_semantics_can_clear_rigid_geometry", + "canonical_triton_mutation_allowed", + "missing_or_unknown_is_free", + "navigation_authority", + "outside_center_crop_is_free", + ) + if ( + invariants.get("one_heavy_candidate_at_a_time") is not True + or invariants.get("raw_fisheye_is_immutable") is not True + or any(invariants.get(name) is not False for name in required_false) + ): + raise PortableLabV1PlanError("DDRNet fail-closed invariants changed") + copied["ravnoves"] = { + "source_id": ( + f"portable-k1/{source.source_session_id}/" + f"{source.camera_source_id}@{source.camera_input_sha256}" + ), + "source_sha256": source.camera_input_sha256, + "base_m4_result_id": None, + "expected_width": 800, + "expected_height": 600, + "expected_frame_count": source.frame_count, + "frame_indices": _representative_frame_indices(source.frame_count), + "crop_contract": source_binding["crop_contract"], + } + return copied + + +@dataclass(frozen=True, slots=True) +class PortableLabV1ResultAssembly: + root: Path + result_id: str + result_document_sha256: str + result_document: Mapping[str, object] + artifacts: tuple[PortableResultArtifact, ...] + + def __post_init__(self) -> None: + _real_directory(self.root, "portable LAB V1 result assembly") + _pattern(self.result_id, _SESSION_ID, "portable LAB V1 result id") + _digest(self.result_document_sha256, "portable LAB V1 result document sha256") + roles = tuple(item.role for item in self.artifacts) + if roles != tuple(sorted(roles)) or len(roles) != len(set(roles)): + raise PortableLabV1ResultError("assembled artifacts are not canonical") + result_artifact = tuple( + item for item in self.artifacts if item.role == "result-document" + ) + if ( + len(result_artifact) != 1 + or result_artifact[0].sha256 != self.result_document_sha256 + ): + raise PortableLabV1ResultError("assembled result document is not bound") + + +def assemble_lab_v1_result_v2( + *, + plan: PortableLabV1OrchestrationPlan, + definition: PortableRunDefinition, + eomt_result_root: Path, + ddrnet_result_root: Path, + output_parent: Path, +) -> PortableLabV1ResultAssembly: + """Assemble exact legacy component outputs into the portable v2 document. + + Production callers must use an executable plan. Contract tests construct + an explicit in-memory admitted inspection; the checked-in release candidate + itself remains blocked and cannot produce such a plan. + """ + + _verify_plan_definition(plan, definition) + plan.require_executable() + eomt_root = _real_directory(eomt_result_root, "EoMT result root") + ddrnet_root = _real_directory(ddrnet_result_root, "DDRNet result root") + eomt_result_path = eomt_root / "result.json" + ddrnet_result_path = ddrnet_root / "result.json" + eomt = _read_json_file(eomt_result_path, eomt_root, "EoMT result") + ddrnet = _read_json_file(ddrnet_result_path, ddrnet_root, "DDRNet result") + _validate_eomt_component(eomt, eomt_root, plan, definition) + _validate_ddrnet_component(ddrnet, ddrnet_root, plan, definition) + + parent = _prepare_real_directory(output_parent, "portable result output parent") + staging = parent / f".lab-v1-{secrets.token_hex(16)}" + staging.mkdir(mode=0o700) + published = False + try: + artifact_specs: list[tuple[str, str, bytes | Path]] = [ + ( + "ddrnet-effective-config", + "application/json", + canonical_json(plan.effective_ddrnet_config), + ), + ( + "ddrnet-result-document", + "application/json", + ddrnet_result_path, + ), + ( + "eomt-result-document", + "application/json", + eomt_result_path, + ), + ( + "orchestration-plan", + "application/json", + plan.canonical_bytes, + ), + ( + "source-input-manifest", + "application/json", + plan.source_input.canonical_bytes, + ), + ] + for artifact in _eomt_component_artifacts(eomt, eomt_root): + artifact_specs.append( + ( + f"eomt-{artifact['kind']}", + cast(str, artifact["media_type"]), + cast(Path, artifact["path"]), + ) + ) + artifact_specs.extend( + ( + ( + "eomt-decode-repair", + "application/json", + _required_component_file(eomt_root, "decode-repair.json"), + ), + ( + "ddrnet-decode-repair", + "application/json", + _required_component_file(ddrnet_root, "decode-repair.json"), + ), + ( + "ddrnet-semantic-mask-archive", + "application/zip", + _ddrnet_mask_archive(ddrnet, ddrnet_root), + ), + ) + ) + if len(artifact_specs) > _MAX_ARTIFACTS - 1: + raise PortableLabV1ResultError("portable LAB V1 artifact count is too large") + artifact_specs.sort(key=lambda item: item[0]) + artifacts: list[PortableResultArtifact] = [] + for role, media_type, source in artifact_specs: + _pattern(role, _ROLE, "portable result artifact role") + suffix = ".json" if media_type == "application/json" else _source_suffix(source) + relative = f"artifacts/{role}{suffix}" + destination = staging / Path(*PurePosixPath(relative).parts) + if isinstance(source, bytes): + _write_exact_bytes(destination, source) + else: + _copy_exact_file(source, destination) + artifacts.append( + PortableResultArtifact( + role=role, + relative_path=relative, + media_type=media_type, + byte_length=destination.stat().st_size, + sha256=_sha256_file(destination), + ) + ) + + component_identity = { + "eomt": { + "result_id": _string(eomt.get("result_id"), "EoMT result id"), + "result_document_sha256": _sha256_file(eomt_result_path), + "model_release_id": _EOMT_RELEASE_ID, + "frames_processed": plan.source_input.frame_count, + }, + "ddrnet": { + "result_id": _string(ddrnet.get("result_id"), "DDRNet result id"), + "result_document_sha256": _sha256_file(ddrnet_result_path), + "model_release_id": _DDRNET_RELEASE_ID, + "frames_processed": plan.source_input.frame_count, + }, + } + artifact_documents = [artifact.as_dict() for artifact in artifacts] + identity = _result_identity_document( + plan=plan, + definition=definition, + components=component_identity, + artifacts=artifact_documents, + ) + identity_sha256 = canonical_sha256(identity) + result_id = f"lab-v1-eomt-ddrnet-{identity_sha256}" + result_document = { + "schema_version": PORTABLE_LAB_V1_RESULT_SCHEMA, + "result_id": result_id, + "result_kind": definition.result_contract.result_kind, + "identity_sha256": identity_sha256, + "identity": identity, + "source": { + "session_id": plan.source_input.source_session_id, + "catalog_sha256": plan.source_input.source_catalog_sha256, + "bundle_sha256": plan.source_input.source_bundle_sha256, + "capability_manifest_sha256": ( + plan.source_input.source_capability_manifest_sha256 + ), + "camera_input_sha256": plan.source_input.camera_input_sha256, + "frame_count": plan.source_input.frame_count, + "timeline_start_seconds": plan.source_input.timeline_start_seconds, + "timeline_end_seconds": plan.source_input.timeline_end_seconds, + }, + "run_definition": { + "setup_id": definition.setup_id, + "definition_id": definition.definition_id, + "version": definition.version, + "definition_sha256": definition.definition_sha256, + "result_contract_sha256": ( + definition.result_contract.contract_sha256 + ), + "release_candidate_sha256": plan.release_candidate_sha256, + "plan_sha256": plan.plan_sha256, + }, + "components": component_identity, + "qualification": { + "mode": "recorded-observation", + "ground_truth_available": False, + "navigation_or_safety_accepted": False, + "production_accepted": False, + }, + "artifacts": artifact_documents, + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + result_bytes = canonical_json(result_document) + result_relative = "artifacts/result.json" + result_path = staging / result_relative + _write_exact_bytes(result_path, result_bytes) + result_artifact = PortableResultArtifact( + role="result-document", + relative_path=result_relative, + media_type="application/json", + byte_length=len(result_bytes), + sha256=hashlib.sha256(result_bytes).hexdigest(), + ) + complete_artifacts = tuple( + sorted((*artifacts, result_artifact), key=lambda item: item.role) + ) + final = parent / result_id + if final.exists(): + raise PortableLabV1ResultError( + "an assembled result with this identity already exists" + ) + _fsync_tree(staging) + os.replace(staging, final) + _fsync_directory(parent) + published = True + assembly = PortableLabV1ResultAssembly( + root=final, + result_id=result_id, + result_document_sha256=result_artifact.sha256, + result_document=result_document, + artifacts=complete_artifacts, + ) + _validate_assembly(assembly, plan=plan, definition=definition) + return assembly + finally: + if not published and staging.exists(): + shutil.rmtree(staging) + + +def package_lab_v1_result( + *, + assembly: PortableLabV1ResultAssembly, + plan: PortableLabV1OrchestrationPlan, + job: ObservatoryRecordedJob, + definition: PortableRunDefinition, + created_at_utc: str, + output_parent: Path, +) -> PortableWorkerResultDraft: + """Wrap a validated assembly in the generic portable result package. + + This step intentionally requires the durable server job because the current + sealed Worker claim omits its submission receipt. Until that receipt is + transported into the local adapter (or the server owns this step), release + admission remains blocked instead of weakening the package identity. + """ + + _verify_plan_definition(plan, definition) + if ( + job.job_id != plan.observatory_job_id + or job.request_sha256 != plan.observatory_request_sha256 + or job.identity_sha256 != plan.observatory_identity_sha256 + or job.result_id not in (None, assembly.result_id) + ): + raise PortableLabV1ResultError("durable job differs from the assembly plan") + _validate_assembly(assembly, plan=plan, definition=definition) + package = PortableResultPackageManifest.create( + job=job, + definition=definition, + result_id=assembly.result_id, + created_at_utc=created_at_utc, + artifacts=assembly.artifacts, + ) + parent = _prepare_real_directory(output_parent, "portable package output parent") + staging = parent / f".package-{secrets.token_hex(16)}" + staging.mkdir(mode=0o700) + complete = False + try: + for artifact in assembly.artifacts: + source = assembly.root / Path(*PurePosixPath(artifact.relative_path).parts) + destination = staging / Path(*PurePosixPath(artifact.relative_path).parts) + _copy_exact_file(source, destination) + _write_exact_bytes(staging / "manifest.json", package.canonical_bytes) + final = parent / package.manifest_sha256 + if final.exists(): + raise PortableLabV1ResultError( + "a portable result package with this identity already exists" + ) + _fsync_tree(staging) + os.replace(staging, final) + _fsync_directory(parent) + complete = True + return PortableWorkerResultDraft( + root=final, + result_id=assembly.result_id, + result_sha256=package.manifest_sha256, + result_contract_sha256=definition.result_contract.contract_sha256, + ) + finally: + if not complete and staging.exists(): + shutil.rmtree(staging) + + +def validate_lab_v1_result_v2(context: PortableResultValidationContext) -> None: + """Exact publisher validator for ``recorded-eomt-ddrnet-review/v2``.""" + + definition = context.definition + if ( + definition.setup_id != _EXPECTED_SETUP_ID + or definition.definition_id != _EXPECTED_DEFINITION_ID + or definition.result_contract.result_schema != PORTABLE_LAB_V1_RESULT_SCHEMA + or definition.result_contract.contract_sha256 + != _EXPECTED_RESULT_CONTRACT_SHA256 + ): + raise PortableLabV1ResultError("LAB V1 validator received another definition") + document = dict(context.result_document) + _validate_v2_document_shape(document) + if ( + document["result_id"] != context.job.result_id + or document["result_kind"] != definition.result_contract.result_kind + or document["authority"] != OBSERVATION_ONLY_AUTHORITY + ): + raise PortableLabV1ResultError("portable LAB V1 result envelope changed") + identity = _object(document["identity"], "portable LAB V1 identity") + if ( + canonical_sha256(identity) != document["identity_sha256"] + or document["result_id"] + != f"lab-v1-eomt-ddrnet-{document['identity_sha256']}" + ): + raise PortableLabV1ResultError("portable LAB V1 result identity changed") + source = _object(document["source"], "portable LAB V1 source") + run_definition = _object( + document["run_definition"], + "portable LAB V1 run definition", + ) + _exact_keys( + source, + { + "session_id", + "catalog_sha256", + "bundle_sha256", + "capability_manifest_sha256", + "camera_input_sha256", + "frame_count", + "timeline_start_seconds", + "timeline_end_seconds", + }, + "portable LAB V1 source", + ) + _exact_keys( + run_definition, + { + "setup_id", + "definition_id", + "version", + "definition_sha256", + "result_contract_sha256", + "release_candidate_sha256", + "plan_sha256", + }, + "portable LAB V1 run definition", + ) + if ( + source.get("session_id") != context.job.source_session_id + or source.get("catalog_sha256") != context.job.source_catalog_sha256 + or source.get("bundle_sha256") != context.job.source_bundle_sha256 + or source.get("capability_manifest_sha256") + != context.job.source_capability_manifest_sha256 + or run_definition.get("setup_id") != definition.setup_id + or run_definition.get("definition_id") != definition.definition_id + or run_definition.get("version") != definition.version + or run_definition.get("definition_sha256") != definition.definition_sha256 + or run_definition.get("result_contract_sha256") + != definition.result_contract.contract_sha256 + ): + raise PortableLabV1ResultError("portable LAB V1 provenance changed") + artifact_rows = document["artifacts"] + if not isinstance(artifact_rows, list): + raise PortableLabV1ResultError("portable LAB V1 artifacts are not an array") + declared = tuple(_artifact_from_document(row) for row in artifact_rows) + declared_roles = tuple(artifact.role for artifact in declared) + if ( + declared_roles != tuple(sorted(declared_roles)) + or len(declared_roles) != len(set(declared_roles)) + ): + raise PortableLabV1ResultError( + "portable result artifacts are not canonical" + ) + declared_by_role = {artifact.role: artifact for artifact in declared} + package_by_role = {artifact.role: artifact for artifact in context.manifest.artifacts} + if set(package_by_role) != {*declared_by_role, "result-document"}: + raise PortableLabV1ResultError("portable package artifact roles changed") + for role, artifact in declared_by_role.items(): + if package_by_role[role] != artifact: + raise PortableLabV1ResultError("portable artifact descriptor changed") + for artifact in context.manifest.artifacts: + path = context.artifact_paths.get(artifact.role) + if path is None or _sha256_file(path) != artifact.sha256: + raise PortableLabV1ResultError("portable result artifact content changed") + source_manifest = _read_json_file( + context.artifact_paths["source-input-manifest"], + context.artifact_paths["source-input-manifest"].parent, + "portable source input manifest", + require_canonical=True, + ) + orchestration = _read_json_file( + context.artifact_paths["orchestration-plan"], + context.artifact_paths["orchestration-plan"].parent, + "portable orchestration plan", + require_canonical=True, + ) + source_input = _source_input_from_document(source_manifest) + _verify_source_and_job(source_input, context.job, definition) + plan = _orchestration_plan_from_document( + orchestration, + source_input=source_input, + ) + _verify_plan_definition(plan, definition) + plan.require_executable() + if ( + source_input.identity_sha256 != identity.get("source_input_sha256") + or plan.plan_sha256 != run_definition.get("plan_sha256") + or plan.release_candidate_sha256 + != run_definition.get("release_candidate_sha256") + or plan.observatory_job_id != context.job.job_id + or plan.observatory_request_sha256 != context.job.request_sha256 + or plan.observatory_identity_sha256 != context.job.identity_sha256 + or source.get("camera_input_sha256") != source_input.camera_input_sha256 + or source.get("frame_count") != source_input.frame_count + or source.get("timeline_start_seconds") + != source_input.timeline_start_seconds + or source.get("timeline_end_seconds") != source_input.timeline_end_seconds + ): + raise PortableLabV1ResultError("portable source or plan artifact changed") + _validate_component_documents_from_context( + context, + document=document, + definition=definition, + plan=plan, + ) + expected_identity = _result_identity_from_document(document) + if identity != expected_identity: + raise PortableLabV1ResultError("portable LAB V1 identity projection changed") + + +def _source_input_from_document( + document: Mapping[str, object], +) -> PortableLabV1SourceInput: + _exact_keys( + document, + { + "schema_version", + "observatory_job", + "source", + "camera_compute_job", + "authority", + }, + "portable source input manifest", + ) + if ( + document["schema_version"] != PORTABLE_LAB_V1_SOURCE_SCHEMA + or document["authority"] != OBSERVATION_ONLY_AUTHORITY + ): + raise PortableLabV1ResultError("portable source input contract changed") + observatory_job = _object( + document["observatory_job"], + "portable source Observatory job", + ) + source = _object(document["source"], "portable source identity") + camera = _object(document["camera_compute_job"], "portable camera compute job") + _exact_keys( + observatory_job, + {"job_id", "request_sha256", "identity_sha256"}, + "portable source Observatory job", + ) + _exact_keys( + source, + { + "session_id", + "catalog_sha256", + "bundle_sha256", + "capability_manifest_sha256", + "adapter_sha256", + }, + "portable source identity", + ) + _exact_keys( + camera, + { + "job_id", + "input_sha256", + "source_id", + "codec_epoch", + "input_byte_length", + "frame_count", + "timeline_start_seconds", + "timeline_end_seconds", + "generation_sha256", + "calibration_sha256", + }, + "portable camera compute job", + ) + return PortableLabV1SourceInput( + observatory_job_id=_string( + observatory_job["job_id"], "portable source Observatory job id" + ), + observatory_request_sha256=_string( + observatory_job["request_sha256"], + "portable source Observatory request sha256", + ), + observatory_identity_sha256=_string( + observatory_job["identity_sha256"], + "portable source Observatory identity sha256", + ), + source_session_id=_string(source["session_id"], "portable source session id"), + source_catalog_sha256=_string( + source["catalog_sha256"], "portable source catalog sha256" + ), + source_bundle_sha256=_string( + source["bundle_sha256"], "portable source bundle sha256" + ), + source_capability_manifest_sha256=_string( + source["capability_manifest_sha256"], + "portable source capability sha256", + ), + source_adapter_sha256=_string( + source["adapter_sha256"], "portable source adapter sha256" + ), + camera_job_id=_string(camera["job_id"], "portable camera job id"), + camera_input_sha256=_string( + camera["input_sha256"], "portable camera input sha256" + ), + camera_source_id=_string(camera["source_id"], "portable camera source id"), + codec_epoch=_positive_int(camera["codec_epoch"], "portable codec epoch"), + input_byte_length=_positive_int( + camera["input_byte_length"], "portable camera input byte length" + ), + frame_count=_positive_int(camera["frame_count"], "portable frame count"), + timeline_start_seconds=_finite_float( + camera["timeline_start_seconds"], "portable timeline start" + ), + timeline_end_seconds=_finite_float( + camera["timeline_end_seconds"], "portable timeline end" + ), + camera_generation_sha256=_string( + camera["generation_sha256"], "portable camera generation sha256" + ), + calibration_sha256=_string( + camera["calibration_sha256"], "portable calibration sha256" + ), + ) + + +def _orchestration_plan_from_document( + document: Mapping[str, object], + *, + source_input: PortableLabV1SourceInput, +) -> PortableLabV1OrchestrationPlan: + _exact_keys( + document, + { + "schema_version", + "plan_sha256", + "observatory_job", + "run_definition", + "source_input_sha256", + "release_candidate_sha256", + "release_admission", + "effective_ddrnet_config_sha256", + "phases", + "authority", + "effective_ddrnet_config", + "blockers", + "execution_admitted", + }, + "portable orchestration plan", + ) + if ( + document["schema_version"] != PORTABLE_LAB_V1_PLAN_SCHEMA + or document["authority"] != OBSERVATION_ONLY_AUTHORITY + ): + raise PortableLabV1ResultError("portable orchestration contract changed") + observatory_job = _object( + document["observatory_job"], + "portable plan Observatory job", + ) + run_definition = _object( + document["run_definition"], + "portable plan RunDefinition", + ) + release_admission = _object( + document["release_admission"], + "portable plan release admission", + ) + effective = _object( + document["effective_ddrnet_config"], + "portable plan effective DDRNet config", + ) + _exact_keys( + observatory_job, + {"job_id", "request_sha256", "identity_sha256"}, + "portable plan Observatory job", + ) + _exact_keys( + run_definition, + { + "setup_id", + "definition_id", + "version", + "definition_sha256", + "result_contract_sha256", + }, + "portable plan RunDefinition", + ) + _exact_keys( + release_admission, + {"ready", "blockers"}, + "portable plan release admission", + ) + blockers_value = document["blockers"] + phases_value = document["phases"] + admission_blockers_value = release_admission["blockers"] + if ( + not isinstance(blockers_value, list) + or not isinstance(phases_value, list) + or not isinstance(admission_blockers_value, list) + ): + raise PortableLabV1ResultError("portable plan arrays changed") + blockers = tuple(_string(value, "portable plan blocker") for value in blockers_value) + if ( + admission_blockers_value != blockers_value + or release_admission["ready"] is not (not blockers) + or document["execution_admitted"] is not (not blockers) + ): + raise PortableLabV1ResultError("portable plan admission changed") + phases = tuple(_plan_phase_from_document(value) for value in phases_value) + plan = PortableLabV1OrchestrationPlan( + observatory_job_id=_string( + observatory_job["job_id"], "portable plan Observatory job id" + ), + observatory_request_sha256=_string( + observatory_job["request_sha256"], + "portable plan Observatory request sha256", + ), + observatory_identity_sha256=_string( + observatory_job["identity_sha256"], + "portable plan Observatory identity sha256", + ), + setup_id=_string(run_definition["setup_id"], "portable plan setup id"), + definition_id=_string( + run_definition["definition_id"], "portable plan definition id" + ), + definition_version=_positive_int( + run_definition["version"], "portable plan definition version" + ), + definition_sha256=_string( + run_definition["definition_sha256"], + "portable plan definition sha256", + ), + result_contract_sha256=_string( + run_definition["result_contract_sha256"], + "portable plan result contract sha256", + ), + source_input=source_input, + release_candidate_sha256=_string( + document["release_candidate_sha256"], + "portable plan release candidate sha256", + ), + effective_ddrnet_config=effective, + effective_ddrnet_config_sha256=_string( + document["effective_ddrnet_config_sha256"], + "portable plan effective config sha256", + ), + phases=phases, + blockers=blockers, + plan_sha256=_string(document["plan_sha256"], "portable plan sha256"), + ) + if document["source_input_sha256"] != source_input.identity_sha256: + raise PortableLabV1ResultError("portable plan source identity changed") + return plan + + +def _plan_phase_from_document(value: object) -> PortableLabV1PlanPhase: + row = _object(value, "portable plan phase") + _exact_keys( + row, + {"phase_id", "component_sha256s", "input_roles", "output_roles"}, + "portable plan phase", + ) + component_values = row["component_sha256s"] + input_values = row["input_roles"] + output_values = row["output_roles"] + if not all( + isinstance(values, list) + for values in (component_values, input_values, output_values) + ): + raise PortableLabV1ResultError("portable plan phase arrays changed") + return PortableLabV1PlanPhase( + phase_id=_string(row["phase_id"], "portable plan phase id"), + component_sha256s=tuple( + _string(item, "portable plan component sha256") + for item in cast(list[object], component_values) + ), + input_roles=tuple( + _string(item, "portable plan input role") + for item in cast(list[object], input_values) + ), + output_roles=tuple( + _string(item, "portable plan output role") + for item in cast(list[object], output_values) + ), + ) + + +def _validate_source_documents( + *, + source_bundle: Mapping[str, object], + capability: Mapping[str, object], + job: SealedObservatoryRecordedJob, + definition: PortableRunDefinition, + camera_job: CameraComputeJob, +) -> None: + requirements = definition.source_requirements + if ( + source_bundle.get("schema_version") != PORTABLE_SOURCE_BUNDLE_SCHEMA + or capability.get("schema_version") != PORTABLE_SOURCE_CAPABILITY_SCHEMA + or source_bundle.get("authority") != OBSERVATION_ONLY_AUTHORITY + or capability.get("authority") != OBSERVATION_ONLY_AUTHORITY + ): + raise PortableLabV1SourceError("source document schema or authority changed") + adapter = _object(source_bundle.get("source_adapter"), "source adapter") + if ( + source_bundle.get("source_session_id") != job.source_session_id + or capability.get("source_session_id") != job.source_session_id + or camera_job.session_id != job.source_session_id + or source_bundle.get("source_catalog_sha256") + != job.source_catalog_sha256 + or capability.get("source_catalog_sha256") != job.source_catalog_sha256 + or capability.get("source_bundle_sha256") != job.source_bundle_sha256 + or capability.get("source_adapter_sha256") != job.source_adapter_sha256 + or adapter + != { + "id": definition.source_adapter.adapter_id, + "version": definition.source_adapter.version, + "sha256": definition.source_adapter.contract_sha256, + } + or source_bundle.get("plugin_id") != requirements.plugin_id + or source_bundle.get("archive_id") != requirements.archive_id + ): + raise PortableLabV1SourceError("source document identity differs from the job") + modalities = capability.get("modalities") + if not isinstance(modalities, list): + raise PortableLabV1SourceError("source modalities are invalid") + modality_rows: dict[str, Mapping[str, object]] = {} + for value in modalities: + row = _object(value, "source modality") + modality = _string(row.get("modality"), "modality") + if modality in modality_rows: + raise PortableLabV1SourceError("source modality is duplicated") + modality_rows[modality] = row + if set(modality_rows) != set(requirements.required_modalities): + raise PortableLabV1SourceError("source modality set changed") + video = _object(modality_rows.get("video"), "video modality") + camera = _object(source_bundle.get("camera"), "source camera") + epoch = _object(camera.get("epoch"), "source camera epoch") + init = _object(epoch.get("init"), "source camera init") + profile = _object(capability.get("camera_profile"), "camera profile") + calibration = _object(capability.get("calibration"), "camera calibration") + segments = epoch.get("segments") + if not isinstance(segments, list) or len(segments) != camera_job.segment_count: + raise PortableLabV1SourceError("source camera segment count changed") + if ( + video.get("source_id") != requirements.camera_source_id + or video.get("semantic_channel_id") + != requirements.camera_semantic_channel_id + or video.get("seekable") is not True + or camera_job.source_id != requirements.camera_source_id + or camera_job.codec_epoch != epoch.get("ordinal") + or epoch.get("media_type") != requirements.recorded_media_type + or init.get("sha256") != requirements.recorded_media_init_sha256 + or profile.get("media_type") != requirements.recorded_media_type + or profile.get("init_sha256") != requirements.recorded_media_init_sha256 + or profile.get("width") != requirements.camera_width + or profile.get("height") != requirements.camera_height + or profile.get("frame_count") != camera_job.segment_count + or profile.get("timeline_start_seconds") != camera_job.timeline_start_seconds + or profile.get("timeline_end_seconds") != camera_job.timeline_end_seconds + or epoch.get("timeline_start_seconds") != camera_job.timeline_start_seconds + or epoch.get("timeline_end_seconds") != camera_job.timeline_end_seconds + or profile.get("generation_sha256") != camera.get("generation_sha256") + or calibration.get("slot") != requirements.calibration_slot + or calibration.get("sha256") != requirements.calibration_identity_sha256 + ): + raise PortableLabV1SourceError("source camera capability changed") + manifest = _read_json_file( + camera_job.manifest_path, + camera_job.job_root, + "camera compute job manifest", + ) + input_document = _object(manifest.get("input"), "camera compute job input") + if ( + manifest.get("input_sha256") != camera_job.input_sha256 + or input_document.get("segment_count") != camera_job.segment_count + or input_document.get("byte_length") != camera_job.input_byte_length + ): + raise PortableLabV1SourceError("camera compute job summary changed") + files = input_document.get("files") + if not isinstance(files, list): + raise PortableLabV1SourceError("camera compute job file set is invalid") + file_rows = { + PurePosixPath(_string(_object(row, "camera file").get("path"), "camera file path")).name: + _object(row, "camera file") + for row in files + if PurePosixPath( + _string(_object(row, "camera file").get("path"), "camera file path") + ).parent.name + in (f"epoch-{camera_job.codec_epoch}", "segments") + } + if ( + init.get("sha256") != _object(file_rows.get("init.mp4"), "camera init file").get("sha256") + or init.get("byte_length") + != _object(file_rows.get("init.mp4"), "camera init file").get("byte_length") + ): + raise PortableLabV1SourceError("camera init differs from the admitted source") + for index, row in enumerate(segments, start=1): + segment = _object(row, "source camera segment") + staged = _object(file_rows.get(f"{index}.m4s"), "camera segment file") + if ( + segment.get("sequence") != index + or segment.get("sha256") != staged.get("sha256") + or segment.get("byte_length") != staged.get("byte_length") + ): + raise PortableLabV1SourceError("camera segment differs from source admission") + + +def _verify_definition_and_job( + definition: PortableRunDefinition, + job: SealedObservatoryRecordedJob, +) -> None: + if ( + definition.setup_id != _EXPECTED_SETUP_ID + or definition.definition_id != _EXPECTED_DEFINITION_ID + or definition.result_contract.contract_sha256 + != _EXPECTED_RESULT_CONTRACT_SHA256 + or job.setup_id != definition.setup_id + or job.definition_id != definition.definition_id + or job.definition_version != definition.version + or job.definition_sha256 != definition.definition_sha256 + or job.source_adapter_id != definition.source_adapter.adapter_id + or job.source_adapter_version != definition.source_adapter.version + or job.source_adapter_sha256 != definition.source_adapter.contract_sha256 + or job.model_release_ids != definition.learned_models + or job.resource_profile_id != definition.resource_profile.profile_id + or job.executor_identity.model_manifest_sha256 + != definition.model_manifest_sha256 + or job.executor_identity.resource_profile_sha256 + != definition.resource_profile.profile_sha256 + or job.checkpoint_policy != definition.resource_profile.checkpoint_policy + or job.allowed_checkpoints != definition.resource_profile.allowed_checkpoints + ): + raise PortableLabV1SourceError("job is not the exact portable LAB V1 definition") + + +def _verify_source_and_job( + source: PortableLabV1SourceInput, + job: SealedObservatoryRecordedJob | ObservatoryRecordedJob, + definition: PortableRunDefinition, +) -> None: + """Re-bind a materialized descriptor before it can influence a plan. + + ``PortableLabV1SourceInput`` is intentionally path-free and can be persisted + between materialization and execution. Its constructor validates shape, + not provenance, so the orchestration boundary must repeat every sealed job + and RunDefinition equality instead of trusting an arbitrary instance. + """ + + requirements = definition.source_requirements + if ( + source.observatory_job_id != job.job_id + or source.observatory_request_sha256 != job.request_sha256 + or source.observatory_identity_sha256 != job.identity_sha256 + or source.source_session_id != job.source_session_id + or source.source_catalog_sha256 != job.source_catalog_sha256 + or source.source_bundle_sha256 != job.source_bundle_sha256 + or source.source_capability_manifest_sha256 + != job.source_capability_manifest_sha256 + or source.source_adapter_sha256 != job.source_adapter_sha256 + or source.camera_source_id != requirements.camera_source_id + or source.calibration_sha256 != requirements.calibration_identity_sha256 + ): + raise PortableLabV1PlanError( + "portable LAB V1 source descriptor differs from the sealed job" + ) + + +def _plan_identity_document( + *, + observatory_job_id: str, + observatory_request_sha256: str, + observatory_identity_sha256: str, + setup_id: str, + definition_id: str, + definition_version: int, + definition_sha256: str, + result_contract_sha256: str, + source_input_sha256: str, + release_candidate_sha256: str, + effective_ddrnet_config_sha256: str, + phases: Sequence[PortableLabV1PlanPhase], + release_blockers: Sequence[str], +) -> dict[str, object]: + return { + "schema_version": PORTABLE_LAB_V1_PLAN_IDENTITY_SCHEMA, + "observatory_job": { + "job_id": observatory_job_id, + "request_sha256": observatory_request_sha256, + "identity_sha256": observatory_identity_sha256, + }, + "run_definition": { + "setup_id": setup_id, + "definition_id": definition_id, + "version": definition_version, + "definition_sha256": definition_sha256, + "result_contract_sha256": result_contract_sha256, + }, + "source_input_sha256": source_input_sha256, + "release_candidate_sha256": release_candidate_sha256, + "release_admission": { + "ready": not release_blockers, + "blockers": list(release_blockers), + }, + "effective_ddrnet_config_sha256": effective_ddrnet_config_sha256, + "phases": [phase.as_dict() for phase in phases], + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + + +def _verify_plan_definition( + plan: PortableLabV1OrchestrationPlan, + definition: PortableRunDefinition, +) -> None: + if ( + plan.setup_id != definition.setup_id + or plan.definition_id != definition.definition_id + or plan.definition_version != definition.version + or plan.definition_sha256 != definition.definition_sha256 + or plan.result_contract_sha256 + != definition.result_contract.contract_sha256 + ): + raise PortableLabV1PlanError("orchestration plan belongs to another definition") + + +def _validate_eomt_component( + document: Mapping[str, object], + root: Path, + plan: PortableLabV1OrchestrationPlan, + definition: PortableRunDefinition, +) -> None: + identity = _object(document.get("identity"), "EoMT identity") + if ( + document.get("schema_version") != EOMT_RESULT_SCHEMA + or document.get("identity_sha256") != canonical_sha256(identity) + or document.get("result_id") != f"result-{document.get('identity_sha256')}" + or document.get("job_id") != plan.source_input.camera_job_id + or document.get("input_sha256") != plan.source_input.camera_input_sha256 + or document.get("session_id") != plan.source_input.source_session_id + or document.get("source_id") != plan.source_input.camera_source_id + or document.get("codec_epoch") != plan.source_input.codec_epoch + or document.get("timestamp_basis") != "session-time-seconds" + or document.get("timeline_start_seconds") + != plan.source_input.timeline_start_seconds + or document.get("timeline_end_seconds") != plan.source_input.timeline_end_seconds + or document.get("frames_processed") != plan.source_input.frame_count + or document.get("ground_truth") is not False + ): + raise PortableLabV1ResultError("EoMT component source identity changed") + models = _object(identity.get("models"), "EoMT identity models") + semantic = _object(models.get("semantic"), "EoMT semantic model") + eomt_model = _model(definition, _EOMT_RELEASE_ID) + if ( + semantic.get("id") != eomt_model.model_id + or semantic.get("revision") != eomt_model.revision + or semantic.get("architecture") != eomt_model.architecture + ): + raise PortableLabV1ResultError("EoMT component model identity changed") + configuration = _object(identity.get("configuration"), "EoMT configuration") + if configuration.get("pipeline") != _EOMT_PIPELINE: + raise PortableLabV1ResultError("EoMT component pipeline changed") + _eomt_component_artifacts(document, root) + _validate_decode_repair(root / "decode-repair.json", plan.source_input.frame_count) + + +def _eomt_component_artifacts( + document: Mapping[str, object], + root: Path, +) -> tuple[dict[str, object], ...]: + rows = document.get("artifacts") + if not isinstance(rows, list) or len(rows) != 5: + raise PortableLabV1ResultError("EoMT artifact set changed") + expected_kinds = { + "panoptic-overlay-video", + "panoptic-mask-archive", + "panoptic-frame-metadata", + "worker-gpu-telemetry", + "perception-run-report", + } + normalized: list[dict[str, object]] = [] + for row in rows: + artifact = _object(row, "EoMT artifact") + kind = _string(artifact.get("kind"), "EoMT artifact kind") + relative = _safe_relative_path(_string(artifact.get("path"), "EoMT artifact path")) + if len(relative.parts) != 1: + raise PortableLabV1ResultError("EoMT artifact is not a direct result child") + path = _required_component_file(root, relative.as_posix()) + if ( + artifact.get("byte_length") != path.stat().st_size + or artifact.get("sha256") != _sha256_file(path) + or not isinstance(artifact.get("media_type"), str) + ): + raise PortableLabV1ResultError("EoMT artifact identity changed") + normalized.append({**artifact, "kind": kind, "path": path}) + if {cast(str, row["kind"]) for row in normalized} != expected_kinds: + raise PortableLabV1ResultError("EoMT artifact roles changed") + return tuple(normalized) + + +def _validate_ddrnet_component( + document: Mapping[str, object], + root: Path, + plan: PortableLabV1OrchestrationPlan, + definition: PortableRunDefinition, +) -> None: + candidate = _object(document.get("candidate"), "DDRNet candidate") + source = _object(document.get("source"), "DDRNet source") + semantics = _object(document.get("video_semantics"), "DDRNet video semantics") + archive = _object(semantics.get("mask_archive"), "DDRNet mask archive") + provenance = _object(document.get("provenance"), "DDRNet provenance") + authority = _object(document.get("authority"), "DDRNet authority") + model = _model(definition, _DDRNET_RELEASE_ID) + checkpoint = model.artifacts[0] + components = {component.component_id: component for component in definition.components} + effective_source = _object( + plan.effective_ddrnet_config.get("ravnoves"), + "effective DDRNet source", + ) + if ( + document.get("schema_version") != DDRNET_RESULT_SCHEMA + or document.get("mode") != "ravnoves-video" + or candidate.get("candidate_id") != _DDRNET_CANDIDATE_ID + or candidate.get("candidate_key") != _DDRNET_CANDIDATE_KEY + or candidate.get("checkpoint_size_bytes") != checkpoint.byte_length + or candidate.get("checkpoint_sha256") != checkpoint.sha256 + or source.get("source_id") != effective_source.get("source_id") + or source.get("input_count") != plan.source_input.frame_count + or source.get("ground_truth_available") is not False + or source.get("mapping_sha256") != _GOOSE_MAPPING_SHA256 + or archive.get("frame_count") != plan.source_input.frame_count + or archive.get("width") != 800 + or archive.get("height") != 600 + or archive.get("encoding") != "uint8-class-id-png" + or semantics.get("center_crop_xyxy") != [100, 0, 700, 600] + or semantics.get("outside_crop_state") != "undefined" + or semantics.get("base_m4_result_id") is not None + or provenance.get("config_sha256") + != plan.effective_ddrnet_config_sha256 + or provenance.get("policy_sha256") + != components["vegetation-mission-policy-v1"].sha256 + or provenance.get("provider_map_sha256") + != components["vegetation-provider-label-map-v1"].sha256 + or authority + != { + "navigation_accepted": False, + "safety_accepted": False, + "actuation_accepted": False, + "camera_semantics_can_clear_rigid_geometry": False, + } + ): + raise PortableLabV1ResultError("DDRNet component contract changed") + identity_value = { + "schema_version": document["schema_version"], + "candidate": document["candidate"], + "source": document["source"], + "video_semantics": document["video_semantics"], + "preprocessing": document["preprocessing"], + "metrics": document["metrics"], + "timing": document["timing"], + "resource": document["resource"], + "visual_cases": document["visual_cases"], + "authority": document["authority"], + "config_sha256": provenance["config_sha256"], + "policy_sha256": provenance["policy_sha256"], + "provider_map_sha256": provenance["provider_map_sha256"], + } + expected_id = f"lab-v1-ravnoves-video-ddrnet-{canonical_sha256(identity_value)}" + if document.get("result_id") != expected_id: + raise PortableLabV1ResultError("DDRNet component result identity changed") + _ddrnet_mask_archive(document, root) + _validate_decode_repair(root / "decode-repair.json", plan.source_input.frame_count) + + +def _ddrnet_mask_archive(document: Mapping[str, object], root: Path) -> Path: + semantics = _object(document.get("video_semantics"), "DDRNet video semantics") + archive = _object(semantics.get("mask_archive"), "DDRNet mask archive") + relative = _safe_relative_path(_string(archive.get("path"), "DDRNet archive path")) + if len(relative.parts) != 1: + raise PortableLabV1ResultError("DDRNet archive is not a direct result child") + path = _required_component_file(root, relative.as_posix()) + if ( + archive.get("byte_length") != path.stat().st_size + or archive.get("sha256") != _sha256_file(path) + or archive.get("media_type") != "application/zip" + ): + raise PortableLabV1ResultError("DDRNet mask archive identity changed") + return path + + +def _validate_decode_repair(path: Path, frame_count: int) -> None: + document = _read_json_file(path, path.parent, "decode repair") + repairs = document.get("repairs") + repaired = document.get("repaired_frame_count") + if ( + document.get("schema_version") != DECODE_REPAIR_SCHEMA + or document.get("packets_requested") != frame_count + or not isinstance(repaired, int) + or isinstance(repaired, bool) + or repaired < 0 + or repaired > 1 + or not isinstance(repairs, list) + or len(repairs) != repaired + ): + raise PortableLabV1ResultError("decode repair contract changed") + + +def _result_identity_document( + *, + plan: PortableLabV1OrchestrationPlan, + definition: PortableRunDefinition, + components: Mapping[str, object], + artifacts: Sequence[Mapping[str, object]], +) -> dict[str, object]: + return { + "schema_version": PORTABLE_LAB_V1_RESULT_IDENTITY_SCHEMA, + "observatory_job_id": plan.observatory_job_id, + "source_input_sha256": plan.source_input.identity_sha256, + "plan_sha256": plan.plan_sha256, + "setup_id": definition.setup_id, + "definition_sha256": definition.definition_sha256, + "result_contract_sha256": definition.result_contract.contract_sha256, + "components": dict(components), + "artifacts": list(artifacts), + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + + +def _validate_assembly( + assembly: PortableLabV1ResultAssembly, + *, + plan: PortableLabV1OrchestrationPlan, + definition: PortableRunDefinition, +) -> None: + _verify_plan_definition(plan, definition) + document = dict(assembly.result_document) + _validate_v2_document_shape(document) + identity = _object(document["identity"], "portable LAB V1 identity") + if ( + canonical_sha256(identity) != document["identity_sha256"] + or document["result_id"] != assembly.result_id + or document["result_id"] + != f"lab-v1-eomt-ddrnet-{document['identity_sha256']}" + ): + raise PortableLabV1ResultError("assembled result identity changed") + for artifact in assembly.artifacts: + path = assembly.root / Path(*PurePosixPath(artifact.relative_path).parts) + if ( + not path.is_file() + or path.is_symlink() + or path.stat().st_size != artifact.byte_length + or _sha256_file(path) != artifact.sha256 + ): + raise PortableLabV1ResultError("assembled artifact content changed") + + +def _validate_v2_document_shape(document: Mapping[str, object]) -> None: + _exact_keys( + document, + { + "schema_version", + "result_id", + "result_kind", + "identity_sha256", + "identity", + "source", + "run_definition", + "components", + "qualification", + "artifacts", + "authority", + }, + "portable LAB V1 result", + ) + if document["schema_version"] != PORTABLE_LAB_V1_RESULT_SCHEMA: + raise PortableLabV1ResultError("portable LAB V1 result schema changed") + qualification = _object(document["qualification"], "LAB V1 qualification") + if qualification != { + "mode": "recorded-observation", + "ground_truth_available": False, + "navigation_or_safety_accepted": False, + "production_accepted": False, + }: + raise PortableLabV1ResultError("portable LAB V1 qualification claim changed") + + +def _result_identity_from_document(document: Mapping[str, object]) -> dict[str, object]: + run_definition = _object(document["run_definition"], "run definition") + identity = _object(document["identity"], "identity") + return { + "schema_version": PORTABLE_LAB_V1_RESULT_IDENTITY_SCHEMA, + "observatory_job_id": identity.get("observatory_job_id"), + "source_input_sha256": identity.get("source_input_sha256"), + "plan_sha256": run_definition.get("plan_sha256"), + "setup_id": run_definition.get("setup_id"), + "definition_sha256": run_definition.get("definition_sha256"), + "result_contract_sha256": run_definition.get("result_contract_sha256"), + "components": document["components"], + "artifacts": document["artifacts"], + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + + +def _validate_component_documents_from_context( + context: PortableResultValidationContext, + *, + document: Mapping[str, object], + definition: PortableRunDefinition, + plan: PortableLabV1OrchestrationPlan, +) -> None: + source = _object(document["source"], "portable source") + eomt = _read_json_file( + context.artifact_paths["eomt-result-document"], + context.artifact_paths["eomt-result-document"].parent, + "published EoMT result", + ) + ddrnet = _read_json_file( + context.artifact_paths["ddrnet-result-document"], + context.artifact_paths["ddrnet-result-document"].parent, + "published DDRNet result", + ) + components = _object(document["components"], "portable components") + _exact_keys(components, {"eomt", "ddrnet"}, "portable components") + eomt_component = _object(components.get("eomt"), "EoMT component") + ddrnet_component = _object(components.get("ddrnet"), "DDRNet component") + expected_component_keys = { + "result_id", + "result_document_sha256", + "model_release_id", + "frames_processed", + } + _exact_keys(eomt_component, expected_component_keys, "EoMT component") + _exact_keys(ddrnet_component, expected_component_keys, "DDRNet component") + if ( + eomt.get("result_id") != eomt_component.get("result_id") + or _sha256_file(context.artifact_paths["eomt-result-document"]) + != eomt_component.get("result_document_sha256") + or eomt.get("frames_processed") != source.get("frame_count") + or ddrnet.get("result_id") != ddrnet_component.get("result_id") + or _sha256_file(context.artifact_paths["ddrnet-result-document"]) + != ddrnet_component.get("result_document_sha256") + or _object(ddrnet.get("source"), "DDRNet source").get("input_count") + != source.get("frame_count") + or eomt_component.get("frames_processed") != source.get("frame_count") + or ddrnet_component.get("frames_processed") != source.get("frame_count") + ): + raise PortableLabV1ResultError("published component binding changed") + eomt_model = _model(definition, _EOMT_RELEASE_ID) + ddrnet_model = _model(definition, _DDRNET_RELEASE_ID) + if ( + eomt_component.get("model_release_id") != eomt_model.release_id + or ddrnet_component.get("model_release_id") != ddrnet_model.release_id + ): + raise PortableLabV1ResultError("published component model identity changed") + _validate_published_eomt_component( + context, + document=eomt, + plan=plan, + definition=definition, + ) + _validate_published_ddrnet_component( + context, + document=ddrnet, + plan=plan, + definition=definition, + ) + _validate_decode_repair( + context.artifact_paths["eomt-decode-repair"], + _positive_int(source.get("frame_count"), "published frame count"), + ) + _validate_decode_repair( + context.artifact_paths["ddrnet-decode-repair"], + _positive_int(source.get("frame_count"), "published frame count"), + ) + + +def _validate_published_eomt_component( + context: PortableResultValidationContext, + *, + document: Mapping[str, object], + plan: PortableLabV1OrchestrationPlan, + definition: PortableRunDefinition, +) -> None: + identity = _object(document.get("identity"), "published EoMT identity") + models = _object(identity.get("models"), "published EoMT models") + semantic = _object(models.get("semantic"), "published EoMT semantic model") + configuration = _object( + identity.get("configuration"), + "published EoMT configuration", + ) + model = _model(definition, _EOMT_RELEASE_ID) + if ( + document.get("schema_version") != EOMT_RESULT_SCHEMA + or document.get("identity_sha256") != canonical_sha256(identity) + or document.get("result_id") != f"result-{document.get('identity_sha256')}" + or document.get("job_id") != plan.source_input.camera_job_id + or document.get("input_sha256") != plan.source_input.camera_input_sha256 + or document.get("session_id") != plan.source_input.source_session_id + or document.get("source_id") != plan.source_input.camera_source_id + or document.get("codec_epoch") != plan.source_input.codec_epoch + or document.get("timestamp_basis") != "session-time-seconds" + or document.get("timeline_start_seconds") + != plan.source_input.timeline_start_seconds + or document.get("timeline_end_seconds") + != plan.source_input.timeline_end_seconds + or document.get("frames_processed") != plan.source_input.frame_count + or document.get("ground_truth") is not False + or semantic.get("id") != model.model_id + or semantic.get("revision") != model.revision + or semantic.get("architecture") != model.architecture + or configuration.get("pipeline") != _EOMT_PIPELINE + ): + raise PortableLabV1ResultError("published EoMT contract changed") + rows = document.get("artifacts") + expected_kinds = { + "panoptic-overlay-video", + "panoptic-mask-archive", + "panoptic-frame-metadata", + "worker-gpu-telemetry", + "perception-run-report", + } + if not isinstance(rows, list) or len(rows) != len(expected_kinds): + raise PortableLabV1ResultError("published EoMT artifact set changed") + package_by_role = { + artifact.role: artifact for artifact in context.manifest.artifacts + } + observed_kinds: set[str] = set() + for value in rows: + row = _object(value, "published EoMT artifact") + kind = _string(row.get("kind"), "published EoMT artifact kind") + if kind not in expected_kinds or kind in observed_kinds: + raise PortableLabV1ResultError("published EoMT artifact roles changed") + observed_kinds.add(kind) + relative = _safe_relative_path( + _string(row.get("path"), "published EoMT artifact path") + ) + if len(relative.parts) != 1: + raise PortableLabV1ResultError( + "published EoMT artifact path changed" + ) + packaged = package_by_role.get(f"eomt-{kind}") + if ( + packaged is None + or row.get("media_type") != packaged.media_type + or row.get("byte_length") != packaged.byte_length + or row.get("sha256") != packaged.sha256 + ): + raise PortableLabV1ResultError( + "published EoMT artifact identity changed" + ) + + +def _validate_published_ddrnet_component( + context: PortableResultValidationContext, + *, + document: Mapping[str, object], + plan: PortableLabV1OrchestrationPlan, + definition: PortableRunDefinition, +) -> None: + candidate = _object(document.get("candidate"), "published DDRNet candidate") + source = _object(document.get("source"), "published DDRNet source") + semantics = _object( + document.get("video_semantics"), + "published DDRNet video semantics", + ) + archive = _object( + semantics.get("mask_archive"), + "published DDRNet mask archive", + ) + provenance = _object( + document.get("provenance"), + "published DDRNet provenance", + ) + authority = _object(document.get("authority"), "published DDRNet authority") + model = _model(definition, _DDRNET_RELEASE_ID) + checkpoint = model.artifacts[0] + effective_source = _object( + plan.effective_ddrnet_config.get("ravnoves"), + "published effective DDRNet source", + ) + components = {component.component_id: component for component in definition.components} + package_archive = next( + ( + artifact + for artifact in context.manifest.artifacts + if artifact.role == "ddrnet-semantic-mask-archive" + ), + None, + ) + if ( + document.get("schema_version") != DDRNET_RESULT_SCHEMA + or document.get("mode") != "ravnoves-video" + or candidate.get("candidate_id") != _DDRNET_CANDIDATE_ID + or candidate.get("candidate_key") != _DDRNET_CANDIDATE_KEY + or candidate.get("loaded_model_name") != model.architecture + or candidate.get("checkpoint_size_bytes") != checkpoint.byte_length + or candidate.get("checkpoint_sha256") != checkpoint.sha256 + or source.get("source_id") != effective_source.get("source_id") + or source.get("input_count") != plan.source_input.frame_count + or source.get("ground_truth_available") is not False + or source.get("mapping_sha256") != _GOOSE_MAPPING_SHA256 + or archive.get("frame_count") != plan.source_input.frame_count + or archive.get("width") != 800 + or archive.get("height") != 600 + or archive.get("encoding") != "uint8-class-id-png" + or semantics.get("center_crop_xyxy") != [100, 0, 700, 600] + or semantics.get("outside_crop_state") != "undefined" + or semantics.get("base_m4_result_id") is not None + or provenance.get("config_sha256") + != plan.effective_ddrnet_config_sha256 + or provenance.get("policy_sha256") + != components["vegetation-mission-policy-v1"].sha256 + or provenance.get("provider_map_sha256") + != components["vegetation-provider-label-map-v1"].sha256 + or package_archive is None + or archive.get("media_type") != package_archive.media_type + or archive.get("byte_length") != package_archive.byte_length + or archive.get("sha256") != package_archive.sha256 + or authority + != { + "navigation_accepted": False, + "safety_accepted": False, + "actuation_accepted": False, + "camera_semantics_can_clear_rigid_geometry": False, + } + ): + raise PortableLabV1ResultError("published DDRNet contract changed") + identity_value = { + "schema_version": document["schema_version"], + "candidate": document["candidate"], + "source": document["source"], + "video_semantics": document["video_semantics"], + "preprocessing": document["preprocessing"], + "metrics": document["metrics"], + "timing": document["timing"], + "resource": document["resource"], + "visual_cases": document["visual_cases"], + "authority": document["authority"], + "config_sha256": provenance["config_sha256"], + "policy_sha256": provenance["policy_sha256"], + "provider_map_sha256": provenance["provider_map_sha256"], + } + if document.get("result_id") != ( + "lab-v1-ravnoves-video-ddrnet-" + canonical_sha256(identity_value) + ): + raise PortableLabV1ResultError("published DDRNet result identity changed") +def _model(definition: PortableRunDefinition, release_id: str): # type: ignore[no-untyped-def] + for model in definition.models: + if model.release_id == release_id: + return model + raise PortableLabV1ResultError(f"required model release is absent: {release_id}") + + +def _representative_frame_indices(frame_count: int) -> list[int]: + if frame_count <= 12: + return list(range(frame_count)) + return sorted({round(index * (frame_count - 1) / 11) for index in range(12)}) + + +def _release_asset(value: object) -> PortableLabV1ReleaseAsset: + row = _object(value, "release asset") + _exact_keys( + row, + {"asset_id", "kind", "sha256", "byte_length", "repository_path"}, + "release asset", + ) + kind = _string(row["kind"], "release asset kind") + path = row["repository_path"] + if path is not None and not isinstance(path, str): + raise PortableLabV1ReleaseError("release repository path is invalid") + byte_length = row["byte_length"] + if byte_length is not None and ( + isinstance(byte_length, bool) or not isinstance(byte_length, int) + ): + raise PortableLabV1ReleaseError("release asset byte length is invalid") + return PortableLabV1ReleaseAsset( + asset_id=_string(row["asset_id"], "release asset id"), + kind=cast(ReleaseAssetKind, kind), + sha256=_string(row["sha256"], "release asset sha256"), + byte_length=byte_length, + repository_path=path, + ) + + +def _release_binding_matches( + asset: PortableLabV1ReleaseAsset, + binding: ReleaseAssetBinding, +) -> bool: + if isinstance(binding, str): + return asset.byte_length is None and binding == asset.sha256 + try: + path = binding.expanduser().absolute() + if path.is_symlink() or not path.is_file(): + return False + return ( + (asset.byte_length is None or path.stat().st_size == asset.byte_length) + and _sha256_file(path) == asset.sha256 + ) + except OSError: + return False + + +def _artifact_from_document(value: object) -> PortableResultArtifact: + row = _object(value, "portable result artifact") + _exact_keys( + row, + {"role", "relative_path", "media_type", "byte_length", "sha256"}, + "portable result artifact", + ) + return PortableResultArtifact( + role=_string(row["role"], "artifact role"), + relative_path=_string(row["relative_path"], "artifact path"), + media_type=_string(row["media_type"], "artifact media type"), + byte_length=_positive_int(row["byte_length"], "artifact byte length", allow_zero=True), + sha256=_string(row["sha256"], "artifact sha256"), + ) + + +def _canonical_document(payload: bytes, label: str, *, maximum: int) -> dict[str, object]: + document = _decoded_document(payload, label, maximum=maximum) + canonical = canonical_json(document) + if payload not in (canonical, canonical + b"\n"): + raise PortableLabV1Error(f"{label} is not canonical JSON") + return document + + +def _decoded_document(payload: bytes, label: str, *, maximum: int) -> dict[str, object]: + if not 0 < len(payload) <= maximum: + raise PortableLabV1Error(f"{label} size is invalid") + try: + decoded: object = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PortableLabV1Error(f"{label} is not valid JSON") from exc + return _object(decoded, label) + + +def _read_json_file( + path: Path, + root: Path, + label: str, + *, + require_canonical: bool = False, +) -> dict[str, object]: + payload = _read_bounded_regular_file(path, root, label) + try: + decoded: object = json.loads(payload.decode("utf-8-sig")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PortableLabV1Error(f"{label} is not valid JSON") from exc + document = _object(decoded, label) + if require_canonical and canonical_json(document) != payload: + raise PortableLabV1Error(f"{label} is not canonical JSON") + return document + + +def _read_bounded_regular_file( + path: Path, + root: Path, + label: str, + *, + maximum: int = _MAX_DOCUMENT_BYTES, +) -> bytes: + try: + resolved_root = root.resolve(strict=True) + resolved = path.resolve(strict=True) + metadata = path.lstat() + except OSError as exc: + raise PortableLabV1Error(f"{label} is unavailable") from exc + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or not resolved.is_relative_to(resolved_root) + or not 0 < metadata.st_size <= maximum + ): + raise PortableLabV1Error(f"{label} is not a confined bounded file") + return path.read_bytes() + + +def _required_component_file(root: Path, name: str) -> Path: + relative = _safe_relative_path(name) + path = root / Path(*relative.parts) + _read_bounded_regular_file(path, root, f"component file {name}", maximum=1 << 63) + return path + + +def _direct_real_directories(root: Path) -> tuple[Path, ...]: + parent = _real_directory(root, "camera job parent") + result: list[Path] = [] + for child in parent.iterdir(): + if child.is_symlink() or not child.is_dir(): + raise PortableLabV1SourceError("camera job parent contains an invalid member") + resolved = child.resolve(strict=True) + if resolved.parent != parent: + raise PortableLabV1SourceError("camera job member escapes its parent") + result.append(resolved) + return tuple(sorted(result)) + + +def _real_directory(path: Path, label: str) -> Path: + try: + metadata = path.lstat() + resolved = path.resolve(strict=True) + except OSError as exc: + raise PortableLabV1Error(f"{label} is unavailable") from exc + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise PortableLabV1Error(f"{label} is not a real directory") + return resolved + + +def _prepare_real_directory(path: Path, label: str) -> Path: + path.mkdir(mode=0o700, parents=True, exist_ok=True) + root = _real_directory(path, label) + os.chmod(root, 0o700) + return root + + +def _safe_relative_path(value: str) -> PurePosixPath: + if not value or "\\" in value: + raise PortableLabV1Error("relative path is invalid") + path = PurePosixPath(value) + if path.is_absolute() or ".." in path.parts or not path.parts: + raise PortableLabV1Error("relative path escapes its root") + return path + + +def _copy_exact_file(source: Path, destination: Path) -> None: + if source.is_symlink() or not source.is_file(): + raise PortableLabV1ResultError("result source artifact is not a regular file") + destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + with source.open("rb") as reader, destination.open("xb") as writer: + shutil.copyfileobj(reader, writer, length=1024 * 1024) + writer.flush() + os.fsync(writer.fileno()) + os.chmod(destination, 0o600) + + +def _write_exact_bytes(path: Path, payload: bytes) -> None: + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + with path.open("xb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(path, 0o600) + + +def _source_suffix(value: bytes | Path) -> str: + if isinstance(value, bytes): + return ".bin" + suffix = value.suffix.lower() + if not suffix or len(suffix) > 16 or not re.fullmatch(r"\.[a-z0-9]+", suffix): + return ".bin" + return suffix + + +def _sha256_file(path: Path) -> str: + digest_value = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest_value.update(chunk) + return digest_value.hexdigest() + + +def _fsync_tree(root: Path) -> None: + directories = sorted( + (path for path in root.rglob("*") if path.is_dir()), + key=lambda path: len(path.parts), + reverse=True, + ) + for directory in directories: + _fsync_directory(directory) + _fsync_directory(root) + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _exact_keys(value: Mapping[str, object], expected: set[str], label: str) -> None: + if set(value) != expected: + raise PortableLabV1Error(f"{label} fields are invalid") + + +def _object(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise PortableLabV1Error(f"{label} is not an object") + return cast(dict[str, object], value) + + +def _string(value: object, label: str) -> str: + if not isinstance(value, str) or not value: + raise PortableLabV1Error(f"{label} is not a string") + return value + + +def _positive_int(value: object, label: str, *, allow_zero: bool = False) -> int: + minimum = 0 if allow_zero else 1 + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise PortableLabV1Error(f"{label} is not an integer") + return value + + +def _pattern(value: str, pattern: re.Pattern[str], label: str) -> None: + if pattern.fullmatch(value) is None: + raise PortableLabV1Error(f"{label} is invalid") + + +def _digest(value: str, label: str) -> None: + _pattern(value, _SHA256, label) + + +def _finite_number(value: object) -> bool: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and float("-inf") < float(value) < float("inf") + ) + + +def _finite_float(value: object, label: str) -> float: + if not _finite_number(value): + raise PortableLabV1Error(f"{label} is not a finite number") + return float(cast(int | float, value)) diff --git a/src/k1link/observatory/portable_lab_v1_worker.py b/src/k1link/observatory/portable_lab_v1_worker.py new file mode 100644 index 0000000..1c91617 --- /dev/null +++ b/src/k1link/observatory/portable_lab_v1_worker.py @@ -0,0 +1,1225 @@ +"""Worker-local bridge for the portable LAB V1 EoMT plus DDRNet profile. + +The shared Worker transport deliberately materializes a small, fixed source +layout. Historical EoMT tooling, however, consumes a validated +``CameraComputeJob``. This module closes that boundary without accepting a +path, command, image, model, or configuration from a queued job: + +* the shared materialization manifest and every selected camera member are + rebound to the exact sealed claim; +* a deterministic camera compute job is published from those admitted bytes; +* Worker-local EoMT and DDRNet ports are invoked sequentially; +* the exact LAB V1 v2 assembler and portable package contract produce the + ``PortableWorkerResultDraft`` uploaded by the shared result transport. + +This is composition code, not an installed executor. The checked-in LAB V1 +release and runtime candidates remain blocked until their assets, combined +image, release receipt, and Worker registration are independently sealed. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import secrets +import shutil +import stat +import tempfile +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Final, Literal, Protocol, cast + +from k1link.compute.jobs import ( + COMPUTE_JOB_SCHEMA, + COMPUTE_PROFILE, + COMPUTE_RESULT_SCHEMA, + CameraComputeJob, + validate_camera_compute_job, +) +from k1link.observatory.portable_lab_v1_executor import ( + PortableLabV1MaterializedSource, + PortableLabV1OrchestrationPlan, + PortableLabV1ReleaseCandidate, + PortableLabV1ReleaseInspection, + PortableLabV1SourceError, + assemble_lab_v1_result_v2, + materialize_lab_v1_source_input, + package_lab_v1_result, +) +from k1link.observatory.portable_result_contract import ( + OBSERVATION_ONLY_AUTHORITY, + canonical_json, +) +from k1link.observatory.portable_run_definitions import PortableRunDefinition +from k1link.observatory.portable_worker_runtime import ( + PortableWorkerExecutorAdapter, + PortableWorkerResultDraft, + PortableWorkerResultPublisher, + PortableWorkerRuntimeAdmission, + PortableWorkerRuntimeCandidate, + PortableWorkerRuntimeJobRejectedError, + PortableWorkerRuntimePlan, + PortableWorkerRuntimeUnavailableError, + PortableWorkerSourceMaterializer, + PortableWorkerSourceStage, +) +from k1link.observatory.recorded_jobs import ObservatoryRecordedJob +from k1link.observatory.source_admission import PORTABLE_SOURCE_BUNDLE_SCHEMA +from k1link.observatory.worker_agent import SealedObservatoryRecordedJob + +PORTABLE_LAB_V1_CAMERA_SUMMARY_SCHEMA: Final = ( + "missioncore.observatory-portable-camera-epoch-summary/v1" +) +PORTABLE_LAB_V1_CAMERA_INDEX_SCHEMA: Final = ( + "missioncore.observatory-portable-camera-epoch-index/v1" +) +PORTABLE_SOURCE_MATERIALIZATION_SCHEMA: Final = ( + "missioncore.observatory-portable-source-materialization/v1" +) +PORTABLE_LAB_V1_RUNTIME_PHASES: Final = ( + "source-delivery", + "eomt-runtime", + "ddrnet-portable-runtime", + "portable-lab-orchestrator", + "result-v2-assembler", + "observatory-result-publisher", +) + +_EXPECTED_SETUP_ID: Final = "lab-v1-eomt-ddrnet-portable-v1" +_EXPECTED_DEFINITION_ID: Final = "lab-v1-eomt-ddrnet-portable" +_EXPECTED_RESULT_CONTRACT_SHA256: Final = ( + "b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a" +) +_PORTABLE_CONFIG_ASSET_ID: Final = "ddrnet-portable-config" +_MAX_SOURCE_DOCUMENT_BYTES: Final = 8 * 1024 * 1024 +_MAX_MATERIALIZATION_MANIFEST_BYTES: Final = 64 * 1024 * 1024 +_MAX_SOURCE_MEMBERS: Final = 100_000 +_MAX_SOURCE_BYTES: Final = 2 * 1024 * 1024 * 1024 * 1024 +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_SAFE_COMPONENT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +_MEMBER_KIND = { + "source-bundle", + "source-capability", + "spatial-replay", + "spatial-replay-metadata", + "camera-init", + "camera-segment", +} + +type SourceMemberKind = Literal[ + "source-bundle", + "source-capability", + "spatial-replay", + "spatial-replay-metadata", + "camera-init", + "camera-segment", +] + + +class PortableLabV1WorkerError(PortableWorkerRuntimeUnavailableError): + """The portable LAB V1 Worker bridge is unavailable or changed.""" + + +class PortableLabV1EomtRunner(Protocol): + """Installed, release-owned EoMT component port.""" + + def __call__( + self, + *, + source: PortableLabV1MaterializedSource, + plan: PortableLabV1OrchestrationPlan, + output_root: Path, + ) -> None: ... + + +class PortableLabV1DdrnetRunner(Protocol): + """Installed, release-owned DDRNet component port.""" + + def __call__( + self, + *, + source: PortableLabV1MaterializedSource, + plan: PortableLabV1OrchestrationPlan, + effective_config_path: Path, + eomt_result_root: Path, + output_root: Path, + ) -> None: ... + + +@dataclass(frozen=True, slots=True) +class _WorkerSourceMember: + member_id: str + kind: SourceMemberKind + media_type: str + byte_length: int + sha256: str + artifact_id: str | None + primary: bool + camera_epoch: int | None + camera_sequence: int | None + + +@dataclass(frozen=True, slots=True) +class _CameraContract: + artifact_id: str + public_source_id: str + generation_sha256: str + synchronization: str + epoch: int + media_type: str + timeline_start_seconds: float + timeline_end_seconds: float + init: _WorkerSourceMember + segments: tuple[_WorkerSourceMember, ...] + init_path: Path + segment_paths: tuple[Path, ...] + + +@dataclass(frozen=True, slots=True) +class _SealedPackageJobView: + """Exact package fields projected from a sealed, not-yet-finished job.""" + + job_id: str + request_sha256: str + identity_sha256: str + submission_receipt_sha256: str + claim_generation: int + source_session_id: str + source_catalog_sha256: str + source_bundle_sha256: str + source_capability_manifest_sha256: str + source_adapter_id: str + source_adapter_version: int + source_adapter_sha256: str + result_id: None = None + + @classmethod + def from_job( + cls, + job: SealedObservatoryRecordedJob, + ) -> _SealedPackageJobView: + return cls( + job_id=job.job_id, + request_sha256=job.request_sha256, + identity_sha256=job.identity_sha256, + submission_receipt_sha256=job.submission_receipt_sha256, + claim_generation=job.claim_generation, + source_session_id=job.source_session_id, + source_catalog_sha256=job.source_catalog_sha256, + source_bundle_sha256=job.source_bundle_sha256, + source_capability_manifest_sha256=(job.source_capability_manifest_sha256), + source_adapter_id=job.source_adapter_id, + source_adapter_version=job.source_adapter_version, + source_adapter_sha256=job.source_adapter_sha256, + ) + + +@dataclass(frozen=True, slots=True) +class PortableLabV1BoundSourceStage(PortableWorkerSourceStage): + """Claim-bound in-memory extension of the shared source-stage port.""" + + job: SealedObservatoryRecordedJob + lab_source: PortableLabV1MaterializedSource + + def __post_init__(self) -> None: + PortableWorkerSourceStage.__post_init__(self) + descriptor = self.lab_source.descriptor + if ( + self.root != self.lab_source.root + or descriptor.observatory_job_id != self.job.job_id + or descriptor.observatory_request_sha256 != self.job.request_sha256 + or descriptor.observatory_identity_sha256 != self.job.identity_sha256 + or self.source_bundle_sha256 != self.job.source_bundle_sha256 + or self.source_capability_manifest_sha256 != self.job.source_capability_manifest_sha256 + or self.source_adapter_sha256 != self.job.source_adapter_sha256 + ): + raise PortableLabV1WorkerError( + "portable LAB V1 bound source differs from its sealed job" + ) + + +@dataclass(frozen=True, slots=True) +class PortableLabV1SourceMaterializerAdapter: + """Adapt the shared Worker source layout to one exact camera compute job.""" + + upstream: PortableWorkerSourceMaterializer + definition: PortableRunDefinition + output_parent: Path + + def __post_init__(self) -> None: + _verify_definition(self.definition) + object.__setattr__( + self, + "output_parent", + _prepare_real_directory( + self.output_parent, + "portable LAB V1 source output parent", + ), + ) + + def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage: + delivered = self.upstream.materialize(job) + materialized = materialize_lab_v1_source_from_worker_stage( + worker_stage=delivered, + job=job, + definition=self.definition, + output_parent=self.output_parent, + ) + return PortableLabV1BoundSourceStage( + root=materialized.root, + source_bundle_sha256=job.source_bundle_sha256, + source_capability_manifest_sha256=(job.source_capability_manifest_sha256), + source_adapter_sha256=job.source_adapter_sha256, + job=job, + lab_source=materialized, + ) + + +@dataclass(frozen=True, slots=True) +class PortableLabV1RunnerInstallation: + """Exact release inspection and local output roots selected by an installer.""" + + release: PortableLabV1ReleaseCandidate + inspection: PortableLabV1ReleaseInspection + portable_ddrnet_config_path: Path + output_parent: Path + + def __post_init__(self) -> None: + expected_assets = tuple(asset.asset_id for asset in self.release.assets) + if ( + self.inspection.candidate_sha256 != self.release.candidate_sha256 + or not self.inspection.ready + or self.inspection.blockers + or self.inspection.matched_assets != expected_assets + or self.release.executor_image_sha256 is None + or self.release.declared_blockers + ): + raise PortableLabV1WorkerError( + "portable LAB V1 release inspection is not fully admitted" + ) + self.release.seal(self.inspection) + config_asset = next( + (asset for asset in self.release.assets if asset.asset_id == _PORTABLE_CONFIG_ASSET_ID), + None, + ) + if config_asset is None: + raise PortableLabV1WorkerError( + "portable LAB V1 release has no exact portable DDRNet config" + ) + config = _exact_file( + self.portable_ddrnet_config_path, + expected_sha256=config_asset.sha256, + expected_byte_length=config_asset.byte_length, + label="portable LAB V1 portable DDRNet config", + ) + _read_json_object(config, "portable LAB V1 portable DDRNet config") + output = _prepare_real_directory( + self.output_parent, + "portable LAB V1 result output parent", + ) + object.__setattr__(self, "portable_ddrnet_config_path", config) + object.__setattr__(self, "output_parent", output) + + +@dataclass(frozen=True, slots=True) +class PortableLabV1ProfileRunnerAdapter: + """Execute EoMT then DDRNet and package their exact validated outputs.""" + + definition: PortableRunDefinition + installation: PortableLabV1RunnerInstallation + created_at_utc: Callable[[], str] + eomt_runner: PortableLabV1EomtRunner + ddrnet_runner: PortableLabV1DdrnetRunner + + def __post_init__(self) -> None: + _verify_definition(self.definition) + self.installation.release.bind_definition(self.definition) + if not callable(self.created_at_utc): + raise ValueError("portable LAB V1 clock is not callable") + + def run( + self, + plan: PortableWorkerRuntimePlan, + source: PortableWorkerSourceStage, + ) -> PortableWorkerResultDraft: + if not isinstance(source, PortableLabV1BoundSourceStage): + raise PortableWorkerRuntimeJobRejectedError( + "portable LAB V1 runner requires its claim-bound source stage" + ) + job = source.job + _verify_runtime_plan(plan, job=job, definition=self.definition) + _verify_installation_unchanged(self.installation) + portable_config = _read_json_object( + self.installation.portable_ddrnet_config_path, + "portable LAB V1 portable DDRNet config", + ) + lab_plan = PortableLabV1OrchestrationPlan.create( + job=job, + definition=self.definition, + source=source.lab_source.descriptor, + release=self.installation.release, + release_inspection=self.installation.inspection, + legacy_ddrnet_config=portable_config, + ) + lab_plan.require_executable() + + workspace = Path( + tempfile.mkdtemp( + prefix=".lab-v1-portable-run-", + dir=self.installation.output_parent, + ) + ) + try: + effective_config_path = workspace / "effective-ddrnet-config.json" + _write_exact(effective_config_path, canonical_json(lab_plan.effective_ddrnet_config)) + if _sha256_file(effective_config_path) != lab_plan.effective_ddrnet_config_sha256: + raise PortableLabV1WorkerError( + "portable LAB V1 effective DDRNet config identity changed" + ) + eomt_root = workspace / "eomt-result" + ddrnet_root = workspace / "ddrnet-result" + self.eomt_runner( + source=source.lab_source, + plan=lab_plan, + output_root=eomt_root, + ) + _require_component_result(eomt_root, "portable LAB V1 EoMT result") + self.ddrnet_runner( + source=source.lab_source, + plan=lab_plan, + effective_config_path=effective_config_path, + eomt_result_root=eomt_root, + output_root=ddrnet_root, + ) + _require_component_result(ddrnet_root, "portable LAB V1 DDRNet result") + validate_camera_compute_job(source.lab_source.camera_job_root) + _verify_installation_unchanged(self.installation) + assembly = assemble_lab_v1_result_v2( + plan=lab_plan, + definition=self.definition, + eomt_result_root=eomt_root, + ddrnet_result_root=ddrnet_root, + output_parent=self.installation.output_parent / "assemblies", + ) + return package_lab_v1_result( + assembly=assembly, + plan=lab_plan, + job=cast( + ObservatoryRecordedJob, + _SealedPackageJobView.from_job(job), + ), + definition=self.definition, + created_at_utc=self.created_at_utc(), + output_parent=self.installation.output_parent / "packages", + ) + finally: + shutil.rmtree(workspace, ignore_errors=True) + + +def materialize_lab_v1_source_from_worker_stage( + *, + worker_stage: PortableWorkerSourceStage, + job: SealedObservatoryRecordedJob, + definition: PortableRunDefinition, + output_parent: Path, +) -> PortableLabV1MaterializedSource: + """Build a deterministic camera job from only manifested Worker members.""" + + _verify_definition(definition) + if ( + worker_stage.source_bundle_sha256 != job.source_bundle_sha256 + or worker_stage.source_capability_manifest_sha256 != job.source_capability_manifest_sha256 + or worker_stage.source_adapter_sha256 != job.source_adapter_sha256 + ): + raise PortableLabV1SourceError("Worker source stage belongs to another sealed LAB V1 job") + root = _real_directory(worker_stage.root, "portable Worker source stage") + bundle_bytes, bundle = _read_canonical_document( + root / "source-bundle.json", + root=root, + expected_sha256=job.source_bundle_sha256, + label="portable source bundle", + ) + capability_bytes, _capability = _read_canonical_document( + root / "source-capability.json", + root=root, + expected_sha256=job.source_capability_manifest_sha256, + label="portable source capability", + ) + manifest_bytes, manifest = _read_canonical_document( + root / "materialization-manifest.json", + root=root, + expected_sha256=None, + label="portable source materialization manifest", + maximum=_MAX_MATERIALIZATION_MANIFEST_BYTES, + ) + del manifest_bytes + camera = _verify_materialization_manifest( + root, + manifest, + bundle=bundle, + bundle_byte_length=len(bundle_bytes), + capability_byte_length=len(capability_bytes), + job=job, + ) + parent = _prepare_real_directory(output_parent, "portable LAB V1 source output parent") + stage_root = _prepare_real_directory( + parent / job.source_bundle_sha256, + "portable LAB V1 digest source stage", + ) + if stage_root.parent != parent: + raise PortableLabV1SourceError("portable LAB V1 source stage is not a direct digest child") + camera_job = _publish_camera_compute_job( + camera=camera, + job=job, + output_root=stage_root / "camera-job", + ) + descriptor = materialize_lab_v1_source_input( + job=job, + definition=definition, + camera_job=camera_job, + source_bundle_bytes=bundle_bytes, + capability_bytes=capability_bytes, + ) + return PortableLabV1MaterializedSource( + root=stage_root, + camera_job_root=camera_job.job_root, + descriptor=descriptor, + ) + + +def compose_lab_v1_portable_executor_adapter( + *, + candidate: PortableWorkerRuntimeCandidate, + definition: PortableRunDefinition, + admission: PortableWorkerRuntimeAdmission, + source_transport: PortableWorkerSourceMaterializer, + result_transport: PortableWorkerResultPublisher, + installation: PortableLabV1RunnerInstallation, + source_output_parent: Path, + created_at_utc: Callable[[], str], + eomt_runner: PortableLabV1EomtRunner, + ddrnet_runner: PortableLabV1DdrnetRunner, +) -> PortableWorkerExecutorAdapter: + """Compose the shared Worker ports without enabling or registering them.""" + + _verify_candidate_release(candidate, installation) + return PortableWorkerExecutorAdapter( + candidate=candidate, + definition=definition, + admission=admission, + source_materializer=PortableLabV1SourceMaterializerAdapter( + upstream=source_transport, + definition=definition, + output_parent=source_output_parent, + ), + runner=PortableLabV1ProfileRunnerAdapter( + definition=definition, + installation=installation, + created_at_utc=created_at_utc, + eomt_runner=eomt_runner, + ddrnet_runner=ddrnet_runner, + ), + publisher=result_transport, + ) + + +def _verify_definition(definition: PortableRunDefinition) -> None: + if ( + definition.setup_id != _EXPECTED_SETUP_ID + or definition.definition_id != _EXPECTED_DEFINITION_ID + or definition.result_contract.contract_sha256 != _EXPECTED_RESULT_CONTRACT_SHA256 + or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY + ): + raise PortableLabV1WorkerError("portable LAB V1 RunDefinition identity changed") + + +def _verify_runtime_plan( + plan: PortableWorkerRuntimePlan, + *, + job: SealedObservatoryRecordedJob, + definition: PortableRunDefinition, +) -> None: + if ( + plan.job_id != job.job_id + or plan.setup_id != job.setup_id + or plan.definition_sha256 != job.definition_sha256 + or plan.source_bundle_sha256 != job.source_bundle_sha256 + or plan.source_capability_manifest_sha256 != job.source_capability_manifest_sha256 + or plan.result_contract_sha256 != _EXPECTED_RESULT_CONTRACT_SHA256 + or plan.phases != PORTABLE_LAB_V1_RUNTIME_PHASES + or job.definition_sha256 != definition.definition_sha256 + ): + raise PortableWorkerRuntimeJobRejectedError( + "portable LAB V1 runtime plan differs from its sealed job" + ) + + +def _verify_candidate_release( + candidate: PortableWorkerRuntimeCandidate, + installation: PortableLabV1RunnerInstallation, +) -> None: + release = installation.release + release_assets = {asset.asset_id: asset for asset in release.assets} + executor = candidate.executor + release_seal = release.seal(installation.inspection) + if ( + candidate.setup_id != release.setup_id + or candidate.definition_id != release.definition_id + or candidate.definition_version != release.definition_version + or candidate.definition_sha256 != release.definition_sha256 + or candidate.result_contract_sha256 != release.result_contract_sha256 + or tuple(phase.phase_id for phase in candidate.phases) != PORTABLE_LAB_V1_RUNTIME_PHASES + or executor is None + or executor.release_id != release_seal.release_id + or executor.release_sha256 != release_seal.release_sha256 + or executor.image_sha256 != release_seal.executor_image_sha256 + ): + raise PortableLabV1WorkerError("portable LAB V1 runtime and release candidates disagree") + for requirement in candidate.reusable_assets: + asset = release_assets.get(requirement.asset_id) + if ( + asset is None + or asset.sha256 != requirement.sha256 + or ( + requirement.byte_length is not None and asset.byte_length != requirement.byte_length + ) + ): + raise PortableLabV1WorkerError("portable LAB V1 runtime asset differs from its release") + + +def _verify_installation_unchanged( + installation: PortableLabV1RunnerInstallation, +) -> None: + config_asset = next( + asset + for asset in installation.release.assets + if asset.asset_id == _PORTABLE_CONFIG_ASSET_ID + ) + _exact_file( + installation.portable_ddrnet_config_path, + expected_sha256=config_asset.sha256, + expected_byte_length=config_asset.byte_length, + label="portable LAB V1 portable DDRNet config", + ) + + +def _verify_materialization_manifest( + root: Path, + manifest: Mapping[str, object], + *, + bundle: Mapping[str, object], + bundle_byte_length: int, + capability_byte_length: int, + job: SealedObservatoryRecordedJob, +) -> _CameraContract: + if set(manifest) != { + "schema_version", + "job_id", + "job_identity_sha256", + "claim_generation", + "source", + "members", + "authority", + } or ( + manifest.get("schema_version") != PORTABLE_SOURCE_MATERIALIZATION_SCHEMA + or manifest.get("job_id") != job.job_id + or manifest.get("job_identity_sha256") != job.identity_sha256 + or manifest.get("claim_generation") != job.claim_generation + or manifest.get("authority") != OBSERVATION_ONLY_AUTHORITY + ): + raise PortableLabV1SourceError( + "source materialization belongs to another sealed LAB V1 job" + ) + source = _object(manifest.get("source"), "source materialization identity") + if source != { + "session_id": job.source_session_id, + "bundle_sha256": job.source_bundle_sha256, + "capability_manifest_sha256": job.source_capability_manifest_sha256, + }: + raise PortableLabV1SourceError("source materialization identity changed") + raw_members = manifest.get("members") + if not isinstance(raw_members, list) or not 2 <= len(raw_members) <= _MAX_SOURCE_MEMBERS: + raise PortableLabV1SourceError("source materialization members are invalid") + members = tuple(_source_member(value, job=job) for value in raw_members) + if ( + tuple(member.member_id for member in members) + != tuple(sorted(member.member_id for member in members)) + or len({member.member_id for member in members}) != len(members) + or sum(member.byte_length for member in members) > _MAX_SOURCE_BYTES + ): + raise PortableLabV1SourceError("source materialization inventory changed") + bundle_members = tuple(member for member in members if member.kind == "source-bundle") + capability_members = tuple(member for member in members if member.kind == "source-capability") + if ( + len(bundle_members) != 1 + or bundle_members[0].sha256 != job.source_bundle_sha256 + or bundle_members[0].byte_length != bundle_byte_length + or bundle_members[0].media_type != "application/json" + or len(capability_members) != 1 + or capability_members[0].sha256 != job.source_capability_manifest_sha256 + or capability_members[0].byte_length != capability_byte_length + or capability_members[0].media_type != "application/json" + ): + raise PortableLabV1SourceError("source materialization documents are incomplete") + + if ( + bundle.get("schema_version") != PORTABLE_SOURCE_BUNDLE_SCHEMA + or bundle.get("authority") != OBSERVATION_ONLY_AUTHORITY + ): + raise PortableLabV1SourceError("portable source bundle contract changed") + camera = _object(bundle.get("camera"), "portable source camera") + epoch = _object(camera.get("epoch"), "portable source camera epoch") + init = _object(epoch.get("init"), "portable source camera init") + segment_rows = epoch.get("segments") + if not isinstance(segment_rows, list) or not segment_rows: + raise PortableLabV1SourceError("portable source camera segments are invalid") + artifact_id = _string(camera.get("artifact_id"), "camera artifact id") + public_source_id = _string(camera.get("public_source_id"), "camera public source id") + if _SAFE_COMPONENT.fullmatch(public_source_id) is None: + raise PortableLabV1SourceError("camera public source id is unsafe") + generation_sha256 = _digest(camera.get("generation_sha256"), "camera generation sha256") + synchronization = _string(camera.get("synchronization"), "camera synchronization") + epoch_ordinal = _positive_int(epoch.get("ordinal"), "camera epoch") + media_type = _media_type(epoch.get("media_type"), "camera media type") + timeline_start = _finite_number(epoch.get("timeline_start_seconds"), "camera timeline start") + timeline_end = _finite_number(epoch.get("timeline_end_seconds"), "camera timeline end") + if timeline_end <= timeline_start: + raise PortableLabV1SourceError("portable camera timeline is invalid") + init_members = tuple(member for member in members if member.kind == "camera-init") + segment_members = tuple(member for member in members if member.kind == "camera-segment") + if len(init_members) != 1 or len(segment_members) != len(segment_rows): + raise PortableLabV1SourceError("portable camera member inventory changed") + init_member = init_members[0] + expected_init = { + "artifact_id": artifact_id, + "camera_epoch": epoch_ordinal, + "camera_sequence": None, + "media_type": media_type, + "byte_length": _positive_int(init.get("byte_length"), "camera init bytes"), + "sha256": _digest(init.get("sha256"), "camera init sha256"), + } + if ( + init_member.artifact_id != expected_init["artifact_id"] + or init_member.camera_epoch != expected_init["camera_epoch"] + or init_member.camera_sequence is not None + or init_member.media_type != expected_init["media_type"] + or init_member.byte_length != expected_init["byte_length"] + or init_member.sha256 != expected_init["sha256"] + or init_member.primary + ): + raise PortableLabV1SourceError("portable camera init member changed") + ordered_segments = tuple( + sorted(segment_members, key=lambda member: member.camera_sequence or 0) + ) + for expected_sequence, (row_value, member) in enumerate( + zip(segment_rows, ordered_segments, strict=True), + start=1, + ): + row = _object(row_value, "portable camera segment") + if ( + _positive_int(row.get("sequence"), "camera segment sequence") != expected_sequence + or member.artifact_id != artifact_id + or member.camera_epoch != epoch_ordinal + or member.camera_sequence != expected_sequence + or member.media_type != "video/iso.segment" + or member.byte_length != _positive_int(row.get("byte_length"), "camera segment bytes") + or member.sha256 != _digest(row.get("sha256"), "camera segment sha256") + or member.primary + ): + raise PortableLabV1SourceError("portable camera segment member changed") + + init_path = root / "camera" / f"epoch-{epoch_ordinal}" / "init.mp4" + segment_paths = tuple( + root / "camera" / f"epoch-{epoch_ordinal}" / "segments" / f"{sequence}.m4s" + for sequence in range(1, len(ordered_segments) + 1) + ) + _exact_file( + init_path, + expected_sha256=init_member.sha256, + expected_byte_length=init_member.byte_length, + label="portable camera init", + confinement_root=root, + ) + for member, path in zip(ordered_segments, segment_paths, strict=True): + _exact_file( + path, + expected_sha256=member.sha256, + expected_byte_length=member.byte_length, + label="portable camera segment", + confinement_root=root, + ) + return _CameraContract( + artifact_id=artifact_id, + public_source_id=public_source_id, + generation_sha256=generation_sha256, + synchronization=synchronization, + epoch=epoch_ordinal, + media_type=media_type, + timeline_start_seconds=timeline_start, + timeline_end_seconds=timeline_end, + init=init_member, + segments=ordered_segments, + init_path=init_path, + segment_paths=segment_paths, + ) + + +def _source_member( + value: object, + *, + job: SealedObservatoryRecordedJob, +) -> _WorkerSourceMember: + row = _object(value, "source materialization member") + expected_keys = { + "member_id", + "kind", + "media_type", + "byte_length", + "sha256", + "artifact_id", + "primary", + "camera_epoch", + "camera_sequence", + } + if set(row) != expected_keys: + raise PortableLabV1SourceError("source materialization member fields changed") + kind_value = row.get("kind") + if not isinstance(kind_value, str) or kind_value not in _MEMBER_KIND: + raise PortableLabV1SourceError("source materialization member kind changed") + member_id = _digest(row.get("member_id"), "source member id") + sha256 = _digest(row.get("sha256"), "source member sha256") + byte_length = _nonnegative_int(row.get("byte_length"), "source member byte length") + media_type = _media_type(row.get("media_type"), "source member media type") + artifact_id_value = row.get("artifact_id") + if artifact_id_value is not None and ( + not isinstance(artifact_id_value, str) + or _SAFE_COMPONENT.fullmatch(artifact_id_value) is None + ): + raise PortableLabV1SourceError("source member artifact id is invalid") + primary = row.get("primary") + if not isinstance(primary, bool): + raise PortableLabV1SourceError("source member primary flag is invalid") + camera_epoch = _optional_positive_int(row.get("camera_epoch"), "source camera epoch") + camera_sequence = _optional_positive_int(row.get("camera_sequence"), "source camera sequence") + if kind_value in {"source-bundle", "source-capability"}: + if ( + artifact_id_value is not None + or primary + or camera_epoch is not None + or camera_sequence is not None + ): + raise PortableLabV1SourceError("source document member metadata is invalid") + elif kind_value in {"spatial-replay", "spatial-replay-metadata"}: + if ( + artifact_id_value is None + or camera_epoch is not None + or camera_sequence is not None + or (kind_value == "spatial-replay-metadata" and primary) + ): + raise PortableLabV1SourceError("spatial source member metadata is invalid") + elif kind_value == "camera-init": + if ( + artifact_id_value is None + or primary + or camera_epoch is None + or camera_sequence is not None + ): + raise PortableLabV1SourceError("camera init member metadata is invalid") + elif artifact_id_value is None or primary or camera_epoch is None or camera_sequence is None: + raise PortableLabV1SourceError("camera segment member metadata is invalid") + identity = { + "job_identity_sha256": job.identity_sha256, + "source_bundle_sha256": job.source_bundle_sha256, + "kind": kind_value, + "artifact_id": artifact_id_value, + "primary": primary, + "camera_epoch": camera_epoch, + "camera_sequence": camera_sequence, + "media_type": media_type, + "byte_length": byte_length, + "sha256": sha256, + } + if hashlib.sha256(canonical_json(identity)).hexdigest() != member_id: + raise PortableLabV1SourceError("source materialization member identity changed") + return _WorkerSourceMember( + member_id=member_id, + kind=cast(SourceMemberKind, kind_value), + media_type=media_type, + byte_length=byte_length, + sha256=sha256, + artifact_id=artifact_id_value, + primary=primary, + camera_epoch=camera_epoch, + camera_sequence=camera_sequence, + ) + + +def _publish_camera_compute_job( + *, + camera: _CameraContract, + job: SealedObservatoryRecordedJob, + output_root: Path, +) -> CameraComputeJob: + summary = { + "schema_version": PORTABLE_LAB_V1_CAMERA_SUMMARY_SCHEMA, + "source_session_id": job.source_session_id, + "source_catalog_sha256": job.source_catalog_sha256, + "source_bundle_sha256": job.source_bundle_sha256, + "camera": { + "artifact_id": camera.artifact_id, + "public_source_id": camera.public_source_id, + "generation_sha256": camera.generation_sha256, + "synchronization": camera.synchronization, + "epoch": camera.epoch, + "media_type": camera.media_type, + "timeline_start_seconds": camera.timeline_start_seconds, + "timeline_end_seconds": camera.timeline_end_seconds, + "segment_count": len(camera.segments), + }, + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + summary_bytes = canonical_json(summary) + index_bytes = b"".join( + canonical_json( + { + "schema_version": PORTABLE_LAB_V1_CAMERA_INDEX_SCHEMA, + "sequence": sequence, + "byte_length": member.byte_length, + "sha256": member.sha256, + } + ) + + b"\n" + for sequence, member in enumerate(camera.segments, start=1) + ) + prefix = PurePosixPath("input") / "camera" / camera.public_source_id / f"epoch-{camera.epoch}" + file_records: list[dict[str, object]] = [ + _compute_file_record(prefix / "summary.json", summary_bytes), + _compute_file_record(prefix / "index.jsonl", index_bytes), + { + "path": (prefix / "init.mp4").as_posix(), + "byte_length": camera.init.byte_length, + "sha256": camera.init.sha256, + }, + ] + file_records.extend( + { + "path": (prefix / "segments" / f"{sequence}.m4s").as_posix(), + "byte_length": member.byte_length, + "sha256": member.sha256, + } + for sequence, member in enumerate(camera.segments, start=1) + ) + input_document = { + "kind": "canonical-camera-epoch", + "session_id": job.source_session_id, + "source_id": camera.public_source_id, + "codec_epoch": camera.epoch, + "synchronization": camera.synchronization, + "media_type": camera.media_type, + "timeline": { + "basis": "session-time-seconds", + "start_seconds": camera.timeline_start_seconds, + "end_seconds": camera.timeline_end_seconds, + }, + "segment_count": len(camera.segments), + "byte_length": sum(cast(int, record["byte_length"]) for record in file_records), + "archive_summary_sha256": hashlib.sha256(summary_bytes).hexdigest(), + "archive_index_sha256": hashlib.sha256(index_bytes).hexdigest(), + "files": file_records, + } + input_sha256 = hashlib.sha256(canonical_json(input_document)).hexdigest() + job_id = f"recorded-camera-{input_sha256[:24]}" + manifest = { + "schema_version": COMPUTE_JOB_SCHEMA, + "job_id": job_id, + "profile": COMPUTE_PROFILE, + "input_sha256": input_sha256, + "input": input_document, + "result_contract": { + "schema_version": COMPUTE_RESULT_SCHEMA, + "timestamp_basis": "session-time-seconds", + }, + } + parent = _prepare_real_directory(output_root, "portable camera job parent") + final = parent / job_id + if final.is_symlink() or final.exists(): + existing_root = _real_directory(final, "portable camera compute job") + if existing_root.parent != parent: + raise PortableLabV1SourceError("portable camera compute job escapes its digest stage") + existing = validate_camera_compute_job(existing_root) + if existing.input_sha256 != input_sha256: + raise PortableLabV1SourceError("portable camera compute job identity collided") + return existing + staging = parent / f".tmp-{secrets.token_hex(16)}" + staging.mkdir(mode=0o700) + published = False + try: + epoch_root = ( + staging / "input" / "camera" / camera.public_source_id / f"epoch-{camera.epoch}" + ) + (epoch_root / "segments").mkdir(mode=0o700, parents=True) + _write_exact(epoch_root / "summary.json", summary_bytes) + _write_exact(epoch_root / "index.jsonl", index_bytes) + _copy_exact( + camera.init_path, + epoch_root / "init.mp4", + expected_sha256=camera.init.sha256, + expected_byte_length=camera.init.byte_length, + ) + for sequence, (member, source_path) in enumerate( + zip(camera.segments, camera.segment_paths, strict=True), + start=1, + ): + _copy_exact( + source_path, + epoch_root / "segments" / f"{sequence}.m4s", + expected_sha256=member.sha256, + expected_byte_length=member.byte_length, + ) + _write_exact(staging / "job.json", canonical_json(manifest)) + _fsync_tree(staging) + os.replace(staging, final) + _fsync_directory(parent) + published = True + finally: + if not published and staging.exists(): + shutil.rmtree(staging) + return validate_camera_compute_job(final) + + +def _compute_file_record(path: PurePosixPath, payload: bytes) -> dict[str, object]: + return { + "path": path.as_posix(), + "byte_length": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + + +def _read_canonical_document( + path: Path, + *, + root: Path, + expected_sha256: str | None, + label: str, + maximum: int = _MAX_SOURCE_DOCUMENT_BYTES, +) -> tuple[bytes, dict[str, object]]: + payload = _read_exact_file(path, root=root, maximum=maximum, label=label) + if expected_sha256 is not None and hashlib.sha256(payload).hexdigest() != expected_sha256: + raise PortableLabV1SourceError(f"{label} digest changed") + try: + value: object = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PortableLabV1SourceError(f"{label} is invalid") from exc + document = _object(value, label) + if canonical_json(document) != payload: + raise PortableLabV1SourceError(f"{label} is not canonical JSON") + return payload, document + + +def _read_json_object(path: Path, label: str) -> dict[str, object]: + try: + root = path.parent.resolve(strict=True) + except OSError as exc: + raise PortableLabV1WorkerError(f"{label} is unavailable") from exc + payload = _read_exact_file( + path, + root=root, + maximum=_MAX_SOURCE_DOCUMENT_BYTES, + label=label, + ) + try: + return _object(json.loads(payload.decode("utf-8-sig")), label) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PortableLabV1WorkerError(f"{label} is invalid") from exc + + +def _read_exact_file(path: Path, *, root: Path, maximum: int, label: str) -> bytes: + try: + resolved_root = root.resolve(strict=True) + resolved = path.resolve(strict=True) + metadata = path.lstat() + except OSError as exc: + raise PortableLabV1SourceError(f"{label} is unavailable") from exc + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or not resolved.is_relative_to(resolved_root) + or not 0 < metadata.st_size <= maximum + ): + raise PortableLabV1SourceError(f"{label} is not a confined bounded file") + return path.read_bytes() + + +def _exact_file( + path: Path, + *, + expected_sha256: str, + expected_byte_length: int | None, + label: str, + confinement_root: Path | None = None, +) -> Path: + candidate = path.expanduser().absolute() + try: + metadata = candidate.lstat() + resolved = candidate.resolve(strict=True) + root = ( + confinement_root.resolve(strict=True) + if confinement_root is not None + else candidate.parent.resolve(strict=True) + ) + except OSError as exc: + raise PortableLabV1WorkerError(f"{label} is unavailable") from exc + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or not resolved.is_relative_to(root) + or (expected_byte_length is not None and metadata.st_size != expected_byte_length) + or _sha256_file(resolved) != expected_sha256 + ): + raise PortableLabV1WorkerError(f"{label} identity changed") + return resolved + + +def _copy_exact( + source: Path, + destination: Path, + *, + expected_sha256: str, + expected_byte_length: int, +) -> None: + destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + with source.open("rb") as reader, destination.open("xb") as writer: + shutil.copyfileobj(reader, writer, length=1024 * 1024) + writer.flush() + os.fsync(writer.fileno()) + os.chmod(destination, 0o600) + if ( + destination.stat().st_size != expected_byte_length + or _sha256_file(destination) != expected_sha256 + ): + raise PortableLabV1SourceError("portable camera member changed during copy") + + +def _write_exact(path: Path, payload: bytes) -> None: + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + with path.open("xb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(path, 0o600) + + +def _require_component_result(path: Path, label: str) -> None: + root = _real_directory(path, label) + _read_json_object(root / "result.json", f"{label} document") + + +def _prepare_real_directory(path: Path, label: str) -> Path: + candidate = path.expanduser().absolute() + candidate.mkdir(mode=0o700, parents=True, exist_ok=True) + root = _real_directory(candidate, label) + os.chmod(root, 0o700) + return root + + +def _real_directory(path: Path, label: str) -> Path: + try: + metadata = path.lstat() + resolved = path.resolve(strict=True) + except OSError as exc: + raise PortableLabV1WorkerError(f"{label} is unavailable") from exc + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise PortableLabV1WorkerError(f"{label} is not a real directory") + return resolved + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + try: + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + except OSError as exc: + raise PortableLabV1WorkerError("portable LAB V1 file is unavailable") from exc + return digest.hexdigest() + + +def _fsync_tree(root: Path) -> None: + directories = sorted( + (path for path in root.rglob("*") if path.is_dir()), + key=lambda path: len(path.parts), + reverse=True, + ) + for directory in directories: + _fsync_directory(directory) + _fsync_directory(root) + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _object(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise PortableLabV1SourceError(f"{label} is not an object") + return cast(dict[str, object], value) + + +def _string(value: object, label: str) -> str: + if not isinstance(value, str) or not value: + raise PortableLabV1SourceError(f"{label} is invalid") + return value + + +def _media_type(value: object, label: str) -> str: + media_type = _string(value, label) + if ( + not 3 <= len(media_type) <= 255 + or "/" not in media_type + or media_type != media_type.strip() + or any(ord(character) < 32 or ord(character) > 126 for character in media_type) + ): + raise PortableLabV1SourceError(f"{label} is invalid") + return media_type + + +def _digest(value: object, label: str) -> str: + if not isinstance(value, str) or _SHA256.fullmatch(value) is None: + raise PortableLabV1SourceError(f"{label} is invalid") + return value + + +def _nonnegative_int(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise PortableLabV1SourceError(f"{label} is invalid") + return value + + +def _positive_int(value: object, label: str) -> int: + result = _nonnegative_int(value, label) + if result < 1: + raise PortableLabV1SourceError(f"{label} is invalid") + return result + + +def _optional_positive_int(value: object, label: str) -> int | None: + if value is None: + return None + return _positive_int(value, label) + + +def _finite_number(value: object, label: str) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not float("-inf") < float(value) < float("inf") + ): + raise PortableLabV1SourceError(f"{label} is invalid") + return float(value) diff --git a/src/k1link/observatory/portable_queue_binding.py b/src/k1link/observatory/portable_queue_binding.py index 51aecba..2e6f54b 100644 --- a/src/k1link/observatory/portable_queue_binding.py +++ b/src/k1link/observatory/portable_queue_binding.py @@ -133,9 +133,12 @@ class PortableRecordedQueueBindingService: ) -> PortableRecordedSourceCapability: """Return cheap authoritative compatibility without replay preparation.""" - portable, recorded = self._resolve_definition(setup_id, definition_sha256) + # Catalog projection must remain available for not-yet-installed + # executors. Probe only resolves the immutable portable definition; + # check/admit/submit still require a ready executor before source reads. + portable = self._definitions.resolve(setup_id, definition_sha256) capability = self._source_service(portable).probe(source_session_id) - if recorded.source_adapter_sha256 != capability.source_adapter_sha256: + if portable.source_adapter.contract_sha256 != capability.source_adapter_sha256: raise PortableQueueBindingIntegrityError( "portable registry and source capability adapter identities disagree" ) diff --git a/src/k1link/observatory/portable_result_contract.py b/src/k1link/observatory/portable_result_contract.py new file mode 100644 index 0000000..286f7cf --- /dev/null +++ b/src/k1link/observatory/portable_result_contract.py @@ -0,0 +1,601 @@ +"""Strict, path-free contracts for portable Observatory result packages.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Final, cast + +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinition, + canonical_sha256, +) +from k1link.observatory.recorded_jobs import ObservatoryRecordedJob + +PORTABLE_RESULT_PACKAGE_SCHEMA: Final = ( + "missioncore.observatory-portable-result-package/v1" +) +PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA: Final = ( + "missioncore.observatory-portable-result-package-identity/v1" +) +PORTABLE_RESULT_PUBLICATION_SCHEMA: Final = ( + "missioncore.observatory-portable-result-publication/v1" +) +OBSERVATORY_CALCULATION_PROFILE_SCHEMA: Final = ( + "missioncore.observatory-calculation-profile/v1" +) +RESULT_PACKAGE_MANIFEST_NAME: Final = "manifest.json" +RESULT_DOCUMENT_ROLE: Final = "result-document" +RESULT_PACKAGE_MANIFEST_ROLE: Final = "result-package-manifest" + +_MAX_MANIFEST_BYTES: Final = 1024 * 1024 +_MAX_RESULT_DOCUMENT_BYTES: Final = 8 * 1024 * 1024 +_MAX_ARTIFACTS: Final = 128 +_MAX_ARTIFACT_BYTES: Final = (1 << 63) - 1 +_SHA256: Final = re.compile(r"^[a-f0-9]{64}$") +_IDENTIFIER: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$") +_SESSION_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +_ROLE: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$") +_PATH_COMPONENT: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +_LAB_ID: Final = re.compile(r"^LAB [A-Z][A-Z0-9._-]{0,31}$") +OBSERVATION_ONLY_AUTHORITY: Final = { + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, + "production_accepted": False, +} + + +class PortableResultPublisherError(RuntimeError): + """Base class for verified portable result publication failures.""" + + +class PortableResultPackageIntegrityError(PortableResultPublisherError): + """The supplied package or one of its content identities is invalid.""" + + +class PortableResultPublicationBlockedError(PortableResultPublisherError): + """A required server-owned definition, policy, or validator is absent.""" + + +@dataclass(frozen=True, slots=True) +class PortableResultArtifact: + """One confined, content-addressed package member.""" + + role: str + relative_path: str + media_type: str + byte_length: int + sha256: str + + def __post_init__(self) -> None: + _pattern(self.role, _ROLE, "portable result artifact role") + if self.role == RESULT_PACKAGE_MANIFEST_ROLE: + raise ValueError("portable result artifact role is reserved") + relative_artifact_path(self.relative_path) + _media_type(self.media_type) + if ( + not isinstance(self.byte_length, int) + or isinstance(self.byte_length, bool) + or not 0 <= self.byte_length <= _MAX_ARTIFACT_BYTES + ): + raise ValueError("portable result artifact byte length is invalid") + digest(self.sha256, "portable result artifact sha256") + + def as_dict(self) -> dict[str, object]: + return { + "role": self.role, + "relative_path": self.relative_path, + "media_type": self.media_type, + "byte_length": self.byte_length, + "sha256": self.sha256, + } + + +@dataclass(frozen=True, slots=True) +class PortableResultPackageManifest: + """Strict Worker-produced manifest with a separately hashed identity.""" + + identity_sha256: str + created_at_utc: str + job: dict[str, object] + source: dict[str, object] + run_definition: dict[str, object] + result: dict[str, object] + authority: dict[str, object] + artifacts: tuple[PortableResultArtifact, ...] + + def __post_init__(self) -> None: + digest(self.identity_sha256, "portable result package identity sha256") + _timestamp(self.created_at_utc, "portable result package creation time") + if self.authority != OBSERVATION_ONLY_AUTHORITY: + raise PortableResultPackageIntegrityError( + "portable result package is not observation-only" + ) + if not 1 <= len(self.artifacts) <= _MAX_ARTIFACTS: + raise PortableResultPackageIntegrityError( + "portable result package artifact count is invalid" + ) + roles = tuple(artifact.role for artifact in self.artifacts) + paths = tuple(artifact.relative_path for artifact in self.artifacts) + if roles != tuple(sorted(roles)) or len(set(roles)) != len(roles): + raise PortableResultPackageIntegrityError( + "portable result package artifacts are not canonically ordered" + ) + if len(set(paths)) != len(paths): + raise PortableResultPackageIntegrityError( + "portable result package artifact paths are not unique" + ) + result_documents = tuple( + artifact for artifact in self.artifacts if artifact.role == RESULT_DOCUMENT_ROLE + ) + if ( + len(result_documents) != 1 + or result_documents[0].media_type != "application/json" + or not 0 < result_documents[0].byte_length <= _MAX_RESULT_DOCUMENT_BYTES + ): + raise PortableResultPackageIntegrityError( + "portable result package requires one JSON result document" + ) + if self.identity_sha256 != canonical_sha256(self.identity_document()): + raise PortableResultPackageIntegrityError( + "portable result package identity digest changed" + ) + + @classmethod + def create( + cls, + *, + job: ObservatoryRecordedJob, + definition: PortableRunDefinition, + result_id: str, + created_at_utc: str, + artifacts: Sequence[PortableResultArtifact], + ) -> PortableResultPackageManifest: + """Build the canonical package envelope used by a future Worker assembler.""" + + _pattern(result_id, _SESSION_ID, "portable result id") + job_document = job_identity_document(job) + source_document = source_identity_document(job) + definition_document = run_definition_document(definition) + result_document = result_identity_document(definition, result_id) + authority: dict[str, object] = dict(OBSERVATION_ONLY_AUTHORITY) + normalized_artifacts = tuple( + sorted(artifacts, key=lambda artifact: artifact.role) + ) + identity_document = _package_identity_document( + created_at_utc=created_at_utc, + job=job_document, + source=source_document, + run_definition=definition_document, + result=result_document, + authority=authority, + artifacts=normalized_artifacts, + ) + return cls( + identity_sha256=canonical_sha256(identity_document), + created_at_utc=created_at_utc, + job=job_document, + source=source_document, + run_definition=definition_document, + result=result_document, + authority=authority, + artifacts=normalized_artifacts, + ) + + @classmethod + def from_bytes(cls, payload: bytes) -> PortableResultPackageManifest: + if not 0 < len(payload) <= _MAX_MANIFEST_BYTES: + raise PortableResultPackageIntegrityError( + "portable result package manifest size is invalid" + ) + try: + decoded: object = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PortableResultPackageIntegrityError( + "portable result package manifest is not valid JSON" + ) from exc + document = object_document(decoded, "portable result package manifest") + exact_keys( + document, + { + "schema_version", + "identity_sha256", + "created_at_utc", + "job", + "source", + "run_definition", + "result", + "authority", + "artifacts", + }, + "portable result package manifest", + ) + if document["schema_version"] != PORTABLE_RESULT_PACKAGE_SCHEMA: + raise PortableResultPackageIntegrityError( + "portable result package manifest schema is invalid" + ) + if payload != canonical_json(document): + raise PortableResultPackageIntegrityError( + "portable result package manifest is not canonical JSON" + ) + artifacts_value = document["artifacts"] + if not isinstance(artifacts_value, list): + raise PortableResultPackageIntegrityError( + "portable result package artifacts are not an array" + ) + artifacts = tuple(_artifact(value) for value in artifacts_value) + try: + return cls( + identity_sha256=string( + document["identity_sha256"], + "portable result package identity sha256", + ), + created_at_utc=string( + document["created_at_utc"], + "portable result package creation time", + ), + job=object_document(document["job"], "portable result job identity"), + source=object_document( + document["source"], "portable result source identity" + ), + run_definition=object_document( + document["run_definition"], + "portable result RunDefinition", + ), + result=object_document(document["result"], "portable result identity"), + authority=object_document( + document["authority"], "portable result authority" + ), + artifacts=artifacts, + ) + except ValueError as exc: + raise PortableResultPackageIntegrityError( + "portable result package manifest field is invalid" + ) from exc + + def identity_document(self) -> dict[str, object]: + return _package_identity_document( + created_at_utc=self.created_at_utc, + job=self.job, + source=self.source, + run_definition=self.run_definition, + result=self.result, + authority=self.authority, + artifacts=self.artifacts, + ) + + def as_dict(self) -> dict[str, object]: + identity = self.identity_document() + identity.pop("schema_version") + return { + "schema_version": PORTABLE_RESULT_PACKAGE_SCHEMA, + "identity_sha256": self.identity_sha256, + **identity, + } + + @property + def canonical_bytes(self) -> bytes: + return canonical_json(self.as_dict()) + + @property + def manifest_sha256(self) -> str: + return hashlib.sha256(self.canonical_bytes).hexdigest() + + +@dataclass(frozen=True, slots=True) +class PortableCalculationProfilePolicy: + """Server-owned presentation identity bound to one exact RunDefinition.""" + + setup_id: str + definition_id: str + definition_version: int + definition_sha256: str + lab_id: str + display_name: str + include_recorded_media: bool = False + + def __post_init__(self) -> None: + _pattern(self.setup_id, _IDENTIFIER, "portable calculation profile setup id") + _pattern( + self.definition_id, + _IDENTIFIER, + "portable calculation profile definition id", + ) + if ( + not isinstance(self.definition_version, int) + or isinstance(self.definition_version, bool) + or self.definition_version < 1 + ): + raise ValueError("portable calculation profile version is invalid") + digest( + self.definition_sha256, + "portable calculation profile definition sha256", + ) + if _LAB_ID.fullmatch(self.lab_id) is None: + raise ValueError("portable calculation profile LAB id is invalid") + _text(self.display_name, "portable calculation profile display name", maximum=160) + if not isinstance(self.include_recorded_media, bool): + raise ValueError("portable calculation profile media policy is invalid") + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA, + "setup_id": self.setup_id, + "display_name": self.display_name, + "origin": "archived-definition", + "definition_id": self.definition_id, + "definition_version": self.definition_version, + "definition_sha256": self.definition_sha256, + } + + @property + def identity_sha256(self) -> str: + return canonical_sha256(self.as_dict()) + + +@dataclass(frozen=True, slots=True) +class PortableCalculationProfileRegistry: + policies: tuple[PortableCalculationProfilePolicy, ...] + + def __post_init__(self) -> None: + keys = tuple( + (policy.setup_id, policy.definition_sha256) for policy in self.policies + ) + if len(keys) != len(set(keys)): + raise ValueError("portable calculation profile policies are not unique") + + def resolve( + self, + definition: PortableRunDefinition, + ) -> PortableCalculationProfilePolicy: + for policy in self.policies: + if ( + policy.setup_id == definition.setup_id + and policy.definition_sha256 == definition.definition_sha256 + ): + if ( + policy.definition_id != definition.definition_id + or policy.definition_version != definition.version + ): + raise PortableResultPublicationBlockedError( + "calculation profile policy disagrees with the RunDefinition" + ) + return policy + raise PortableResultPublicationBlockedError( + "exact calculation profile policy is not registered" + ) + + +@dataclass(frozen=True, slots=True) +class PortableResultValidationContext: + """Read-only inputs supplied to one exact result-contract validator.""" + + manifest: PortableResultPackageManifest + job: ObservatoryRecordedJob + definition: PortableRunDefinition + result_document: Mapping[str, object] + artifact_paths: Mapping[str, Path] + + +type PortableResultContractValidator = Callable[[PortableResultValidationContext], None] + + +@dataclass(frozen=True, slots=True) +class PortableResultContractValidatorRegistration: + contract_sha256: str + validator: PortableResultContractValidator + + def __post_init__(self) -> None: + digest(self.contract_sha256, "portable result validator contract sha256") + if not callable(self.validator): + raise ValueError("portable result contract validator is not callable") + + +@dataclass(frozen=True, slots=True) +class PortableResultContractValidatorRegistry: + registrations: tuple[PortableResultContractValidatorRegistration, ...] + + def __post_init__(self) -> None: + digests = tuple(registration.contract_sha256 for registration in self.registrations) + if len(digests) != len(set(digests)): + raise ValueError("portable result contract validators are not unique") + + def resolve(self, contract_sha256: str) -> PortableResultContractValidator: + digest(contract_sha256, "portable result contract sha256") + for registration in self.registrations: + if registration.contract_sha256 == contract_sha256: + return registration.validator + raise PortableResultPublicationBlockedError( + "exact portable result-contract validator is not installed" + ) + + +def job_identity_document(job: ObservatoryRecordedJob) -> dict[str, object]: + return { + "job_id": job.job_id, + "request_sha256": job.request_sha256, + "identity_sha256": job.identity_sha256, + "submission_receipt_sha256": job.submission_receipt_sha256, + "claim_generation": job.claim_generation, + } + + +def source_identity_document(job: ObservatoryRecordedJob) -> dict[str, object]: + return { + "session_id": job.source_session_id, + "catalog_sha256": job.source_catalog_sha256, + "bundle_sha256": job.source_bundle_sha256, + "capability_manifest_sha256": job.source_capability_manifest_sha256, + "adapter": { + "adapter_id": job.source_adapter_id, + "version": job.source_adapter_version, + "adapter_sha256": job.source_adapter_sha256, + }, + } + + +def run_definition_document(definition: PortableRunDefinition) -> dict[str, object]: + return { + **definition.identity_document(), + "definition_sha256": definition.definition_sha256, + } + + +def result_identity_document( + definition: PortableRunDefinition, + result_id: str, +) -> dict[str, object]: + contract = definition.result_contract + return { + "result_id": result_id, + "result_schema": contract.result_schema, + "result_kind": contract.result_kind, + "result_contract_sha256": contract.contract_sha256, + } + + +def _package_identity_document( + *, + created_at_utc: str, + job: Mapping[str, object], + source: Mapping[str, object], + run_definition: Mapping[str, object], + result: Mapping[str, object], + authority: Mapping[str, object], + artifacts: Sequence[PortableResultArtifact], +) -> dict[str, object]: + return { + "schema_version": PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA, + "created_at_utc": created_at_utc, + "job": dict(job), + "source": dict(source), + "run_definition": dict(run_definition), + "result": dict(result), + "authority": dict(authority), + "artifacts": [artifact.as_dict() for artifact in artifacts], + } + + +def _artifact(value: object) -> PortableResultArtifact: + row = object_document(value, "portable result artifact") + exact_keys( + row, + {"role", "relative_path", "media_type", "byte_length", "sha256"}, + "portable result artifact", + ) + byte_length = row["byte_length"] + if not isinstance(byte_length, int) or isinstance(byte_length, bool): + raise PortableResultPackageIntegrityError( + "portable result artifact byte length is invalid" + ) + return PortableResultArtifact( + role=string(row["role"], "portable result artifact role"), + relative_path=string( + row["relative_path"], + "portable result artifact relative path", + ), + media_type=string(row["media_type"], "portable result artifact media type"), + byte_length=byte_length, + sha256=string(row["sha256"], "portable result artifact sha256"), + ) + + +def relative_artifact_path(value: object) -> PurePosixPath: + if not isinstance(value, str) or not 1 <= len(value) <= 512: + raise ValueError("portable result artifact path is invalid") + path = PurePosixPath(value) + if ( + path.is_absolute() + or path.as_posix() != value + or len(path.parts) < 2 + or path.parts[0] != "artifacts" + or any( + part in {"", ".", ".."} or _PATH_COMPONENT.fullmatch(part) is None + for part in path.parts + ) + ): + raise ValueError("portable result artifact path is invalid") + return path + + +def _media_type(value: object) -> str: + if ( + not isinstance(value, str) + or not 3 <= len(value) <= 255 + or "/" not in value + or value != value.strip() + or any(ord(character) < 32 or ord(character) > 126 for character in value) + ): + raise ValueError("portable result artifact media type is invalid") + return value + + +def canonical_json(value: object) -> bytes: + try: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise PortableResultPackageIntegrityError( + "portable result package is not JSON-compatible" + ) from exc + + +def object_document(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise PortableResultPackageIntegrityError(f"{label} must be an object") + return cast(dict[str, object], value) + + +def exact_keys(value: Mapping[str, object], expected: set[str], label: str) -> None: + if set(value) != expected: + raise PortableResultPackageIntegrityError(f"{label} fields are invalid") + + +def string(value: object, label: str) -> str: + if not isinstance(value, str): + raise PortableResultPackageIntegrityError(f"{label} must be text") + return value + + +def _pattern(value: object, pattern: re.Pattern[str], label: str) -> str: + if not isinstance(value, str) or pattern.fullmatch(value) is None: + raise ValueError(f"{label} is invalid") + return value + + +def digest(value: object, label: str) -> str: + return _pattern(value, _SHA256, label) + + +def _text(value: object, label: str, *, maximum: int) -> str: + if ( + not isinstance(value, str) + or not 1 <= len(value) <= maximum + or value != value.strip() + ): + raise ValueError(f"{label} is invalid") + return value + + +def _timestamp(value: object, label: str) -> str: + from datetime import datetime + + if not isinstance(value, str): + raise ValueError(f"{label} is invalid") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError(f"{label} is invalid") from exc + if parsed.tzinfo is None: + raise ValueError(f"{label} must include a timezone") + return value diff --git a/src/k1link/observatory/portable_result_publisher.py b/src/k1link/observatory/portable_result_publisher.py new file mode 100644 index 0000000..ce5affa --- /dev/null +++ b/src/k1link/observatory/portable_result_publisher.py @@ -0,0 +1,809 @@ +"""Verified publication boundary for portable recorded Observatory results. + +A Worker success acknowledgement is only a transport receipt. This module +admits a result into the Session catalog only after a canonical result package +is bound to the exact durable job, persisted source contracts, portable +RunDefinition, result-contract validator, and observation-only authority. + +The publisher deliberately has no production default validators or presentation +policy. An unknown result contract or definition-bound calculation profile is +therefore a hard publication blocker rather than an invitation to infer one from +the current UI selection or a historical LAB label. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import stat +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from k1link.artifact_gateway import ( + ArtifactGatewayError, + ArtifactManifest, + ArtifactMember, + CentralArtifactStore, +) +from k1link.observatory.portable_result_contract import ( + OBSERVATION_ONLY_AUTHORITY, + PORTABLE_RESULT_PACKAGE_SCHEMA, + PORTABLE_RESULT_PUBLICATION_SCHEMA, + RESULT_DOCUMENT_ROLE, + RESULT_PACKAGE_MANIFEST_NAME, + RESULT_PACKAGE_MANIFEST_ROLE, + PortableCalculationProfilePolicy, + PortableCalculationProfileRegistry, + PortableResultArtifact, + PortableResultContractValidatorRegistry, + PortableResultPackageIntegrityError, + PortableResultPackageManifest, + PortableResultPublicationBlockedError, + PortableResultPublisherError, + PortableResultValidationContext, + canonical_json, + exact_keys, + job_identity_document, + object_document, + relative_artifact_path, + result_identity_document, + run_definition_document, + source_identity_document, + string, +) +from k1link.observatory.portable_result_contract import ( + digest as validate_digest, +) +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinition, + PortableRunDefinitionRegistry, + PortableRunDefinitionRegistryError, + canonical_sha256, +) +from k1link.observatory.recorded_jobs import ObservatoryRecordedJob +from k1link.observatory.source_admission import ( + PORTABLE_SOURCE_BUNDLE_SCHEMA, + PORTABLE_SOURCE_CAPABILITY_SCHEMA, + PORTABLE_SOURCE_DOCUMENT_DIRECTORY, +) +from k1link.sessions.models import LabSessionBinding, SessionIntegrityError, SessionSummary +from k1link.sessions.store import SessionStore + +_COPY_CHUNK_BYTES = 1024 * 1024 +_FULL_ROUTE_LABEL = "полный маршрут и воспроизведение" +_LEGACY_CANONICAL_RESULT = re.compile( + r"^lab-v1-vegetation-shadow-[a-f0-9]{64}$" +) + + +@dataclass(frozen=True, slots=True) +class PublishedPortableObservatoryResult: + binding: LabSessionBinding + package: PortableResultPackageManifest + artifact_manifest: ArtifactManifest + + +class PortableObservatoryResultPublisher: + """Validate, archive, and project one exact portable Worker result.""" + + def __init__( + self, + *, + session_store: SessionStore, + artifact_store: CentralArtifactStore, + definitions: PortableRunDefinitionRegistry, + calculation_profiles: PortableCalculationProfileRegistry, + validators: PortableResultContractValidatorRegistry, + ) -> None: + self._session_store = session_store + self._artifact_store = artifact_store + self._definitions = definitions + self._calculation_profiles = calculation_profiles + self._validators = validators + + def publish( + self, + *, + job: ObservatoryRecordedJob, + package_root: Path, + ) -> PublishedPortableObservatoryResult: + """Publish only a verified terminal package; exact retries are idempotent.""" + + _verify_terminal_job(job) + try: + definition = self._definitions.resolve(job.setup_id, job.definition_sha256) + except (PortableRunDefinitionRegistryError, ValueError) as exc: + raise PortableResultPublicationBlockedError( + "recorded job RunDefinition is not in the portable registry" + ) from exc + _verify_definition_job_identity(definition, job) + profile = self._calculation_profiles.resolve(definition) + validator = self._validators.resolve(definition.result_contract.contract_sha256) + + root, manifest_path, package = _read_package(package_root) + _verify_package_identity(package, job=job, definition=definition) + if package.manifest_sha256 != job.result_sha256 or root.name != job.result_sha256: + raise PortableResultPackageIntegrityError( + "portable result package content address disagrees with queue success" + ) + source_summary = self._verify_source(job, definition) + artifact_paths = _verify_package_artifacts(root, package.artifacts) + result_document = _read_canonical_result_document( + artifact_paths[RESULT_DOCUMENT_ROLE] + ) + context = PortableResultValidationContext( + manifest=package, + job=job, + definition=definition, + result_document=result_document, + artifact_paths=artifact_paths, + ) + try: + validator(context) + except PortableResultPublisherError: + raise + except Exception as exc: + raise PortableResultPackageIntegrityError( + "portable result document failed its exact contract validator" + ) from exc + + artifact_manifest = self._archive_package( + job=job, + package=package, + manifest_path=manifest_path, + artifact_paths=artifact_paths, + profile=profile, + ) + provenance = _publication_provenance( + job=job, + definition=definition, + package=package, + artifact_manifest=artifact_manifest, + profile=profile, + ) + try: + binding = self._session_store.publish_lab_instance( + session_id=cast(str, job.result_id), + source_session_id=job.source_session_id, + display_name=_portable_result_display_name(source_summary), + lab_id=profile.lab_id, + result_kind=definition.result_contract.result_kind, + result_id=cast(str, job.result_id), + config_sha256=definition.definition_sha256, + run_created_at_utc=job.updated_at_utc, + replay_capability=None, + provenance=provenance, + include_recorded_media=profile.include_recorded_media, + expected_source_catalog_sha256=job.source_catalog_sha256, + ) + except (SessionIntegrityError, ValueError) as exc: + raise PortableResultPackageIntegrityError( + "portable result could not be projected as immutable LAB provenance" + ) from exc + return PublishedPortableObservatoryResult( + binding=binding, + package=package, + artifact_manifest=artifact_manifest, + ) + + def _verify_source( + self, + job: ObservatoryRecordedJob, + definition: PortableRunDefinition, + ) -> SessionSummary: + try: + detail, catalog_sha256 = ( + self._session_store.get_session_with_catalog_snapshot( + job.source_session_id + ) + ) + except Exception as exc: + raise PortableResultPackageIntegrityError( + "portable result source session is unavailable" + ) from exc + if ( + detail.summary.session_id != job.source_session_id + or detail.summary.lab is not None + or catalog_sha256 != job.source_catalog_sha256 + ): + raise PortableResultPackageIntegrityError( + "portable result source catalog changed after admission" + ) + _verify_source_documents( + self._session_store.data_dir, + job=job, + definition=definition, + ) + return detail.summary + + def _archive_package( + self, + *, + job: ObservatoryRecordedJob, + package: PortableResultPackageManifest, + manifest_path: Path, + artifact_paths: Mapping[str, Path], + profile: PortableCalculationProfilePolicy, + ) -> ArtifactManifest: + members = [ + _publish_exact_member( + self._artifact_store, + role=RESULT_PACKAGE_MANIFEST_ROLE, + media_type="application/json", + source=manifest_path, + expected_sha256=cast(str, job.result_sha256), + expected_byte_length=len(package.canonical_bytes), + ) + ] + for artifact in package.artifacts: + members.append( + _publish_exact_member( + self._artifact_store, + role=artifact.role, + media_type=artifact.media_type, + source=artifact_paths[artifact.role], + expected_sha256=artifact.sha256, + expected_byte_length=artifact.byte_length, + ) + ) + try: + archived = self._artifact_store.publish_manifest( + artifact_type="observatory-portable-result", + subject_id=cast(str, job.result_id), + members=members, + metadata={ + "package-sha256": cast(str, job.result_sha256), + "package-identity-sha256": package.identity_sha256, + "job-id": job.job_id, + "job-identity-sha256": job.identity_sha256, + "definition-sha256": job.definition_sha256, + "source-bundle-sha256": job.source_bundle_sha256, + "result-contract-sha256": string( + package.result["result_contract_sha256"], + "portable result contract sha256", + ), + "calculation-profile-sha256": profile.identity_sha256, + }, + created_at_utc=job.updated_at_utc, + ) + verified = self._artifact_store.read_manifest(archived.manifest_id) + except (ArtifactGatewayError, OSError, ValueError) as exc: + raise PortableResultPackageIntegrityError( + "portable result package could not be archived immutably" + ) from exc + if verified != archived: + raise PortableResultPackageIntegrityError( + "portable result artifact manifest changed after publication" + ) + return archived + + +def _portable_result_display_name(source: SessionSummary) -> str: + """Keep the result label source-owned; profile provenance is rendered separately.""" + + candidate = f"{source.display_name} · {_FULL_ROUTE_LABEL}" + return candidate if len(candidate) <= 160 else source.display_name + + +def resolve_published_portable_calculation_profile( + summary: SessionSummary, + *, + definitions: PortableRunDefinitionRegistry, + calculation_profiles: PortableCalculationProfileRegistry, +) -> dict[str, object] | None: + """Resolve only an exact, immutable portable publication profile. + + Catalog projection must never infer profile identity from a selected setup, + a display label, or a current definition with the same setup id. Any drift + in the stored publication provenance therefore makes the profile absent. + """ + + binding = summary.lab + if binding is None: + return None + try: + provenance = object_document( + binding.provenance, + "portable result publication provenance", + ) + exact_keys( + provenance, + { + "schema_version", + "authority", + "calculation_profile", + "calculation_profile_sha256", + "job", + "source", + "run_definition", + "result_package", + "storage", + "method", + }, + "portable result publication provenance", + ) + profile_document = object_document( + provenance["calculation_profile"], + "portable calculation profile", + ) + run_definition = object_document( + provenance["run_definition"], + "portable result RunDefinition", + ) + source = object_document(provenance["source"], "portable result source") + setup_id = string(profile_document.get("setup_id"), "portable setup id") + definition_sha256 = string( + run_definition.get("definition_sha256"), + "portable RunDefinition sha256", + ) + definition = definitions.resolve(setup_id, definition_sha256) + profile = calculation_profiles.resolve(definition) + if ( + provenance["schema_version"] != PORTABLE_RESULT_PUBLICATION_SCHEMA + or provenance["authority"] != OBSERVATION_ONLY_AUTHORITY + or provenance["calculation_profile_sha256"] != profile.identity_sha256 + or profile_document != profile.as_dict() + or canonical_sha256(profile_document) != profile.identity_sha256 + or run_definition != run_definition_document(definition) + or binding.session_id != summary.session_id + or binding.result_id != summary.session_id + or binding.source_session_id != source.get("session_id") + or binding.lab_id != profile.lab_id + or binding.config_sha256 != definition.definition_sha256 + or binding.result_kind != definition.result_contract.result_kind + ): + return None + return profile.as_dict() + except ( + KeyError, + PortableResultPublisherError, + PortableRunDefinitionRegistryError, + TypeError, + ValueError, + ): + return None + + +def _verify_terminal_job(job: ObservatoryRecordedJob) -> None: + if ( + job.state != "succeeded" + or job.result_id is None + or job.result_sha256 is None + or job.terminal_code != "result-sealed" + or job.terminal_claim_token_sha256 is None + or job.claim_generation < 1 + or job.active_claim_token is not None + or job.active_claimant_id is not None + ): + raise PortableResultPublicationBlockedError( + "recorded job has no exact terminal Worker result receipt" + ) + if _LEGACY_CANONICAL_RESULT.fullmatch(job.result_id) is not None: + raise PortableResultPublicationBlockedError( + "legacy canonical result namespace is immutable and reserved" + ) + + +def _verify_definition_job_identity( + definition: PortableRunDefinition, + job: ObservatoryRecordedJob, +) -> None: + if not definition.executor.ready: + raise PortableResultPublicationBlockedError( + "portable result RunDefinition executor is not installed" + ) + try: + recorded = definition.to_recorded_run_definition() + except Exception as exc: + raise PortableResultPublicationBlockedError( + "portable result RunDefinition cannot produce a queue identity" + ) from exc + expected = ( + recorded.setup_id, + recorded.definition_id, + recorded.definition_version, + recorded.definition_sha256, + recorded.source_adapter_id, + recorded.source_adapter_version, + recorded.source_adapter_sha256, + recorded.executor_release_id, + recorded.executor_release_sha256, + recorded.executor_image_sha256, + recorded.model_release_ids, + recorded.model_manifest_sha256, + recorded.resource_profile_id, + recorded.resource_profile_sha256, + recorded.checkpoint_policy, + recorded.allowed_checkpoints, + ) + actual = ( + job.setup_id, + job.definition_id, + job.definition_version, + job.definition_sha256, + job.source_adapter_id, + job.source_adapter_version, + job.source_adapter_sha256, + job.executor_release_id, + job.executor_release_sha256, + job.executor_image_sha256, + job.model_release_ids, + job.model_manifest_sha256, + job.resource_profile_id, + job.resource_profile_sha256, + job.checkpoint_policy, + job.allowed_checkpoints, + ) + if actual != expected or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY: + raise PortableResultPackageIntegrityError( + "recorded job and portable RunDefinition identities disagree" + ) + + +def _read_package( + package_root: Path, +) -> tuple[Path, Path, PortableResultPackageManifest]: + candidate = package_root.expanduser().absolute() + try: + root_metadata = candidate.lstat() + root = candidate.resolve(strict=True) + except OSError as exc: + raise PortableResultPackageIntegrityError( + "portable result package root is unavailable" + ) from exc + if stat.S_ISLNK(root_metadata.st_mode) or not stat.S_ISDIR(root_metadata.st_mode): + raise PortableResultPackageIntegrityError( + "portable result package root must be a regular directory" + ) + manifest_path = root / RESULT_PACKAGE_MANIFEST_NAME + try: + metadata = manifest_path.lstat() + payload = manifest_path.read_bytes() + except OSError as exc: + raise PortableResultPackageIntegrityError( + "portable result package manifest is unavailable" + ) from exc + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): + raise PortableResultPackageIntegrityError( + "portable result package manifest must be a regular file" + ) + package = PortableResultPackageManifest.from_bytes(payload) + if hashlib.sha256(payload).hexdigest() != package.manifest_sha256: + raise PortableResultPackageIntegrityError( + "portable result package manifest digest changed" + ) + return root, manifest_path, package + + +def _verify_package_identity( + package: PortableResultPackageManifest, + *, + job: ObservatoryRecordedJob, + definition: PortableRunDefinition, +) -> None: + if package.job != job_identity_document(job): + raise PortableResultPackageIntegrityError( + "portable result package is bound to another queue job" + ) + if package.source != source_identity_document(job): + raise PortableResultPackageIntegrityError( + "portable result package is bound to another source" + ) + if package.run_definition != run_definition_document(definition): + raise PortableResultPackageIntegrityError( + "portable result package is bound to another RunDefinition" + ) + if package.result != result_identity_document(definition, cast(str, job.result_id)): + raise PortableResultPackageIntegrityError( + "portable result package violates its sealed result contract" + ) + if package.authority != definition.authority.as_dict(): + raise PortableResultPackageIntegrityError( + "portable result package authority changed" + ) + + +def _verify_source_documents( + data_dir: Path, + *, + job: ObservatoryRecordedJob, + definition: PortableRunDefinition, +) -> None: + root = data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY + try: + metadata = root.lstat() + except OSError as exc: + raise PortableResultPublicationBlockedError( + "admitted portable source documents are unavailable" + ) from exc + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise PortableResultPackageIntegrityError( + "portable source document root is unsafe" + ) + bundle = _read_content_addressed_document(root, job.source_bundle_sha256) + capability = _read_content_addressed_document( + root, + job.source_capability_manifest_sha256, + ) + exact_keys( + bundle, + { + "schema_version", + "source_session_id", + "source_catalog_sha256", + "plugin_id", + "archive_id", + "source_adapter", + "sources", + "spatial_replay", + "camera", + "authority", + }, + "portable source bundle", + ) + exact_keys( + capability, + { + "schema_version", + "source_session_id", + "source_catalog_sha256", + "source_bundle_sha256", + "source_adapter_sha256", + "modalities", + "camera_profile", + "calibration", + "authority", + }, + "portable source capability", + ) + expected_adapter = { + "id": job.source_adapter_id, + "version": job.source_adapter_version, + "sha256": job.source_adapter_sha256, + } + if ( + bundle["schema_version"] != PORTABLE_SOURCE_BUNDLE_SCHEMA + or bundle["source_session_id"] != job.source_session_id + or bundle["source_catalog_sha256"] != job.source_catalog_sha256 + or bundle["plugin_id"] != definition.source_requirements.plugin_id + or bundle["archive_id"] != definition.source_requirements.archive_id + or bundle["source_adapter"] != expected_adapter + or bundle["authority"] != OBSERVATION_ONLY_AUTHORITY + or capability["schema_version"] != PORTABLE_SOURCE_CAPABILITY_SCHEMA + or capability["source_session_id"] != job.source_session_id + or capability["source_catalog_sha256"] != job.source_catalog_sha256 + or capability["source_bundle_sha256"] != job.source_bundle_sha256 + or capability["source_adapter_sha256"] != job.source_adapter_sha256 + or capability["authority"] != OBSERVATION_ONLY_AUTHORITY + ): + raise PortableResultPackageIntegrityError( + "persisted portable source contract disagrees with the queue job" + ) + + +def _read_content_addressed_document(root: Path, document_sha256: str) -> dict[str, object]: + validate_digest(document_sha256, "portable source document sha256") + path = root / f"{document_sha256}.json" + try: + metadata = path.lstat() + payload = path.read_bytes() + except OSError as exc: + raise PortableResultPublicationBlockedError( + "an admitted portable source document is unavailable" + ) from exc + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or hashlib.sha256(payload).hexdigest() != document_sha256 + ): + raise PortableResultPackageIntegrityError( + "admitted portable source document digest changed" + ) + try: + decoded: object = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PortableResultPackageIntegrityError( + "admitted portable source document is not valid JSON" + ) from exc + document = object_document(decoded, "portable source document") + if payload != canonical_json(document): + raise PortableResultPackageIntegrityError( + "admitted portable source document is not canonical JSON" + ) + return document + + +def _verify_package_artifacts( + root: Path, + artifacts: Sequence[PortableResultArtifact], +) -> dict[str, Path]: + resolved: dict[str, Path] = {} + for artifact in artifacts: + path = _resolve_package_member(root, artifact.relative_path) + digest, byte_length = _hash_file(path) + if digest != artifact.sha256 or byte_length != artifact.byte_length: + raise PortableResultPackageIntegrityError( + f"portable result artifact content changed: {artifact.role}" + ) + resolved[artifact.role] = path + return resolved + + +def _resolve_package_member(root: Path, relative_path: str) -> Path: + portable = relative_artifact_path(relative_path) + candidate = root.joinpath(*portable.parts) + current = root + try: + for part in portable.parts: + current = current / part + metadata = current.lstat() + if stat.S_ISLNK(metadata.st_mode): + raise PortableResultPackageIntegrityError( + "portable result artifact path contains a symlink" + ) + resolved = candidate.resolve(strict=True) + metadata = candidate.lstat() + except PortableResultPublisherError: + raise + except OSError as exc: + raise PortableResultPackageIntegrityError( + "portable result artifact is unavailable" + ) from exc + if not resolved.is_relative_to(root) or not stat.S_ISREG(metadata.st_mode): + raise PortableResultPackageIntegrityError( + "portable result artifact escapes its package" + ) + return resolved + + +def _read_canonical_result_document(path: Path) -> dict[str, object]: + try: + payload = path.read_bytes() + decoded: object = json.loads(payload.decode("utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PortableResultPackageIntegrityError( + "portable result document is not valid JSON" + ) from exc + document = object_document(decoded, "portable result document") + if payload != canonical_json(document): + raise PortableResultPackageIntegrityError( + "portable result document is not canonical JSON" + ) + return document + + +def _publish_exact_member( + store: CentralArtifactStore, + *, + role: str, + media_type: str, + source: Path, + expected_sha256: str, + expected_byte_length: int, +) -> ArtifactMember: + try: + published = store.publish_file(source) + except (ArtifactGatewayError, OSError, ValueError) as exc: + raise PortableResultPackageIntegrityError( + f"portable result artifact could not be archived: {role}" + ) from exc + if ( + published.sha256 != expected_sha256 + or published.byte_length != expected_byte_length + ): + raise PortableResultPackageIntegrityError( + f"portable result artifact changed while it was archived: {role}" + ) + return ArtifactMember( + role=role, + media_type=media_type, + sha256=published.sha256, + byte_length=published.byte_length, + ) + + +def _publication_provenance( + *, + job: ObservatoryRecordedJob, + definition: PortableRunDefinition, + package: PortableResultPackageManifest, + artifact_manifest: ArtifactManifest, + profile: PortableCalculationProfilePolicy, +) -> dict[str, object]: + result_document = next( + artifact for artifact in package.artifacts if artifact.role == RESULT_DOCUMENT_ROLE + ) + return { + "schema_version": PORTABLE_RESULT_PUBLICATION_SCHEMA, + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + "calculation_profile": profile.as_dict(), + "calculation_profile_sha256": profile.identity_sha256, + "job": job_identity_document(job), + "source": source_identity_document(job), + "run_definition": run_definition_document(definition), + "result_package": { + "schema_version": PORTABLE_RESULT_PACKAGE_SCHEMA, + "manifest_sha256": package.manifest_sha256, + "identity_sha256": package.identity_sha256, + "artifact_manifest_id": artifact_manifest.manifest_id, + "result_document_sha256": result_document.sha256, + "artifacts": [artifact.as_dict() for artifact in package.artifacts], + }, + "storage": { + "mode": "central-content-addressed-artifact-store", + "include_recorded_media": profile.include_recorded_media, + "replay_capability": None, + }, + "method": _laboratory_method(job, definition), + } + + +def _laboratory_method( + job: ObservatoryRecordedJob, + definition: PortableRunDefinition, +) -> dict[str, object]: + components: list[dict[str, object]] = [ + { + "kind": "source", + "name": job.source_session_id, + "version": f"{job.source_adapter_id}/v{job.source_adapter_version}", + "role": "immutable admitted K1 source bundle", + "identity_sha256": job.source_bundle_sha256, + }, + { + "kind": "algorithm", + "name": definition.definition_id, + "version": f"v{definition.version}", + "role": "portable laboratory RunDefinition", + "identity_sha256": definition.definition_sha256, + }, + { + "kind": "runtime", + "name": job.executor_release_id, + "version": "sealed-release", + "role": "portable Worker executor", + "identity_sha256": job.executor_release_sha256, + }, + { + "kind": "runtime", + "name": job.resource_profile_id, + "version": "sealed-resource-profile", + "role": "exclusive Worker resource contract", + "identity_sha256": job.resource_profile_sha256, + }, + ] + components.extend( + { + "kind": "model", + "name": model.release_id, + "version": model.revision or "sealed-artifacts", + "role": "portable inference model", + "identity_sha256": model.identity_sha256, + } + for model in definition.models + ) + return { + "schema_version": "missioncore.laboratory-method/v1", + "completeness": "complete", + "execution_class": "hybrid" if definition.models else "deterministic", + "pipeline_id": definition.definition_id, + "components": components, + } + + +def _hash_file(path: Path) -> tuple[str, int]: + digest = hashlib.sha256() + byte_length = 0 + try: + with path.open("rb") as stream: + while chunk := stream.read(_COPY_CHUNK_BYTES): + digest.update(chunk) + byte_length += len(chunk) + except OSError as exc: + raise PortableResultPackageIntegrityError( + "portable result artifact could not be read" + ) from exc + return digest.hexdigest(), byte_length diff --git a/src/k1link/observatory/portable_run_definitions.py b/src/k1link/observatory/portable_run_definitions.py index 2904c4b..834c2ff 100644 --- a/src/k1link/observatory/portable_run_definitions.py +++ b/src/k1link/observatory/portable_run_definitions.py @@ -510,26 +510,46 @@ class PortableRunDefinition: by_kind: dict[str, list[ImmutableComponentIdentity]] = {} for component in self.components: by_kind.setdefault(component.kind, []).append(component) - for required_kind in ( - "calibration", + # Calibration is common to every admitted K1 source. Learned-model + # definitions additionally keep the original profile/runner/FOV seals; + # algorithm-only definitions such as M4.9 may remain projectable while + # their portable runner is not installed yet. + required_kinds: tuple[str, ...] = ("calibration",) + if self.models: + required_kinds += ( + "profile", + "runner", + "valid-fov-identity", + "valid-fov-mask", + ) + for required_kind in required_kinds: + if len(by_kind.get(required_kind, [])) != 1: + raise PortableRunDefinitionRegistryError( + f"portable definition requires exactly one {required_kind} component" + ) + for singleton_kind in ( "profile", "runner", "valid-fov-identity", "valid-fov-mask", ): - if len(by_kind.get(required_kind, [])) != 1: + if len(by_kind.get(singleton_kind, [])) > 1: raise PortableRunDefinitionRegistryError( - f"portable definition requires exactly one {required_kind} component" + f"portable definition allows at most one {singleton_kind} component" ) + if not self.models and self.executor.ready and len(by_kind.get("runner", [])) != 1: + raise PortableRunDefinitionRegistryError( + "ready portable definition requires exactly one runner component" + ) calibration = by_kind["calibration"][0] if calibration.sha256 != self.source_requirements.calibration_identity_sha256: raise PortableRunDefinitionRegistryError( "source capability and runtime calibration identities disagree" ) model_ids = [model.release_id for model in self.models] - if not model_ids or model_ids != sorted(model_ids) or len(model_ids) != len(set(model_ids)): + if model_ids != sorted(model_ids) or len(model_ids) != len(set(model_ids)): raise PortableRunDefinitionRegistryError( - "models must be non-empty, unique, and canonically ordered" + "models must be unique and canonically ordered" ) if self.executor.contour_id != self.resource_profile.contour_id: raise PortableRunDefinitionRegistryError( @@ -720,13 +740,38 @@ class PortableRunDefinitionRegistry: "portable setup and definition identity are not allowlisted" ) - def to_recorded_registry(self) -> RecordedRunDefinitionRegistry: - """Convert the complete registry and fail if any definition is blocked.""" + def resolve_setup(self, setup_id: str) -> PortableRunDefinition: + """Resolve the single current definition projected for one setup.""" - return RecordedRunDefinitionRegistry( - tuple(definition.to_recorded_run_definition() for definition in self.definitions) + _pattern(setup_id, _IDENTIFIER, "setup id") + for definition in self.definitions: + if definition.setup_id == setup_id: + return definition + raise PortableRunDefinitionRegistryError("portable setup is not allowlisted") + + def ready_recorded_definitions(self) -> tuple[RecordedRunDefinition, ...]: + """Return only definitions with fully sealed, installed executors. + + Blocked definitions remain valid catalog entries and do not prevent an + unrelated ready definition from entering the durable queue allowlist. + """ + + return tuple( + definition.to_recorded_run_definition() + for definition in self.definitions + if definition.executor.ready ) + def to_recorded_registry(self) -> RecordedRunDefinitionRegistry: + """Build a queue registry from ready definitions, ignoring blocked ones.""" + + definitions = self.ready_recorded_definitions() + if not definitions: + raise PortableRunDefinitionUnavailableError( + "portable registry has no sealed and installed executor" + ) + return RecordedRunDefinitionRegistry(definitions) + def canonical_sha256(value: object) -> str: """Return the repository-wide canonical JSON SHA-256 identity.""" diff --git a/src/k1link/observatory/portable_setup_projection.py b/src/k1link/observatory/portable_setup_projection.py index ab80090..34eff43 100644 --- a/src/k1link/observatory/portable_setup_projection.py +++ b/src/k1link/observatory/portable_setup_projection.py @@ -1,15 +1,9 @@ -"""UI-ready Observatory projection for the portable LAB V1 definition. +"""UI-ready Observatory projection for source-independent portable setups. -This module deliberately does not extend the legacy setup registry and does -not submit work. It projects two independent facts for one selected source: - -* the result of a lightweight recorded-source capability probe; -* the executor state sealed by the portable RunDefinition; - -Keeping those facts separate prevents a compatible new recording from being -described as incompatible merely because the executor is not installed yet. -Historical vegetation-shadow results belong only to the legacy catalog and -never become exact results of the generic portable v2 definition. +The projector keeps source capability, executor availability, and dispatch +availability separate. One blocked definition therefore remains visible +without hiding another definition or weakening either definition's admission +contract. Historical LAB results remain exclusively in the legacy catalog. """ from __future__ import annotations @@ -17,11 +11,16 @@ from __future__ import annotations import re from collections.abc import Callable from dataclasses import dataclass -from typing import Final, Protocol, runtime_checkable +from typing import Final, Protocol +from k1link.observatory.portable_result_contract import ( + PortableCalculationProfilePolicy, + PortableCalculationProfileRegistry, +) from k1link.observatory.portable_run_definitions import ( PortableRunDefinition, PortableRunDefinitionRegistry, + PortableRunDefinitionRegistryError, ) from k1link.observatory.source_admission import ( PortableRecordedSourceCapability, @@ -34,6 +33,8 @@ PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA: Final = ( ) PORTABLE_LAB_V1_SETUP_ID: Final = "lab-v1-eomt-ddrnet-portable-v1" PORTABLE_LAB_V1_DISPLAY_NAME: Final = "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39" +PORTABLE_M49_SETUP_ID: Final = "m49-tgs-portable-v2" +PORTABLE_M49_DISPLAY_NAME: Final = "M4.9T5 · TRAVEL TGS · CPU-only, без ML" _SOURCE_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") _OBSERVATION_ONLY_AUTHORITY: Final = { @@ -43,6 +44,25 @@ _OBSERVATION_ONLY_AUTHORITY: Final = { "production_accepted": False, } +_SETUP_PRESENTATION: Final = { + PORTABLE_LAB_V1_SETUP_ID: { + "lab_id": "LAB V1", + "display_name": PORTABLE_LAB_V1_DISPLAY_NAME, + "description": ("Проверка записанной K1-сессии моделями EoMT и DDRNet; только наблюдение."), + "compatible": "Запись соответствует требованиям EoMT + DDRNet.", + "incompatible": "Запись не соответствует требованиям EoMT + DDRNet.", + "executor_unavailable": "Вычислительный контур LAB V1 пока недоступен.", + }, + PORTABLE_M49_SETUP_ID: { + "lab_id": "LAB M4.9T5", + "display_name": PORTABLE_M49_DISPLAY_NAME, + "description": ("Динамический TGS-разбор записанной K1-сессии без ML; только наблюдение."), + "compatible": "Запись соответствует требованиям TRAVEL TGS.", + "incompatible": "Запись не соответствует требованиям TRAVEL TGS.", + "executor_unavailable": "Переносимый вычислительный контур M4.9T5 пока недоступен.", + }, +} + _MODEL_PRESENTATION: Final = { "eomt-cityscapes-large-1024-v1": "EoMT Cityscapes Large 1024", "lab-v1-ddrnet-39-goose-fine-64-v1": "DDRNet-39", @@ -50,82 +70,104 @@ _MODEL_PRESENTATION: Final = { class PortableSetupProjectionError(RuntimeError): - """The portable setup cannot be projected without weakening its contract.""" + """A portable setup cannot be projected without weakening its contract.""" -@runtime_checkable -class PortableSourceCapabilityProbeService(Protocol): - """A definition-bound lightweight source-capability service.""" +class PortableDefinitionCapabilityProbe(Protocol): + """Definition-bound lightweight capability probe.""" - def probe(self, source_session_id: str) -> PortableRecordedSourceCapability: - """Probe one source without preparing media or persisting documents.""" + def probe( + self, + *, + source_session_id: str, + setup_id: str, + definition_sha256: str, + ) -> PortableRecordedSourceCapability: + """Probe one source against one exact portable definition.""" -type PortableSourceCapabilityProbe = ( - Callable[[str], PortableRecordedSourceCapability] | PortableSourceCapabilityProbeService -) +type PortableSourceCapabilityProbe = Callable[[str], PortableRecordedSourceCapability] @dataclass(frozen=True, slots=True) class _SourceCompatibility: compatible: bool capability: PortableRecordedSourceCapability | None + reason: str def as_dict(self) -> dict[str, object]: return { "outcome": "pass" if self.compatible else "blocked", "compatible": self.compatible, - "reason": ( - "Запись соответствует требованиям EoMT + DDRNet." - if self.compatible - else "Запись не соответствует требованиям EoMT + DDRNet." - ), + "reason": self.reason, } -class PortableLabV1SetupProjector: - """Project the generic portable LAB V1 setup for one selected source.""" +class PortableSetupProjector: + """Project every allowlisted portable setup for one selected source.""" def __init__( self, *, registry: PortableRunDefinitionRegistry, - capability_probe: PortableSourceCapabilityProbe, + capability_probe: PortableDefinitionCapabilityProbe, + dispatch_available: bool = False, ) -> None: - self._definition = _resolve_lab_v1_definition(registry) - if not isinstance( - capability_probe, - PortableSourceCapabilityProbeService, - ) and not callable(capability_probe): + if not hasattr(capability_probe, "probe"): raise PortableSetupProjectionError("portable source capability probe is unavailable") + self._registry = registry self._capability_probe = capability_probe - _validate_model_presentation(self._definition) - if self._definition.authority.as_dict() != _OBSERVATION_ONLY_AUTHORITY: - raise PortableSetupProjectionError("portable LAB V1 authority is not observation-only") + self._dispatch_available = dispatch_available + for definition in registry.definitions: + _validate_model_presentation(definition) + if definition.authority.as_dict() != _OBSERVATION_ONLY_AUTHORITY: + raise PortableSetupProjectionError( + "portable setup authority is not observation-only" + ) + + def has_setup(self, setup_id: str) -> bool: + try: + self._registry.resolve_setup(setup_id) + except (PortableRunDefinitionRegistryError, ValueError): + return False + return True def catalog(self, source: SessionSummary) -> dict[str, object]: - """Return a one-setup v2 catalog projection for ``source``.""" + """Return all independent portable setup projections for ``source``.""" return { "schema_version": PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA, "source_session_id": source.session_id, - "setups": [self.project(source)], + "setups": [ + self.project(source, setup_id=definition.setup_id) + for definition in self._registry.definitions + ], "authority": dict(_OBSERVATION_ONLY_AUTHORITY), } - def project(self, source: SessionSummary) -> dict[str, object]: - """Return a strict, observation-only setup projection.""" + def project( + self, + source: SessionSummary, + *, + setup_id: str, + ) -> dict[str, object]: + """Return one strict observation-only portable setup projection.""" _validate_source_id(source.session_id) - compatibility = self._probe_source(source.session_id) - definition = self._definition + try: + definition = self._registry.resolve_setup(setup_id) + except PortableRunDefinitionRegistryError as exc: + raise PortableSetupProjectionError("portable setup is unavailable") from exc + presentation = _presentation(definition) + compatibility = self._probe_source(definition, source.session_id, presentation) executor = definition.executor + submission_allowed = ( + compatibility.compatible and executor.ready and self._dispatch_available + ) return { "setup_id": definition.setup_id, - "display_name": PORTABLE_LAB_V1_DISPLAY_NAME, - "description": ( - "Проверка записанной K1-сессии моделями EoMT и DDRNet; только наблюдение." - ), + "display_name": presentation["display_name"], + "description": presentation["description"], "origin": "portable-definition", "source_requirements": definition.source_requirements.as_dict(), "run_definition": { @@ -141,65 +183,169 @@ class PortableLabV1SetupProjector: "contour_id": executor.contour_id, "state": executor.state, "ready": executor.ready, + "reason_code": executor.reason_code, "reason": executor.reason, }, "existing_results": [], "preflight": { - "outcome": "blocked", - "action": "blocked", - "reason": self._preflight_reason(compatibility), - "submission_allowed": False, + "outcome": "ready" if submission_allowed else "blocked", + "action": "check" if submission_allowed else "blocked", + "reason": self._preflight_reason( + definition, + compatibility, + presentation, + ), + "submission_allowed": submission_allowed, "existing_result_ids": [], }, "authority": dict(_OBSERVATION_ONLY_AUTHORITY), } - def _probe_source(self, source_session_id: str) -> _SourceCompatibility: + def _probe_source( + self, + definition: PortableRunDefinition, + source_session_id: str, + presentation: dict[str, str], + ) -> _SourceCompatibility: try: - if isinstance( - self._capability_probe, - PortableSourceCapabilityProbeService, - ): - capability = self._capability_probe.probe(source_session_id) - else: - capability = self._capability_probe(source_session_id) + capability = self._capability_probe.probe( + source_session_id=source_session_id, + setup_id=definition.setup_id, + definition_sha256=definition.definition_sha256, + ) except PortableSourceAdmissionError: - return _SourceCompatibility(compatible=False, capability=None) + return _SourceCompatibility( + compatible=False, + capability=None, + reason=presentation["incompatible"], + ) if not isinstance(capability, PortableRecordedSourceCapability): raise PortableSetupProjectionError("capability probe returned an invalid result") if capability.source_session_id != source_session_id: raise PortableSetupProjectionError( "capability probe is bound to another source session" ) - if capability.source_adapter_sha256 != self._definition.source_adapter.contract_sha256: + if capability.source_adapter_sha256 != definition.source_adapter.contract_sha256: raise PortableSetupProjectionError("capability probe uses another source adapter") - return _SourceCompatibility(compatible=True, capability=capability) - - def _preflight_reason(self, compatibility: _SourceCompatibility) -> str: - if not compatibility.compatible: - return "Запись не соответствует требованиям этого сетапа." - if not self._definition.executor.ready: - return "Вычислительный контур LAB V1 пока недоступен." - return ( - "Server-side проверка definition/check SHA и постановка portable " - "LAB V1 в очередь пока недоступны." + return _SourceCompatibility( + compatible=True, + capability=capability, + reason=presentation["compatible"], ) + def _preflight_reason( + self, + definition: PortableRunDefinition, + compatibility: _SourceCompatibility, + presentation: dict[str, str], + ) -> str: + if not compatibility.compatible: + return "Запись не соответствует требованиям этого сетапа." + if not definition.executor.ready: + return presentation["executor_unavailable"] + if not self._dispatch_available: + return "Server-side проверка и постановка portable-сетапа в очередь недоступны." + return "Сетап готов к server-side проверке источника перед постановкой в очередь." -def _resolve_lab_v1_definition( + +class PortableLabV1SetupProjector(PortableSetupProjector): + """Backward-compatible single-definition LAB V1 projector.""" + + def __init__( + self, + *, + registry: PortableRunDefinitionRegistry, + capability_probe: PortableSourceCapabilityProbe | object, + ) -> None: + try: + definition = registry.resolve_setup(PORTABLE_LAB_V1_SETUP_ID) + except PortableRunDefinitionRegistryError as exc: + raise PortableSetupProjectionError( + "portable LAB V1 definition is unavailable or ambiguous" + ) from exc + + class _LegacyProbeAdapter: + def probe( + adapter_self, + *, + source_session_id: str, + setup_id: str, + definition_sha256: str, + ) -> PortableRecordedSourceCapability: + del adapter_self + if ( + setup_id != definition.setup_id + or definition_sha256 != definition.definition_sha256 + ): + raise PortableSetupProjectionError("portable LAB V1 definition changed") + probe_method = getattr(capability_probe, "probe", None) + if callable(probe_method): + capability = probe_method(source_session_id) + elif callable(capability_probe): + capability = capability_probe(source_session_id) + else: + raise PortableSetupProjectionError( + "portable source capability probe is unavailable" + ) + if not isinstance(capability, PortableRecordedSourceCapability): + raise PortableSetupProjectionError( + "capability probe returned an invalid result" + ) + return capability + + super().__init__( + registry=PortableRunDefinitionRegistry((definition,)), + capability_probe=_LegacyProbeAdapter(), + dispatch_available=False, + ) + + def project( + self, + source: SessionSummary, + *, + setup_id: str = PORTABLE_LAB_V1_SETUP_ID, + ) -> dict[str, object]: + return super().project(source, setup_id=setup_id) + + +def _presentation(definition: PortableRunDefinition) -> dict[str, str]: + configured = _SETUP_PRESENTATION.get(definition.setup_id) + if configured is not None: + return dict(configured) + return { + "lab_id": "LAB PORTABLE", + "display_name": definition.setup_id, + "description": "Переносимый анализ записанной K1-сессии; только наблюдение.", + "compatible": "Запись соответствует требованиям сетапа.", + "incompatible": "Запись не соответствует требованиям сетапа.", + "executor_unavailable": "Вычислительный контур сетапа пока недоступен.", + } + + +def portable_calculation_profile_registry( registry: PortableRunDefinitionRegistry, -) -> PortableRunDefinition: - matching = tuple( - definition - for definition in registry.definitions - if definition.setup_id == PORTABLE_LAB_V1_SETUP_ID - ) - if len(matching) != 1: - raise PortableSetupProjectionError("portable LAB V1 definition is unavailable or ambiguous") - return matching[0] +) -> PortableCalculationProfileRegistry: + """Build exact publication policies from the server-owned presentation map.""" + + policies: list[PortableCalculationProfilePolicy] = [] + for definition in registry.definitions: + presentation = _presentation(definition) + policies.append( + PortableCalculationProfilePolicy( + setup_id=definition.setup_id, + definition_id=definition.definition_id, + definition_version=definition.version, + definition_sha256=definition.definition_sha256, + lab_id=presentation["lab_id"], + display_name=presentation["display_name"], + ) + ) + return PortableCalculationProfileRegistry(tuple(policies)) def _validate_model_presentation(definition: PortableRunDefinition) -> None: + if definition.setup_id != PORTABLE_LAB_V1_SETUP_ID: + return releases = {model.release_id: model for model in definition.models} if set(releases) != set(_MODEL_PRESENTATION): raise PortableSetupProjectionError( @@ -217,15 +363,14 @@ def _validate_model_presentation(definition: PortableRunDefinition) -> None: def _project_models(definition: PortableRunDefinition) -> list[dict[str, object]]: - by_release = {model.release_id: model for model in definition.models} return [ { - "name": _MODEL_PRESENTATION[release_id], - "release_id": release_id, - "model_id": by_release[release_id].model_id, - "architecture": by_release[release_id].architecture, + "name": _MODEL_PRESENTATION.get(model.release_id, model.model_id), + "release_id": model.release_id, + "model_id": model.model_id, + "architecture": model.architecture, } - for release_id in _MODEL_PRESENTATION + for model in definition.models ] diff --git a/src/k1link/observatory/portable_worker_integration.py b/src/k1link/observatory/portable_worker_integration.py new file mode 100644 index 0000000..f8c4638 --- /dev/null +++ b/src/k1link/observatory/portable_worker_integration.py @@ -0,0 +1,375 @@ +"""Fail-closed server composition for portable Observatory Worker jobs. + +This module is deliberately only a composition boundary. It does not enable +the Worker router, install an executor, select commands, or grant production +authority. It binds the two admitted portable profiles to their exact result +contracts, then constructs the local artifact transport and verified result +publisher from server-owned dependencies. +""" + +from __future__ import annotations + +import os +import stat +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +from k1link.artifact_gateway import CentralArtifactStore +from k1link.observatory.m49_portable_result import ( + M49_PORTABLE_RESULT_CONTRACT_SHA256, + M49_PORTABLE_RESULT_SCHEMA, + validate_m49_portable_result, +) +from k1link.observatory.portable_artifact_transport import ( + PortableArtifactTransportError, + PortableObservatoryArtifactTransport, +) +from k1link.observatory.portable_lab_v1_executor import ( + PORTABLE_LAB_V1_RESULT_SCHEMA, + validate_lab_v1_result_v2, +) +from k1link.observatory.portable_result_contract import ( + OBSERVATION_ONLY_AUTHORITY, + PortableCalculationProfileRegistry, + PortableResultContractValidator, + PortableResultContractValidatorRegistration, + PortableResultContractValidatorRegistry, + PortableResultPublicationBlockedError, +) +from k1link.observatory.portable_result_publisher import ( + PortableObservatoryResultPublisher, +) +from k1link.observatory.portable_run_definitions import ( + PORTABLE_RESULT_CONTRACT_SCHEMA, + PortableRunDefinition, + PortableRunDefinitionRegistry, + PortableRunDefinitionRegistryError, +) +from k1link.observatory.portable_setup_projection import ( + PORTABLE_LAB_V1_SETUP_ID, + PORTABLE_M49_SETUP_ID, + portable_calculation_profile_registry, +) +from k1link.observatory.recorded_jobs import ObservatoryRecordedJobQueue +from k1link.sessions.media import RecordedMediaInspector +from k1link.sessions.store import SessionStore + +PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256: Final = ( + "b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a" +) +OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_SOURCE_CAS_ROOT" +OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV: Final = ( + "MISSIONCORE_OBSERVATORY_WORKER_RESULT_STAGING_ROOT" +) + + +class PortableWorkerIntegrationError(RuntimeError): + """The exact portable server composition cannot be constructed.""" + + +@dataclass(frozen=True, slots=True) +class PortableWorkerStorageRoots: + """Pre-provisioned server-owned roots for large portable artifacts.""" + + source_cas_root: Path + result_staging_root: Path + + @classmethod + def from_environment( + cls, + *, + artifact_store_root: Path, + environment: Mapping[str, str] | None = None, + ) -> PortableWorkerStorageRoots: + """Load both roots without creating anything on a missing mount.""" + + values = os.environ if environment is None else environment + return cls.from_paths( + artifact_store_root=artifact_store_root, + source_cas_root=_required_environment_path( + values, + OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV, + ), + result_staging_root=_required_environment_path( + values, + OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV, + ), + ) + + @classmethod + def from_paths( + cls, + *, + artifact_store_root: Path, + source_cas_root: Path, + result_staging_root: Path, + ) -> PortableWorkerStorageRoots: + """Validate existing, disjoint roots inside the central store boundary.""" + + artifact_store = _existing_canonical_directory( + artifact_store_root, + "central artifact store", + ) + storage_boundary = _existing_canonical_directory( + artifact_store.parent, + "central artifact storage boundary", + ) + _require_mounted_volume(storage_boundary) + source_cas = _existing_canonical_directory( + source_cas_root, + "portable source CAS", + ) + result_staging = _existing_canonical_directory( + result_staging_root, + "portable result staging", + ) + for label, root in ( + ("portable source CAS", source_cas), + ("portable result staging", result_staging), + ): + if root == storage_boundary or not root.is_relative_to(storage_boundary): + raise PortableWorkerIntegrationError( + f"{label} must be inside the central artifact storage boundary" + ) + if _paths_overlap(root, artifact_store): + raise PortableWorkerIntegrationError( + f"{label} must be disjoint from the central artifact store" + ) + if _paths_overlap(source_cas, result_staging): + raise PortableWorkerIntegrationError( + "portable source CAS and result staging roots must be disjoint" + ) + return cls( + source_cas_root=source_cas, + result_staging_root=result_staging, + ) + + +@dataclass(frozen=True, slots=True) +class PortableObservatoryWorkerIntegration: + """Server-owned objects required by the optional Worker router.""" + + calculation_profiles: PortableCalculationProfileRegistry + validators: PortableResultContractValidatorRegistry + artifact_transport: PortableObservatoryArtifactTransport + result_publisher: PortableObservatoryResultPublisher + supported_setup_ids: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class _ValidatorSpec: + setup_id: str + definition_id: str + definition_version: int + contract_id: str + contract_version: int + result_schema: str + result_kind: str + contract_sha256: str + validator: PortableResultContractValidator + + def expected_contract(self) -> dict[str, object]: + return { + "schema_version": PORTABLE_RESULT_CONTRACT_SCHEMA, + "contract_id": self.contract_id, + "version": self.contract_version, + "result_schema": self.result_schema, + "result_kind": self.result_kind, + "publication": "observatory", + "contract_sha256": self.contract_sha256, + } + + +_VALIDATOR_SPECS: Final = ( + _ValidatorSpec( + setup_id=PORTABLE_LAB_V1_SETUP_ID, + definition_id="lab-v1-eomt-ddrnet-portable", + definition_version=2, + contract_id="recorded-eomt-ddrnet-review-v2", + contract_version=2, + result_schema=PORTABLE_LAB_V1_RESULT_SCHEMA, + result_kind="recorded-perception-qualification", + contract_sha256=PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256, + validator=validate_lab_v1_result_v2, + ), + _ValidatorSpec( + setup_id=PORTABLE_M49_SETUP_ID, + definition_id="m49-tgs-portable", + definition_version=2, + contract_id="m49-tgs-portable-review-v2", + contract_version=2, + result_schema=M49_PORTABLE_RESULT_SCHEMA, + result_kind="recorded-perception-qualification", + contract_sha256=M49_PORTABLE_RESULT_CONTRACT_SHA256, + validator=validate_m49_portable_result, + ), +) + + +def portable_result_validator_registry( + definitions: PortableRunDefinitionRegistry, +) -> PortableResultContractValidatorRegistry: + """Bind both product profiles to fixed result contracts and validators.""" + + registrations: list[PortableResultContractValidatorRegistration] = [] + for spec in _VALIDATOR_SPECS: + try: + definition = definitions.resolve_setup(spec.setup_id) + except PortableRunDefinitionRegistryError as exc: + raise PortableWorkerIntegrationError( + f"required portable setup is unavailable: {spec.setup_id}" + ) from exc + _verify_validator_definition(definition, spec) + registrations.append( + PortableResultContractValidatorRegistration( + contract_sha256=spec.contract_sha256, + validator=spec.validator, + ) + ) + return PortableResultContractValidatorRegistry(tuple(registrations)) + + +def build_portable_observatory_worker_integration( + *, + queue: ObservatoryRecordedJobQueue, + session_store: SessionStore, + media_inspector: RecordedMediaInspector, + definitions: PortableRunDefinitionRegistry, + artifact_store: CentralArtifactStore, + calculation_profiles: PortableCalculationProfileRegistry | None = None, + validators: PortableResultContractValidatorRegistry | None = None, + source_cas_root: Path | None = None, + result_staging_root: Path | None = None, +) -> PortableObservatoryWorkerIntegration: + """Construct the dormant server foundation without enabling any route.""" + + try: + if source_cas_root is None or result_staging_root is None: + raise PortableWorkerIntegrationError( + "portable Worker storage roots must be explicitly configured" + ) + storage_roots = PortableWorkerStorageRoots.from_paths( + artifact_store_root=artifact_store.root, + source_cas_root=source_cas_root, + result_staging_root=result_staging_root, + ) + profiles = calculation_profiles or portable_calculation_profile_registry(definitions) + validator_registry = validators or portable_result_validator_registry(definitions) + _verify_composition(definitions, profiles, validator_registry) + transport = PortableObservatoryArtifactTransport( + queue=queue, + session_store=session_store, + media_inspector=media_inspector, + definitions=definitions, + source_cas_root=storage_roots.source_cas_root, + result_staging_root=storage_roots.result_staging_root, + ) + except PortableWorkerIntegrationError: + raise + except ( + PortableArtifactTransportError, + PortableResultPublicationBlockedError, + PortableRunDefinitionRegistryError, + OSError, + ValueError, + ) as exc: + raise PortableWorkerIntegrationError( + "portable Worker server composition is unavailable" + ) from exc + publisher = PortableObservatoryResultPublisher( + session_store=session_store, + artifact_store=artifact_store, + definitions=definitions, + calculation_profiles=profiles, + validators=validator_registry, + ) + return PortableObservatoryWorkerIntegration( + calculation_profiles=profiles, + validators=validator_registry, + artifact_transport=transport, + result_publisher=publisher, + supported_setup_ids=tuple(spec.setup_id for spec in _VALIDATOR_SPECS), + ) + + +def _verify_validator_definition( + definition: PortableRunDefinition, + spec: _ValidatorSpec, +) -> None: + if ( + definition.definition_id != spec.definition_id + or definition.version != spec.definition_version + or definition.result_contract.as_dict() != spec.expected_contract() + or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY + ): + raise PortableWorkerIntegrationError( + f"portable result contract changed for setup: {spec.setup_id}" + ) + + +def _verify_composition( + definitions: PortableRunDefinitionRegistry, + profiles: PortableCalculationProfileRegistry, + validators: PortableResultContractValidatorRegistry, +) -> None: + expected_validators = {spec.contract_sha256: spec.validator for spec in _VALIDATOR_SPECS} + for definition in definitions.definitions: + profiles.resolve(definition) + if definition.executor.ready: + try: + validators.resolve(definition.result_contract.contract_sha256) + except PortableResultPublicationBlockedError as exc: + raise PortableWorkerIntegrationError( + "a ready portable definition has no exact result validator" + ) from exc + for contract_sha256, expected in expected_validators.items(): + if validators.resolve(contract_sha256) is not expected: + raise PortableWorkerIntegrationError( + "portable result validator registration changed identity" + ) + + +def _required_environment_path( + environment: Mapping[str, str], + name: str, +) -> Path: + raw = environment.get(name) + if not isinstance(raw, str) or not raw or raw != raw.strip(): + raise PortableWorkerIntegrationError(f"{name} is required") + path = Path(raw) + if not path.is_absolute(): + raise PortableWorkerIntegrationError(f"{name} must be an absolute path") + return path + + +def _existing_canonical_directory(path: Path, label: str) -> Path: + candidate = path.expanduser().absolute() + try: + metadata = candidate.lstat() + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise PortableWorkerIntegrationError(f"{label} is unavailable") from exc + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISDIR(metadata.st_mode) + or resolved != candidate + ): + raise PortableWorkerIntegrationError(f"{label} is not a canonical directory") + return resolved + + +def _paths_overlap(left: Path, right: Path) -> bool: + return left == right or left.is_relative_to(right) or right.is_relative_to(left) + + +def _require_mounted_volume(path: Path) -> None: + parts = path.parts + if len(parts) < 3 or parts[0] != os.sep or parts[1] != "Volumes": + return + mount_point = Path(os.sep, "Volumes", parts[2]) + if not os.path.ismount(mount_point): + raise PortableWorkerIntegrationError( + f"central artifact volume is not mounted: {mount_point}" + ) diff --git a/src/k1link/observatory/portable_worker_runtime.py b/src/k1link/observatory/portable_worker_runtime.py new file mode 100644 index 0000000..a3cd2aa --- /dev/null +++ b/src/k1link/observatory/portable_worker_runtime.py @@ -0,0 +1,969 @@ +"""Fail-closed local runtime contract for portable Observatory executors. + +This module belongs on Worker 006, not in the browser or the K1 control path. +It binds one source-independent portable RunDefinition to locally verified +assets and to local Python adapters. A queued job can select an adapter only +through the four server-sealed executor digests; it can never supply a command, +path, environment variable, image reference, or priority. + +The production candidate registry is deliberately allowed to contain blocked +candidates. Reusable historical assets are evidence, not an installed +executor. A blocked candidate cannot yield a Worker registration or execute a +job even when every reusable asset is present. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal, Protocol + +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinition, + PortableRunDefinitionRegistry, + canonical_sha256, +) +from k1link.observatory.worker_agent import ( + ObservatoryWorkerExecutionResult, + ObservatoryWorkerExecutorIdentity, + SealedObservatoryRecordedJob, +) + +PORTABLE_WORKER_RUNTIME_REGISTRY_SCHEMA: Final = ( + "missioncore.observatory-portable-worker-runtime-registry/v1" +) +PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA: Final = ( + "missioncore.observatory-portable-worker-runtime-candidate/v1" +) +PORTABLE_WORKER_RUNTIME_PLAN_SCHEMA: Final = ( + "missioncore.observatory-portable-worker-runtime-plan/v1" +) + +_MAX_REGISTRY_BYTES: Final = 256 * 1024 +_SHA256: Final = re.compile(r"^[a-f0-9]{64}$") +_IDENTIFIER: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$") +_ASSET_ID: Final = re.compile(r"^[a-z][a-z0-9.-]{2,127}$") +_SESSION_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +_JOB_ID: Final = re.compile(r"^observatory-run-[a-f0-9]{32}$") + +_AUTHORITY: Final = { + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, + "production_accepted": False, +} + +type CandidateState = Literal["blocked", "ready"] +type PhaseState = Literal["implemented", "missing"] +type RuntimeAssetKind = Literal[ + "container-image", + "definition-component", + "local-file", + "model-artifact", +] +type AssetVerificationState = Literal["matched", "missing", "mismatched"] + + +class PortableWorkerRuntimeError(RuntimeError): + """Base error for the local portable Worker runtime boundary.""" + + +class PortableWorkerRuntimeRegistryError(PortableWorkerRuntimeError): + """A candidate registry is malformed or drifts from a RunDefinition.""" + + +class PortableWorkerRuntimeUnavailableError(PortableWorkerRuntimeError): + """A candidate is not a complete, sealed, locally admitted executor.""" + + +class PortableWorkerRuntimeJobRejectedError(PortableWorkerRuntimeError): + """A Worker job differs from the exact local candidate identity.""" + + +@dataclass(frozen=True, slots=True) +class PortableWorkerRuntimePhase: + phase_id: str + state: PhaseState + + def __post_init__(self) -> None: + _pattern(self.phase_id, _IDENTIFIER, "runtime phase id") + if self.state not in ("implemented", "missing"): + raise PortableWorkerRuntimeRegistryError("runtime phase state is invalid") + + def as_dict(self) -> dict[str, str]: + return {"phase_id": self.phase_id, "state": self.state} + + +@dataclass(frozen=True, slots=True) +class PortableWorkerAssetRequirement: + """One exact reusable local asset; its locator is intentionally absent.""" + + asset_id: str + kind: RuntimeAssetKind + sha256: str + byte_length: int | None + component_id: str | None + model_release_id: str | None + model_artifact_role: str | None + + def __post_init__(self) -> None: + _pattern(self.asset_id, _ASSET_ID, "runtime asset id") + if self.kind not in ( + "container-image", + "definition-component", + "local-file", + "model-artifact", + ): + raise PortableWorkerRuntimeRegistryError("runtime asset kind is invalid") + _digest(self.sha256, "runtime asset sha256") + if self.byte_length is not None and ( + isinstance(self.byte_length, bool) + or not isinstance(self.byte_length, int) + or self.byte_length < 1 + ): + raise PortableWorkerRuntimeRegistryError("runtime asset byte length is invalid") + if self.kind == "container-image": + if any( + value is not None + for value in ( + self.byte_length, + self.component_id, + self.model_release_id, + self.model_artifact_role, + ) + ): + raise PortableWorkerRuntimeRegistryError( + "container image requirement cannot impersonate a definition asset" + ) + elif self.kind == "definition-component": + _optional_identifier(self.component_id, "definition component id") + if self.component_id is None or any( + value is not None + for value in (self.model_release_id, self.model_artifact_role) + ): + raise PortableWorkerRuntimeRegistryError( + "definition component requirement is incomplete" + ) + elif self.kind == "model-artifact": + _optional_identifier(self.model_release_id, "model release id") + _optional_identifier(self.model_artifact_role, "model artifact role") + if ( + self.model_release_id is None + or self.model_artifact_role is None + or self.component_id is not None + or self.byte_length is None + ): + raise PortableWorkerRuntimeRegistryError( + "model artifact requirement is incomplete" + ) + elif any( + value is not None + for value in ( + self.component_id, + self.model_release_id, + self.model_artifact_role, + ) + ): + raise PortableWorkerRuntimeRegistryError( + "local file requirement cannot impersonate a definition asset" + ) + + def as_dict(self) -> dict[str, object]: + return { + "asset_id": self.asset_id, + "kind": self.kind, + "sha256": self.sha256, + "byte_length": self.byte_length, + "component_id": self.component_id, + "model_release_id": self.model_release_id, + "model_artifact_role": self.model_artifact_role, + } + + +@dataclass(frozen=True, slots=True) +class PortableWorkerExecutorSeal: + release_id: str + release_sha256: str + image_sha256: str + + def __post_init__(self) -> None: + _pattern(self.release_id, _IDENTIFIER, "executor release id") + _digest(self.release_sha256, "executor release sha256") + _digest(self.image_sha256, "executor image sha256") + + def as_dict(self) -> dict[str, str]: + return { + "release_id": self.release_id, + "release_sha256": self.release_sha256, + "image_sha256": self.image_sha256, + } + + +@dataclass(frozen=True, slots=True) +class PortableWorkerRuntimeCandidate: + adapter_id: str + setup_id: str + definition_id: str + definition_version: int + definition_sha256: str + source_adapter_sha256: str + model_manifest_sha256: str + resource_profile_sha256: str + result_contract_sha256: str + state: CandidateState + executor: PortableWorkerExecutorSeal | None + reusable_assets: tuple[PortableWorkerAssetRequirement, ...] + phases: tuple[PortableWorkerRuntimePhase, ...] + blockers: tuple[str, ...] + candidate_sha256: str + + def __post_init__(self) -> None: + for value, label in ( + (self.adapter_id, "runtime adapter id"), + (self.setup_id, "setup id"), + (self.definition_id, "definition id"), + ): + _pattern(value, _IDENTIFIER, label) + if ( + isinstance(self.definition_version, bool) + or not isinstance(self.definition_version, int) + or self.definition_version < 1 + ): + raise PortableWorkerRuntimeRegistryError("definition version is invalid") + for value, label in ( + (self.definition_sha256, "definition sha256"), + (self.source_adapter_sha256, "source adapter sha256"), + (self.model_manifest_sha256, "model manifest sha256"), + (self.resource_profile_sha256, "resource profile sha256"), + (self.result_contract_sha256, "result contract sha256"), + (self.candidate_sha256, "runtime candidate sha256"), + ): + _digest(value, label) + if self.state not in ("blocked", "ready"): + raise PortableWorkerRuntimeRegistryError("runtime candidate state is invalid") + asset_ids = [asset.asset_id for asset in self.reusable_assets] + if asset_ids != sorted(asset_ids) or len(asset_ids) != len(set(asset_ids)): + raise PortableWorkerRuntimeRegistryError( + "runtime assets must be unique and canonically ordered" + ) + phase_ids = [phase.phase_id for phase in self.phases] + if not phase_ids or len(phase_ids) != len(set(phase_ids)): + raise PortableWorkerRuntimeRegistryError("runtime phases must be non-empty and unique") + if self.blockers != tuple(sorted(self.blockers)) or len(self.blockers) != len( + set(self.blockers) + ): + raise PortableWorkerRuntimeRegistryError( + "runtime blockers must be unique and canonically ordered" + ) + for blocker in self.blockers: + _pattern(blocker, _IDENTIFIER, "runtime blocker") + missing_phases = tuple( + phase.phase_id for phase in self.phases if phase.state == "missing" + ) + if self.state == "ready": + if self.executor is None or self.blockers or missing_phases: + raise PortableWorkerRuntimeRegistryError( + "ready runtime requires a sealed executor and complete phases" + ) + elif self.executor is not None or not self.blockers or not missing_phases: + raise PortableWorkerRuntimeRegistryError( + "blocked runtime must keep its executor unsealed and missing phases explicit" + ) + if self.candidate_sha256 != canonical_sha256(self.identity_document()): + raise PortableWorkerRuntimeRegistryError("runtime candidate digest changed") + + @property + def ready(self) -> bool: + return self.state == "ready" + + def identity_document(self) -> dict[str, object]: + return { + "schema_version": PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA, + "adapter_id": self.adapter_id, + "setup_id": self.setup_id, + "definition_id": self.definition_id, + "definition_version": self.definition_version, + "definition_sha256": self.definition_sha256, + "source_adapter_sha256": self.source_adapter_sha256, + "model_manifest_sha256": self.model_manifest_sha256, + "resource_profile_sha256": self.resource_profile_sha256, + "result_contract_sha256": self.result_contract_sha256, + "state": self.state, + "executor": self.executor.as_dict() if self.executor is not None else None, + "reusable_assets": [asset.as_dict() for asset in self.reusable_assets], + "phases": [phase.as_dict() for phase in self.phases], + "blockers": list(self.blockers), + "authority": dict(_AUTHORITY), + } + + def bind_definition(self, definition: PortableRunDefinition) -> None: + if ( + definition.setup_id != self.setup_id + or definition.definition_id != self.definition_id + or definition.version != self.definition_version + or definition.definition_sha256 != self.definition_sha256 + or definition.source_adapter.contract_sha256 != self.source_adapter_sha256 + or definition.model_manifest_sha256 != self.model_manifest_sha256 + or definition.resource_profile.profile_sha256 != self.resource_profile_sha256 + or definition.result_contract.contract_sha256 != self.result_contract_sha256 + ): + raise PortableWorkerRuntimeRegistryError( + "runtime candidate and portable RunDefinition identities disagree" + ) + components = {component.component_id: component for component in definition.components} + models = {model.release_id: model for model in definition.models} + for requirement in self.reusable_assets: + if requirement.kind == "definition-component": + component = components.get(requirement.component_id or "") + if component is None or component.sha256 != requirement.sha256: + raise PortableWorkerRuntimeRegistryError( + "runtime component requirement differs from its RunDefinition" + ) + elif requirement.kind == "model-artifact": + model = models.get(requirement.model_release_id or "") + artifacts = { + artifact.role: artifact for artifact in model.artifacts + } if model is not None else {} + artifact = artifacts.get(requirement.model_artifact_role or "") + if ( + artifact is None + or artifact.sha256 != requirement.sha256 + or artifact.byte_length != requirement.byte_length + ): + raise PortableWorkerRuntimeRegistryError( + "runtime model artifact differs from its RunDefinition" + ) + if self.state == "blocked": + if definition.executor.ready: + raise PortableWorkerRuntimeRegistryError( + "blocked local runtime cannot bind a ready RunDefinition" + ) + return + if not definition.executor.ready or self.executor is None: + raise PortableWorkerRuntimeRegistryError( + "ready local runtime requires a ready RunDefinition" + ) + if ( + definition.executor.release_id != self.executor.release_id + or definition.executor.release_sha256 != self.executor.release_sha256 + or definition.executor.image_sha256 != self.executor.image_sha256 + ): + raise PortableWorkerRuntimeRegistryError( + "local executor seal differs from its RunDefinition" + ) + + def executor_identity(self) -> ObservatoryWorkerExecutorIdentity: + if not self.ready or self.executor is None: + raise PortableWorkerRuntimeUnavailableError( + "blocked runtime candidate has no executor identity" + ) + return ObservatoryWorkerExecutorIdentity( + release_sha256=self.executor.release_sha256, + image_sha256=self.executor.image_sha256, + model_manifest_sha256=self.model_manifest_sha256, + resource_profile_sha256=self.resource_profile_sha256, + ) + + +@dataclass(frozen=True, slots=True) +class PortableWorkerRuntimeRegistry: + candidates: tuple[PortableWorkerRuntimeCandidate, ...] + + def __post_init__(self) -> None: + if not self.candidates: + raise PortableWorkerRuntimeRegistryError("runtime registry cannot be empty") + for label, values in ( + ("runtime adapter ids", [candidate.adapter_id for candidate in self.candidates]), + ("runtime setup ids", [candidate.setup_id for candidate in self.candidates]), + ( + "runtime candidate digests", + [candidate.candidate_sha256 for candidate in self.candidates], + ), + ): + if len(values) != len(set(values)): + raise PortableWorkerRuntimeRegistryError(f"{label} must be unique") + + @classmethod + def from_file( + cls, + path: Path, + *, + definitions: PortableRunDefinitionRegistry, + ) -> PortableWorkerRuntimeRegistry: + candidate_path = path.expanduser().absolute() + if candidate_path.is_symlink() or not candidate_path.is_file(): + raise PortableWorkerRuntimeRegistryError( + "runtime registry must be a regular file" + ) + try: + if candidate_path.stat().st_size > _MAX_REGISTRY_BYTES: + raise PortableWorkerRuntimeRegistryError("runtime registry is too large") + document: object = json.loads(candidate_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PortableWorkerRuntimeRegistryError("runtime registry is unreadable") from exc + _reject_unsafe_keys(document) + root = _object(document, "runtime registry") + _exact_keys(root, {"schema_version", "candidates"}, "runtime registry") + if root["schema_version"] != PORTABLE_WORKER_RUNTIME_REGISTRY_SCHEMA: + raise PortableWorkerRuntimeRegistryError("runtime registry schema is invalid") + rows = _array(root["candidates"], "runtime candidates") + registry = cls(tuple(_candidate(row) for row in rows)) + for candidate in registry.candidates: + candidate.bind_definition( + definitions.resolve(candidate.setup_id, candidate.definition_sha256) + ) + return registry + + def resolve(self, setup_id: str, definition_sha256: str) -> PortableWorkerRuntimeCandidate: + _pattern(setup_id, _IDENTIFIER, "setup id") + _digest(definition_sha256, "definition sha256") + for candidate in self.candidates: + if ( + candidate.setup_id == setup_id + and candidate.definition_sha256 == definition_sha256 + ): + return candidate + raise PortableWorkerRuntimeRegistryError( + "portable Worker candidate identity is not allowlisted" + ) + + +@dataclass(frozen=True, slots=True) +class PortableWorkerLocalAssetBinding: + """Worker-local binding populated by reviewed local configuration only.""" + + asset_id: str + file_path: Path | None = None + image_sha256: str | None = None + + def __post_init__(self) -> None: + _pattern(self.asset_id, _ASSET_ID, "local asset id") + if (self.file_path is None) == (self.image_sha256 is None): + raise ValueError("local asset binding must select exactly one local locator") + if self.image_sha256 is not None: + _digest(self.image_sha256, "local image sha256") + + +@dataclass(frozen=True, slots=True) +class PortableWorkerAssetVerification: + asset_id: str + state: AssetVerificationState + reason: str | None + + def __post_init__(self) -> None: + _pattern(self.asset_id, _ASSET_ID, "verified asset id") + if self.state not in ("matched", "missing", "mismatched"): + raise ValueError("verified asset state is invalid") + if (self.state == "matched") != (self.reason is None): + raise ValueError("verified asset reason disagrees with its state") + + +def verify_local_assets( + candidate: PortableWorkerRuntimeCandidate, + bindings: Mapping[str, PortableWorkerLocalAssetBinding], +) -> tuple[PortableWorkerAssetVerification, ...]: + """Hash locally bound assets without accepting locators from a job.""" + + checks: list[PortableWorkerAssetVerification] = [] + for requirement in candidate.reusable_assets: + binding = bindings.get(requirement.asset_id) + if binding is None or binding.asset_id != requirement.asset_id: + checks.append( + PortableWorkerAssetVerification(requirement.asset_id, "missing", "not-bound") + ) + continue + if requirement.kind == "container-image": + matched = binding.file_path is None and binding.image_sha256 == requirement.sha256 + else: + matched = _matches_file(requirement, binding.file_path) + checks.append( + PortableWorkerAssetVerification( + requirement.asset_id, + "matched" if matched else "mismatched", + None if matched else "identity-mismatch", + ) + ) + return tuple(checks) + + +@dataclass(frozen=True, slots=True) +class PortableWorkerRuntimeAdmission: + candidate_sha256: str + ready: bool + blockers: tuple[str, ...] + assets: tuple[PortableWorkerAssetVerification, ...] + + def __post_init__(self) -> None: + _digest(self.candidate_sha256, "runtime admission candidate sha256") + if self.blockers != tuple(sorted(self.blockers)) or len(self.blockers) != len( + set(self.blockers) + ): + raise ValueError("runtime admission blockers must be canonical") + if self.ready and ( + self.blockers or any(item.state != "matched" for item in self.assets) + ): + raise ValueError("ready runtime admission cannot contain an unresolved asset") + + +def inspect_runtime_candidate( + candidate: PortableWorkerRuntimeCandidate, + bindings: Mapping[str, PortableWorkerLocalAssetBinding], +) -> PortableWorkerRuntimeAdmission: + assets = verify_local_assets(candidate, bindings) + asset_blockers = tuple( + sorted(f"asset-{item.asset_id}-{item.state}" for item in assets if item.state != "matched") + ) + blockers = tuple(sorted((*candidate.blockers, *asset_blockers))) + return PortableWorkerRuntimeAdmission( + candidate_sha256=candidate.candidate_sha256, + ready=candidate.ready and not blockers, + blockers=blockers, + assets=assets, + ) + + +@dataclass(frozen=True, slots=True) +class PortableWorkerSourceStage: + """A locally materialized source; transport/materialization owns its path.""" + + root: Path + source_bundle_sha256: str + source_capability_manifest_sha256: str + source_adapter_sha256: str + + def __post_init__(self) -> None: + for value, label in ( + (self.source_bundle_sha256, "source bundle sha256"), + (self.source_capability_manifest_sha256, "source capability sha256"), + (self.source_adapter_sha256, "source adapter sha256"), + ): + _digest(value, label) + if self.root.is_symlink() or not self.root.is_dir(): + raise PortableWorkerRuntimeUnavailableError( + "portable source stage must be a real local directory" + ) + + +@dataclass(frozen=True, slots=True) +class PortableWorkerResultDraft: + root: Path + result_id: str + result_sha256: str + result_contract_sha256: str + + def __post_init__(self) -> None: + _pattern(self.result_id, _SESSION_ID, "result id") + _digest(self.result_sha256, "result sha256") + _digest(self.result_contract_sha256, "result contract sha256") + if self.root.is_symlink() or not self.root.is_dir(): + raise PortableWorkerRuntimeUnavailableError( + "portable result draft must be a real local directory" + ) + + +@dataclass(frozen=True, slots=True) +class PortableWorkerRuntimePlan: + job_id: str + adapter_id: str + candidate_sha256: str + setup_id: str + definition_sha256: str + source_bundle_sha256: str + source_capability_manifest_sha256: str + result_contract_sha256: str + phases: tuple[str, ...] + + def __post_init__(self) -> None: + _pattern(self.job_id, _JOB_ID, "runtime plan job id") + for value, label in ( + (self.adapter_id, "runtime plan adapter id"), + (self.setup_id, "runtime plan setup id"), + ): + _pattern(value, _IDENTIFIER, label) + for value, label in ( + (self.candidate_sha256, "runtime plan candidate sha256"), + (self.definition_sha256, "runtime plan definition sha256"), + (self.source_bundle_sha256, "runtime plan source bundle sha256"), + ( + self.source_capability_manifest_sha256, + "runtime plan source capability sha256", + ), + (self.result_contract_sha256, "runtime plan result contract sha256"), + ): + _digest(value, label) + if not self.phases or len(self.phases) != len(set(self.phases)): + raise PortableWorkerRuntimeRegistryError( + "runtime plan phases must be non-empty and unique" + ) + for phase in self.phases: + _pattern(phase, _IDENTIFIER, "runtime plan phase id") + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": PORTABLE_WORKER_RUNTIME_PLAN_SCHEMA, + "job_id": self.job_id, + "adapter_id": self.adapter_id, + "candidate_sha256": self.candidate_sha256, + "setup_id": self.setup_id, + "definition_sha256": self.definition_sha256, + "source_bundle_sha256": self.source_bundle_sha256, + "source_capability_manifest_sha256": self.source_capability_manifest_sha256, + "result_contract_sha256": self.result_contract_sha256, + "phases": list(self.phases), + "authority": dict(_AUTHORITY), + } + + +class PortableWorkerSourceMaterializer(Protocol): + def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage: ... + + +class PortableWorkerProfileRunner(Protocol): + def run( + self, + plan: PortableWorkerRuntimePlan, + source: PortableWorkerSourceStage, + ) -> PortableWorkerResultDraft: ... + + +class PortableWorkerResultPublisher(Protocol): + def publish( + self, + job: SealedObservatoryRecordedJob, + draft: PortableWorkerResultDraft, + ) -> ObservatoryWorkerExecutionResult: ... + + +@dataclass(frozen=True, slots=True) +class PortableWorkerExecutorAdapter: + """Local adapter composition; none of its dependencies come from a job.""" + + candidate: PortableWorkerRuntimeCandidate + definition: PortableRunDefinition + admission: PortableWorkerRuntimeAdmission + source_materializer: PortableWorkerSourceMaterializer + runner: PortableWorkerProfileRunner + publisher: PortableWorkerResultPublisher + + def __post_init__(self) -> None: + self.candidate.bind_definition(self.definition) + if not self.candidate.ready or not self.admission.ready: + raise PortableWorkerRuntimeUnavailableError( + "portable Worker adapter cannot bind a blocked runtime candidate" + ) + if self.admission.candidate_sha256 != self.candidate.candidate_sha256: + raise PortableWorkerRuntimeUnavailableError( + "runtime admission belongs to another candidate" + ) + expected_assets = tuple( + requirement.asset_id for requirement in self.candidate.reusable_assets + ) + admitted_assets = tuple(item.asset_id for item in self.admission.assets) + if admitted_assets != expected_assets or any( + item.state != "matched" for item in self.admission.assets + ): + raise PortableWorkerRuntimeUnavailableError( + "runtime admission does not prove every candidate asset" + ) + + def execute( + self, + job: SealedObservatoryRecordedJob, + ) -> ObservatoryWorkerExecutionResult: + self._verify_job(job) + source = self.source_materializer.materialize(job) + if ( + source.source_bundle_sha256 != job.source_bundle_sha256 + or source.source_capability_manifest_sha256 + != job.source_capability_manifest_sha256 + or source.source_adapter_sha256 != job.source_adapter_sha256 + ): + raise PortableWorkerRuntimeJobRejectedError( + "materialized source differs from the sealed job" + ) + plan = PortableWorkerRuntimePlan( + job_id=job.job_id, + adapter_id=self.candidate.adapter_id, + candidate_sha256=self.candidate.candidate_sha256, + setup_id=job.setup_id, + definition_sha256=job.definition_sha256, + source_bundle_sha256=job.source_bundle_sha256, + source_capability_manifest_sha256=job.source_capability_manifest_sha256, + result_contract_sha256=self.candidate.result_contract_sha256, + phases=tuple(phase.phase_id for phase in self.candidate.phases), + ) + draft = self.runner.run(plan, source) + if draft.result_contract_sha256 != self.candidate.result_contract_sha256: + raise PortableWorkerRuntimeJobRejectedError( + "runtime result uses another result contract" + ) + published = self.publisher.publish(job, draft) + if ( + published.result_id != draft.result_id + or published.result_sha256 != draft.result_sha256 + ): + raise PortableWorkerRuntimeJobRejectedError( + "publisher receipt differs from the validated result draft" + ) + return published + + def _verify_job(self, job: SealedObservatoryRecordedJob) -> None: + executor = self.candidate.executor + expected_identity = self.candidate.executor_identity() + if ( + executor is None + or job.setup_id != self.definition.setup_id + or job.definition_id != self.definition.definition_id + or job.definition_version != self.definition.version + or job.definition_sha256 != self.definition.definition_sha256 + or job.source_adapter_id != self.definition.source_adapter.adapter_id + or job.source_adapter_version != self.definition.source_adapter.version + or job.source_adapter_sha256 != self.definition.source_adapter.contract_sha256 + or job.executor_release_id != executor.release_id + or job.executor_identity != expected_identity + or job.model_release_ids != self.definition.learned_models + or job.resource_profile_id != self.definition.resource_profile.profile_id + or job.checkpoint_policy != self.definition.resource_profile.checkpoint_policy + or job.allowed_checkpoints != self.definition.resource_profile.allowed_checkpoints + ): + raise PortableWorkerRuntimeJobRejectedError( + "Worker job differs from the exact local runtime identity" + ) + + +def _matches_file( + requirement: PortableWorkerAssetRequirement, + path: Path | None, +) -> bool: + if path is None: + return False + candidate = path.expanduser().absolute() + try: + if candidate.is_symlink() or not candidate.is_file(): + return False + if requirement.byte_length is not None and candidate.stat().st_size != ( + requirement.byte_length + ): + return False + digest = hashlib.sha256() + with candidate.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() == requirement.sha256 + except OSError: + return False + + +def _candidate(value: object) -> PortableWorkerRuntimeCandidate: + row = _object(value, "runtime candidate") + _exact_keys( + row, + { + "schema_version", + "adapter_id", + "setup_id", + "definition_id", + "definition_version", + "definition_sha256", + "source_adapter_sha256", + "model_manifest_sha256", + "resource_profile_sha256", + "result_contract_sha256", + "state", + "executor", + "reusable_assets", + "phases", + "blockers", + "authority", + "candidate_sha256", + }, + "runtime candidate", + ) + if row["schema_version"] != PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA: + raise PortableWorkerRuntimeRegistryError("runtime candidate schema is invalid") + if row["authority"] != _AUTHORITY: + raise PortableWorkerRuntimeRegistryError( + "runtime candidate authority must remain observation-only" + ) + state = row["state"] + if state not in ("blocked", "ready"): + raise PortableWorkerRuntimeRegistryError("runtime candidate state is invalid") + executor_row = row["executor"] + executor = None if executor_row is None else _executor(executor_row) + assets = _array(row["reusable_assets"], "runtime reusable assets") + phases = _array(row["phases"], "runtime phases") + blockers = _array(row["blockers"], "runtime blockers") + return PortableWorkerRuntimeCandidate( + adapter_id=_string(row["adapter_id"], "runtime adapter id"), + setup_id=_string(row["setup_id"], "setup id"), + definition_id=_string(row["definition_id"], "definition id"), + definition_version=_integer(row["definition_version"], "definition version"), + definition_sha256=_string(row["definition_sha256"], "definition sha256"), + source_adapter_sha256=_string( + row["source_adapter_sha256"], "source adapter sha256" + ), + model_manifest_sha256=_string( + row["model_manifest_sha256"], "model manifest sha256" + ), + resource_profile_sha256=_string( + row["resource_profile_sha256"], "resource profile sha256" + ), + result_contract_sha256=_string( + row["result_contract_sha256"], "result contract sha256" + ), + state=state, + executor=executor, + reusable_assets=tuple(_asset(item) for item in assets), + phases=tuple(_phase(item) for item in phases), + blockers=tuple(_string(item, "runtime blocker") for item in blockers), + candidate_sha256=_string(row["candidate_sha256"], "runtime candidate sha256"), + ) + + +def _asset(value: object) -> PortableWorkerAssetRequirement: + row = _object(value, "runtime asset") + _exact_keys( + row, + { + "asset_id", + "kind", + "sha256", + "byte_length", + "component_id", + "model_release_id", + "model_artifact_role", + }, + "runtime asset", + ) + kind = row["kind"] + if kind not in ( + "container-image", + "definition-component", + "local-file", + "model-artifact", + ): + raise PortableWorkerRuntimeRegistryError("runtime asset kind is invalid") + return PortableWorkerAssetRequirement( + asset_id=_string(row["asset_id"], "runtime asset id"), + kind=kind, + sha256=_string(row["sha256"], "runtime asset sha256"), + byte_length=_optional_integer(row["byte_length"], "runtime asset byte length"), + component_id=_optional_string(row["component_id"], "definition component id"), + model_release_id=_optional_string(row["model_release_id"], "model release id"), + model_artifact_role=_optional_string( + row["model_artifact_role"], "model artifact role" + ), + ) + + +def _phase(value: object) -> PortableWorkerRuntimePhase: + row = _object(value, "runtime phase") + _exact_keys(row, {"phase_id", "state"}, "runtime phase") + state = row["state"] + if state not in ("implemented", "missing"): + raise PortableWorkerRuntimeRegistryError("runtime phase state is invalid") + return PortableWorkerRuntimePhase( + phase_id=_string(row["phase_id"], "runtime phase id"), + state=state, + ) + + +def _executor(value: object) -> PortableWorkerExecutorSeal: + row = _object(value, "runtime executor") + _exact_keys( + row, + {"release_id", "release_sha256", "image_sha256"}, + "runtime executor", + ) + return PortableWorkerExecutorSeal( + release_id=_string(row["release_id"], "executor release id"), + release_sha256=_string(row["release_sha256"], "executor release sha256"), + image_sha256=_string(row["image_sha256"], "executor image sha256"), + ) + + +def _reject_unsafe_keys(value: object, *, parent: str = "registry") -> None: + """Keep executable instructions and local locators out of shared config.""" + + if isinstance(value, dict): + for key, child in value.items(): + if not isinstance(key, str): + raise PortableWorkerRuntimeRegistryError( + "runtime registry object keys must be strings" + ) + normalized = key.lower().replace("-", "_") + if ( + normalized == "path" + or normalized.endswith("_path") + or normalized in {"command", "commands", "argv", "env", "environment"} + or normalized.startswith("command_") + or normalized.endswith("_command") + or "priority" in normalized + ): + raise PortableWorkerRuntimeRegistryError( + f"runtime registry forbids {key!r} in {parent}" + ) + _reject_unsafe_keys(child, parent=key) + elif isinstance(value, list): + for child in value: + _reject_unsafe_keys(child, parent=parent) + + +def _object(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise PortableWorkerRuntimeRegistryError(f"{label} must be an object") + return value + + +def _array(value: object, label: str) -> list[object]: + if not isinstance(value, list): + raise PortableWorkerRuntimeRegistryError(f"{label} must be an array") + return value + + +def _exact_keys(row: Mapping[str, object], expected: set[str], label: str) -> None: + if set(row) != expected: + raise PortableWorkerRuntimeRegistryError(f"{label} fields are invalid") + + +def _string(value: object, label: str) -> str: + if not isinstance(value, str): + raise PortableWorkerRuntimeRegistryError(f"{label} must be a string") + return value + + +def _optional_string(value: object, label: str) -> str | None: + if value is None: + return None + return _string(value, label) + + +def _integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise PortableWorkerRuntimeRegistryError(f"{label} must be an integer") + return value + + +def _optional_integer(value: object, label: str) -> int | None: + if value is None: + return None + return _integer(value, label) + + +def _pattern(value: str, pattern: re.Pattern[str], label: str) -> None: + if pattern.fullmatch(value) is None: + raise PortableWorkerRuntimeRegistryError(f"{label} is invalid") + + +def _optional_identifier(value: str | None, label: str) -> None: + if value is not None: + _pattern(value, _IDENTIFIER, label) + + +def _digest(value: object, label: str) -> None: + if not isinstance(value, str) or _SHA256.fullmatch(value) is None: + raise PortableWorkerRuntimeRegistryError(f"{label} is invalid") diff --git a/src/k1link/observatory/recorded_jobs.py b/src/k1link/observatory/recorded_jobs.py index ea66c1e..8d24fa1 100644 --- a/src/k1link/observatory/recorded_jobs.py +++ b/src/k1link/observatory/recorded_jobs.py @@ -45,6 +45,9 @@ MAX_LIVE_LEASES: Final = 10_000 MAX_RECORDED_JOB_STORAGE_BYTES: Final = 128 * 1024 * 1024 RECORDED_JOB_SQLITE_LOCK_TIMEOUT_SECONDS: Final = 0.1 _SQLITE_BUSY_TIMEOUT_MILLISECONDS: Final = 100 +DEFAULT_RECORDED_CLAIM_LEASE_SECONDS: Final = 120 +MIN_RECORDED_CLAIM_LEASE_SECONDS: Final = 5 +MAX_RECORDED_CLAIM_LEASE_SECONDS: Final = 3_600 LIVE_K1_PRIORITY_RANK: Final = 0 RECORDED_PRIORITY_RANK: Final = 100 @@ -120,6 +123,11 @@ CREATE TABLE IF NOT EXISTS observatory_recorded_jobs ( claim_generation INTEGER NOT NULL CHECK (claim_generation >= 0), active_claim_token TEXT, active_claimant_id TEXT, + claimed_at_utc TEXT, + claim_expires_at_utc TEXT, + claim_heartbeat_at_utc TEXT, + claim_renewal_count INTEGER NOT NULL DEFAULT 0 + CHECK (claim_renewal_count >= 0), last_checkpoint_id TEXT, restart_from_zero INTEGER NOT NULL CHECK (restart_from_zero IN (0, 1)), preemption_receipt_sha256 TEXT, @@ -386,6 +394,10 @@ class ObservatoryRecordedJob: claim_generation: int active_claim_token: str | None active_claimant_id: str | None + claimed_at_utc: str | None + claim_expires_at_utc: str | None + claim_heartbeat_at_utc: str | None + claim_renewal_count: int last_checkpoint_id: str | None restart_from_zero: bool preemption_receipt_sha256: str | None @@ -461,6 +473,46 @@ class ObservatoryRecordedJob: raise ObservatoryRecordedQueueIntegrityError("recorded-job claim generation is invalid") _validate_optional_pattern(self.active_claim_token, _TOKEN, "active claim token") _validate_optional_pattern(self.active_claimant_id, _IDENTIFIER, "claimant id") + lease_values = ( + self.claimed_at_utc, + self.claim_expires_at_utc, + self.claim_heartbeat_at_utc, + ) + if (self.active_claim_token is None) != (self.active_claimant_id is None): + raise ObservatoryRecordedQueueIntegrityError( + "recorded-job claim ownership is partial" + ) + if self.active_claim_token is None: + if any(value is not None for value in lease_values) or self.claim_renewal_count != 0: + raise ObservatoryRecordedQueueIntegrityError( + "inactive recorded-job claim retains lease state" + ) + else: + if self.state not in {"claimed", "running", "paused", "preemption-pending"}: + raise ObservatoryRecordedQueueIntegrityError( + "recorded-job claim is active outside an owned state" + ) + if self.claim_generation < 1 or any(value is None for value in lease_values): + raise ObservatoryRecordedQueueIntegrityError( + "active recorded-job claim has no complete lease" + ) + if self.claim_renewal_count < 0: + raise ObservatoryRecordedQueueIntegrityError( + "recorded-job claim renewal count is invalid" + ) + assert self.claimed_at_utc is not None + assert self.claim_expires_at_utc is not None + assert self.claim_heartbeat_at_utc is not None + claimed_at = _parse_timestamp(self.claimed_at_utc, "claim timestamp") + expires_at = _parse_timestamp(self.claim_expires_at_utc, "claim expiry") + heartbeat_at = _parse_timestamp( + self.claim_heartbeat_at_utc, + "claim heartbeat timestamp", + ) + if not claimed_at <= heartbeat_at < expires_at: + raise ObservatoryRecordedQueueIntegrityError( + "recorded-job claim lease chronology is invalid" + ) _validate_optional_pattern(self.last_checkpoint_id, _CHECKPOINT_ID, "last checkpoint id") if not isinstance(self.restart_from_zero, bool): raise ObservatoryRecordedQueueIntegrityError("recorded-job restart marker is invalid") @@ -532,6 +584,16 @@ class ObservatoryRecordedJob: "restart_from_zero": self.restart_from_zero, "preemption_receipt_sha256": self.preemption_receipt_sha256, "claim_generation": self.claim_generation, + "claim_lease": ( + None + if self.active_claim_token is None + else { + "claimed_at_utc": self.claimed_at_utc, + "expires_at_utc": self.claim_expires_at_utc, + "heartbeat_at_utc": self.claim_heartbeat_at_utc, + "renewal_count": self.claim_renewal_count, + } + ), "result": ( None if self.result_id is None @@ -794,10 +856,12 @@ class ObservatoryRecordedJobQueue: max_jobs: int = MAX_RECORDED_JOBS, max_claim_receipts: int = MAX_RECORDED_CLAIM_RECEIPTS, max_live_leases: int = MAX_LIVE_LEASES, + claim_lease_seconds: int = DEFAULT_RECORDED_CLAIM_LEASE_SECONDS, ) -> None: _validate_quota(max_jobs, MAX_RECORDED_JOBS, "recorded job") _validate_quota(max_claim_receipts, MAX_RECORDED_CLAIM_RECEIPTS, "claim receipt") _validate_quota(max_live_leases, MAX_LIVE_LEASES, "live lease") + _validate_claim_lease_seconds(claim_lease_seconds) self.data_dir = data_dir.expanduser().resolve() self.database_path = self.data_dir / RECORDED_JOB_DATABASE_NAME self._definitions = definitions @@ -806,6 +870,7 @@ class ObservatoryRecordedJobQueue: self._max_jobs = max_jobs self._max_claim_receipts = max_claim_receipts self._max_live_leases = max_live_leases + self._claim_lease_seconds = claim_lease_seconds self._lock = threading.RLock() self._initialize() @@ -946,6 +1011,8 @@ class ObservatoryRecordedJobQueue: _validate_pattern(claim_request_id, _IDEMPOTENCY_KEY, "claim request id") request_sha256 = _claim_request_sha256(claimant_id, claim_request_id) with self._transaction() as connection: + now = self._timestamp() + self._recover_stale_claims(connection, now=now) receipt = connection.execute( "SELECT * FROM observatory_recorded_claim_receipts WHERE claim_request_id = ?", (claim_request_id,), @@ -976,7 +1043,6 @@ class ObservatoryRecordedJobQueue: "WHERE state = 'queued' " "ORDER BY priority_rank, created_at_utc, job_id LIMIT 1" ).fetchone() - now = self._timestamp() if row is None: connection.execute( "INSERT INTO observatory_recorded_claim_receipts " @@ -989,12 +1055,26 @@ class ObservatoryRecordedJobQueue: claim_token = hashlib.sha256( f"{uuid4().hex}:{job_id}:{claim_request_id}".encode() ).hexdigest() + claim_expires_at = _timestamp_after_seconds( + now, + self._claim_lease_seconds, + ) updated = connection.execute( "UPDATE observatory_recorded_jobs SET state = 'claimed', " "claim_generation = claim_generation + 1, active_claim_token = ?, " - "active_claimant_id = ?, preemption_requested = 0, " + "active_claimant_id = ?, claimed_at_utc = ?, " + "claim_expires_at_utc = ?, claim_heartbeat_at_utc = ?, " + "claim_renewal_count = 0, preemption_requested = 0, " "updated_at_utc = ? WHERE job_id = ? AND state = 'queued'", - (claim_token, claimant_id, now, job_id), + ( + claim_token, + claimant_id, + now, + claim_expires_at, + now, + now, + job_id, + ), ) if updated.rowcount != 1: raise ObservatoryRecordedQueueIntegrityError( @@ -1021,12 +1101,125 @@ class ObservatoryRecordedJobQueue: job=self._get_job(connection, job_id), ) + def renew_claim( + self, + job_id: str, + *, + claim_token: str, + claim_generation: int, + heartbeat_sequence: int, + ) -> ObservatoryRecordedJob: + """Renew one exact active claim using an idempotent heartbeat sequence.""" + + _validate_pattern(job_id, _JOB_ID, "recorded job id") + _validate_pattern(claim_token, _TOKEN, "claim token") + _validate_positive_int(claim_generation, "claim generation") + _validate_positive_int(heartbeat_sequence, "heartbeat sequence") + self.recover_stale_claims() + with self._transaction() as connection: + job = self._get_job(connection, job_id) + now = self._timestamp() + self._require_active_claim(job, claim_token, now=now) + if job.claim_generation != claim_generation: + raise ObservatoryRecordedQueueStaleClaimError( + "recorded-job claim generation is stale" + ) + if job.state not in {"claimed", "running", "paused", "preemption-pending"}: + raise ObservatoryRecordedQueueConflictError( + f"cannot renew recorded-job claim from {job.state}" + ) + if heartbeat_sequence == job.claim_renewal_count: + return job + if heartbeat_sequence != job.claim_renewal_count + 1: + raise ObservatoryRecordedQueueConflictError( + "recorded-job heartbeat sequence is not contiguous" + ) + expires_at = _timestamp_after_seconds(now, self._claim_lease_seconds) + connection.execute( + "UPDATE observatory_recorded_jobs SET claim_expires_at_utc = ?, " + "claim_heartbeat_at_utc = ?, claim_renewal_count = ?, " + "updated_at_utc = ? WHERE job_id = ? AND active_claim_token = ? " + "AND claim_generation = ?", + ( + expires_at, + now, + heartbeat_sequence, + now, + job_id, + claim_token, + claim_generation, + ), + ) + return self._get_job(connection, job_id) + + def authorize_claim_access( + self, + job_id: str, + *, + claim_token: str, + claim_generation: int, + claimant_id: str, + allowed_states: tuple[RecordedJobState, ...] = ("claimed", "running"), + ) -> ObservatoryRecordedJob: + """Authorize one bounded side-channel operation for the active lease. + + Source downloads and result staging are deliberately not queue state + transitions, but they still must be fenced by the exact claimant, + token, generation, unexpired lease and an explicitly admitted queue + state. Keeping this check inside the queue transaction prevents an + artifact transport from reimplementing only part of claim semantics. + """ + + _validate_pattern(job_id, _JOB_ID, "recorded job id") + _validate_pattern(claim_token, _TOKEN, "claim token") + _validate_positive_int(claim_generation, "claim generation") + _validate_pattern(claimant_id, _IDENTIFIER, "claimant id") + if ( + not allowed_states + or len(set(allowed_states)) != len(allowed_states) + or any(state not in _recorded_job_states() for state in allowed_states) + ): + raise ValueError("claim-access states are invalid") + self.recover_stale_claims() + with self._transaction() as connection: + job = self._get_job(connection, job_id) + self._require_active_claim(job, claim_token, now=self._timestamp()) + if ( + job.claim_generation != claim_generation + or job.active_claimant_id != claimant_id + ): + raise ObservatoryRecordedQueueStaleClaimError( + "recorded-job claim ownership is stale" + ) + if job.state not in allowed_states: + raise ObservatoryRecordedQueueConflictError( + f"claim-bound access is unavailable from {job.state}" + ) + return job + + def recover_stale_claims(self) -> tuple[ObservatoryRecordedJob, ...]: + """Recover expired ownership without creating a second physical owner. + + A never-started or durably paused claim is safe to requeue. Expired + running ownership is quarantined for scheduler reconciliation because + lease expiry alone does not prove that its Worker process stopped. + """ + + with self._transaction() as connection: + recovered_ids = self._recover_stale_claims( + connection, + now=self._timestamp(), + ) + return tuple(self._get_job(connection, job_id) for job_id in recovered_ids) + def start(self, job_id: str, *, claim_token: str) -> ObservatoryRecordedJob: """Enter running state, or yield before execution when live has priority.""" + self.recover_stale_claims() with self._transaction() as connection: job = self._get_job(connection, job_id) - self._require_active_claim(job, claim_token) + now = self._timestamp() + self._require_active_claim(job, claim_token, now=now) if job.state == "running": return job if job.state == "paused": @@ -1039,7 +1232,7 @@ class ObservatoryRecordedJobQueue: connection.execute( "UPDATE observatory_recorded_jobs SET state = ?, " "preemption_requested = ?, updated_at_utc = ? WHERE job_id = ?", - (state, int(state == "paused"), self._timestamp(), job_id), + (state, int(state == "paused"), now, job_id), ) return self._get_job(connection, job_id) @@ -1053,9 +1246,11 @@ class ObservatoryRecordedJobQueue: """Record an allowlisted cooperative boundary and yield if live is open.""" _validate_pattern(checkpoint_id, _CHECKPOINT_ID, "checkpoint id") + self.recover_stale_claims() with self._transaction() as connection: job = self._get_job(connection, job_id) - self._require_active_claim(job, claim_token) + now = self._timestamp() + self._require_active_claim(job, claim_token, now=now) if job.checkpoint_policy != "cooperative": raise ObservatoryRecordedCheckpointError( "recorded RunDefinition is non-checkpointable" @@ -1079,7 +1274,7 @@ class ObservatoryRecordedJobQueue: connection.execute( "UPDATE observatory_recorded_jobs SET state = ?, " "last_checkpoint_id = ?, updated_at_utc = ? WHERE job_id = ?", - (state, checkpoint_id, self._timestamp(), job_id), + (state, checkpoint_id, now, job_id), ) return self._get_job(connection, job_id) @@ -1195,6 +1390,7 @@ class ObservatoryRecordedJobQueue: def request_live(self, intent: ObservatoryLiveLeaseIntent) -> tuple[ObservatoryLiveLease, bool]: """Close recorded admission without allowing a monolith to delay live K1.""" + self.recover_stale_claims() created = False with self._transaction() as connection: existing = connection.execute( @@ -1341,6 +1537,7 @@ class ObservatoryRecordedJobQueue: """Activate only after every recorded resource owner has yielded.""" _validate_pattern(lease_id, _LEASE_ID, "live lease id") + self.recover_stale_claims() with self._transaction() as connection: lease = self._get_live_lease(connection, lease_id) if lease.state == "active": @@ -1438,7 +1635,9 @@ class ObservatoryRecordedJobQueue: connection.execute( "UPDATE observatory_recorded_jobs SET state = 'queued', " "preemption_requested = 0, active_claim_token = NULL, " - "active_claimant_id = NULL, updated_at_utc = ? " + "active_claimant_id = NULL, claimed_at_utc = NULL, " + "claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, " + "claim_renewal_count = 0, updated_at_utc = ? " "WHERE state = 'paused'", (now,), ) @@ -1488,22 +1687,34 @@ class ObservatoryRecordedJobQueue: _validate_pattern(job_id, _JOB_ID, "recorded job id") _validate_pattern(claim_token, _TOKEN, "claim token") token_sha256 = hashlib.sha256(claim_token.encode()).hexdigest() + self.recover_stale_claims() with self._transaction() as connection: job = self._get_job(connection, job_id) if job.state in _TERMINAL_STATES: + exact_replay = ( + job.state == state + and job.result_id == result_id + and job.result_sha256 == result_sha256 + and job.terminal_code == terminal_code + and job.terminal_message == terminal_message + and job.terminal_claim_token_sha256 == token_sha256 + ) + if exact_replay: + return job if ( - job.state != state - or job.result_id != result_id - or job.result_sha256 != result_sha256 - or job.terminal_code != terminal_code - or job.terminal_message != terminal_message - or job.terminal_claim_token_sha256 != token_sha256 + job.terminal_claim_token_sha256 != token_sha256 + or job.terminal_code + in {"claim-lease-expired", "claim-lease-migration"} ): + raise ObservatoryRecordedQueueStaleClaimError( + "recorded-job terminal acknowledgement is stale" + ) + else: raise ObservatoryRecordedQueueConflictError( "recorded job is bound to another terminal outcome" ) - return job - self._require_active_claim(job, claim_token) + now = self._timestamp() + self._require_active_claim(job, claim_token, now=now) allowed_states = ( ("running",) if state == "succeeded" @@ -1523,7 +1734,9 @@ class ObservatoryRecordedJobQueue: "UPDATE observatory_recorded_jobs SET state = ?, result_id = ?, " "result_sha256 = ?, terminal_code = ?, terminal_message = ?, " "terminal_claim_token_sha256 = ?, active_claim_token = NULL, " - "active_claimant_id = NULL, updated_at_utc = ? WHERE job_id = ?", + "active_claimant_id = NULL, claimed_at_utc = NULL, " + "claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, " + "claim_renewal_count = 0, updated_at_utc = ? WHERE job_id = ?", ( state, result_id, @@ -1531,7 +1744,7 @@ class ObservatoryRecordedJobQueue: terminal_code, terminal_message, token_sha256, - self._timestamp(), + now, job_id, ), ) @@ -1548,18 +1761,105 @@ class ObservatoryRecordedJobQueue: raise ObservatoryRecordedQueueIntegrityError( "stored claim receipt has a partial job identity" ) + job = self._get_job(connection, job_id) + if job.active_claim_token != claim_token: + raise ObservatoryRecordedQueueStaleClaimError( + "recorded-job claim receipt no longer owns the job" + ) return ObservatoryRecordedClaim( claim_request_id=str(receipt["claim_request_id"]), request_sha256=str(receipt["request_sha256"]), claimant_id=str(receipt["claimant_id"]), claim_token=claim_token, - job=self._get_job(connection, job_id), + job=job, ) - def _require_active_claim(self, job: ObservatoryRecordedJob, claim_token: str) -> None: + def _require_active_claim( + self, + job: ObservatoryRecordedJob, + claim_token: str, + *, + now: str | None = None, + ) -> None: _validate_pattern(claim_token, _TOKEN, "claim token") if job.active_claim_token != claim_token: raise ObservatoryRecordedQueueStaleClaimError("recorded-job claim token is stale") + if now is not None and ( + job.claim_expires_at_utc is None + or _parse_timestamp(now, "queue timestamp") + >= _parse_timestamp(job.claim_expires_at_utc, "claim expiry") + ): + raise ObservatoryRecordedQueueStaleClaimError( + "recorded-job claim lease expired" + ) + + def _recover_stale_claims( + self, + connection: sqlite3.Connection, + *, + now: str, + ) -> tuple[str, ...]: + now_value = _parse_timestamp(now, "queue timestamp") + rows = connection.execute( + "SELECT * FROM observatory_recorded_jobs " + "WHERE active_claim_token IS NOT NULL " + "ORDER BY created_at_utc, job_id" + ).fetchall() + recovered: list[str] = [] + live_open = self._open_live_lease_row(connection) is not None + for row in rows: + job = _job_from_row(row) + if job.claim_expires_at_utc is None: + raise ObservatoryRecordedQueueIntegrityError( + "active recorded-job claim has no expiry" + ) + if _parse_timestamp(job.claim_expires_at_utc, "claim expiry") > now_value: + continue + assert job.active_claim_token is not None + token_sha256 = hashlib.sha256(job.active_claim_token.encode()).hexdigest() + if job.state in {"claimed", "paused"}: + next_state = "paused" if job.state == "paused" and live_open else "queued" + restart_from_zero = job.restart_from_zero or job.state == "paused" + connection.execute( + "UPDATE observatory_recorded_jobs SET state = ?, " + "preemption_requested = ?, active_claim_token = NULL, " + "active_claimant_id = NULL, claimed_at_utc = NULL, " + "claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, " + "claim_renewal_count = 0, last_checkpoint_id = ?, " + "restart_from_zero = ?, updated_at_utc = ? WHERE job_id = ?", + ( + next_state, + int(next_state == "paused"), + None if restart_from_zero else job.last_checkpoint_id, + int(restart_from_zero), + now, + job.job_id, + ), + ) + elif job.state in {"running", "preemption-pending"}: + connection.execute( + "UPDATE observatory_recorded_jobs " + "SET state = 'reconciliation-required', result_id = NULL, " + "result_sha256 = NULL, terminal_code = 'claim-lease-expired', " + "terminal_message = ?, terminal_claim_token_sha256 = ?, " + "active_claim_token = NULL, active_claimant_id = NULL, " + "claimed_at_utc = NULL, claim_expires_at_utc = NULL, " + "claim_heartbeat_at_utc = NULL, claim_renewal_count = 0, " + "updated_at_utc = ? WHERE job_id = ?", + ( + "Worker claim lease expired after execution started; " + "physical resource ownership requires reconciliation.", + token_sha256, + now, + job.job_id, + ), + ) + else: + raise ObservatoryRecordedQueueIntegrityError( + "active recorded-job claim is stored in an invalid state" + ) + recovered.append(job.job_id) + return tuple(recovered) def _cancellation_request( self, @@ -1626,7 +1926,9 @@ class ObservatoryRecordedJobQueue: connection.execute( "UPDATE observatory_recorded_jobs SET state = 'paused', " "preemption_requested = 1, active_claim_token = NULL, " - "active_claimant_id = NULL, last_checkpoint_id = NULL, " + "active_claimant_id = NULL, claimed_at_utc = NULL, " + "claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, " + "claim_renewal_count = 0, last_checkpoint_id = NULL, " "restart_from_zero = 1, preemption_receipt_sha256 = ?, " "updated_at_utc = ? WHERE job_id = ?", (receipt.receipt_sha256, now, job.job_id), @@ -1638,6 +1940,8 @@ class ObservatoryRecordedJobQueue: "result_sha256 = NULL, terminal_code = 'preemption-race', " "terminal_message = ?, terminal_claim_token_sha256 = ?, " "active_claim_token = NULL, active_claimant_id = NULL, " + "claimed_at_utc = NULL, claim_expires_at_utc = NULL, " + "claim_heartbeat_at_utc = NULL, claim_renewal_count = 0, " "preemption_receipt_sha256 = ?, updated_at_utc = ? " "WHERE job_id = ?", ( @@ -1707,6 +2011,7 @@ class ObservatoryRecordedJobQueue: self.data_dir.chmod(0o700) with self._connect() as connection: connection.executescript(_SCHEMA_SQL) + self._migrate_claim_lease_schema(connection) self._validate_schema(connection) self._validate_existing_capacity(connection) connection.commit() @@ -1723,7 +2028,7 @@ class ObservatoryRecordedJobQueue: def _validate_schema(self, connection: sqlite3.Connection) -> None: expected = { - "observatory_recorded_jobs": 42, + "observatory_recorded_jobs": 46, "observatory_recorded_claim_receipts": 6, "observatory_live_leases": 13, "observatory_recorded_preemptions": 14, @@ -1734,6 +2039,21 @@ class ObservatoryRecordedJobQueue: ).fetchall() if len(columns) != column_count: raise ObservatoryRecordedQueueIntegrityError(f"{table} schema is incompatible") + job_columns = { + str(row["name"]) + for row in connection.execute( + "SELECT name FROM pragma_table_info('observatory_recorded_jobs')" + ).fetchall() + } + if not { + "claimed_at_utc", + "claim_expires_at_utc", + "claim_heartbeat_at_utc", + "claim_renewal_count", + }.issubset(job_columns): + raise ObservatoryRecordedQueueIntegrityError( + "recorded-job claim lease schema is unavailable" + ) indexes = connection.execute( "SELECT name FROM sqlite_master WHERE type = 'index' " "AND name = 'observatory_one_open_live_lease'" @@ -1743,6 +2063,75 @@ class ObservatoryRecordedJobQueue: "live K1 lease exclusivity index is unavailable" ) + def _migrate_claim_lease_schema(self, connection: sqlite3.Connection) -> None: + """Add renewable claim columns and safely fence legacy active owners.""" + + columns = { + str(row["name"]) + for row in connection.execute( + "SELECT name FROM pragma_table_info('observatory_recorded_jobs')" + ).fetchall() + } + additions = ( + ("claimed_at_utc", "TEXT"), + ("claim_expires_at_utc", "TEXT"), + ("claim_heartbeat_at_utc", "TEXT"), + ( + "claim_renewal_count", + "INTEGER NOT NULL DEFAULT 0 CHECK (claim_renewal_count >= 0)", + ), + ) + missing = [name for name, _definition in additions if name not in columns] + if not missing: + return + for name, definition in additions: + if name not in columns: + connection.execute( + f"ALTER TABLE observatory_recorded_jobs ADD COLUMN {name} {definition}" + ) + + now = self._timestamp() + rows = connection.execute( + "SELECT job_id, state, active_claim_token " + "FROM observatory_recorded_jobs WHERE active_claim_token IS NOT NULL" + ).fetchall() + for row in rows: + job_id = str(row["job_id"]) + state = str(row["state"]) + claim_token = str(row["active_claim_token"]) + if state in {"claimed", "paused"}: + connection.execute( + "UPDATE observatory_recorded_jobs SET state = 'queued', " + "preemption_requested = 0, active_claim_token = NULL, " + "active_claimant_id = NULL, claimed_at_utc = NULL, " + "claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, " + "claim_renewal_count = 0, last_checkpoint_id = NULL, " + "restart_from_zero = ?, updated_at_utc = ? WHERE job_id = ?", + (int(state == "paused"), now, job_id), + ) + elif state in {"running", "preemption-pending"}: + connection.execute( + "UPDATE observatory_recorded_jobs " + "SET state = 'reconciliation-required', result_id = NULL, " + "result_sha256 = NULL, terminal_code = 'claim-lease-migration', " + "terminal_message = ?, terminal_claim_token_sha256 = ?, " + "active_claim_token = NULL, active_claimant_id = NULL, " + "claimed_at_utc = NULL, claim_expires_at_utc = NULL, " + "claim_heartbeat_at_utc = NULL, claim_renewal_count = 0, " + "updated_at_utc = ? WHERE job_id = ?", + ( + "Legacy Worker execution had no expiring lease; physical " + "resource ownership requires reconciliation.", + hashlib.sha256(claim_token.encode()).hexdigest(), + now, + job_id, + ), + ) + else: + raise ObservatoryRecordedQueueIntegrityError( + "legacy active claim is stored in an invalid state" + ) + def _validate_existing_capacity(self, connection: sqlite3.Connection) -> None: for table, limit, label in ( ("observatory_recorded_jobs", self._max_jobs, "recorded job"), @@ -1901,6 +2290,10 @@ def _job_from_row(row: sqlite3.Row) -> ObservatoryRecordedJob: claim_generation=row["claim_generation"], active_claim_token=row["active_claim_token"], active_claimant_id=row["active_claimant_id"], + claimed_at_utc=row["claimed_at_utc"], + claim_expires_at_utc=row["claim_expires_at_utc"], + claim_heartbeat_at_utc=row["claim_heartbeat_at_utc"], + claim_renewal_count=row["claim_renewal_count"], last_checkpoint_id=row["last_checkpoint_id"], restart_from_zero=bool(row["restart_from_zero"]), preemption_receipt_sha256=row["preemption_receipt_sha256"], @@ -2029,6 +2422,17 @@ def _validate_quota(value: object, maximum: int, label: str) -> None: raise ValueError(f"{label} quota is invalid") +def _validate_claim_lease_seconds(value: object) -> None: + if ( + not isinstance(value, int) + or isinstance(value, bool) + or not MIN_RECORDED_CLAIM_LEASE_SECONDS + <= value + <= MAX_RECORDED_CLAIM_LEASE_SECONDS + ): + raise ValueError("recorded-job claim lease duration is invalid") + + def _validate_positive_int(value: object, label: str) -> None: if not isinstance(value, int) or isinstance(value, bool) or value < 1: raise ValueError(f"{label} must be positive") @@ -2059,6 +2463,10 @@ def _validate_text(value: object, label: str, *, max_length: int) -> None: def _validate_timestamp(value: object, label: str) -> None: + _parse_timestamp(value, label) + + +def _parse_timestamp(value: object, label: str) -> datetime: _validate_text(value, label, max_length=64) assert isinstance(value, str) try: @@ -2067,6 +2475,12 @@ def _validate_timestamp(value: object, label: str) -> None: raise ValueError(f"{label} is invalid") from exc if parsed.tzinfo is None or parsed.utcoffset() != timedelta(0) or not value.endswith("Z"): raise ValueError(f"{label} must use UTC") + return parsed + + +def _timestamp_after_seconds(value: str, seconds: int) -> str: + expires_at = _parse_timestamp(value, "queue timestamp") + timedelta(seconds=seconds) + return expires_at.isoformat(timespec="milliseconds").replace("+00:00", "Z") def _fsync_directory(path: Path) -> None: diff --git a/src/k1link/observatory/setups.py b/src/k1link/observatory/setups.py index 3e93f85..0d2f381 100644 --- a/src/k1link/observatory/setups.py +++ b/src/k1link/observatory/setups.py @@ -12,8 +12,11 @@ import json import re from dataclasses import dataclass from pathlib import Path, PurePosixPath -from typing import Any, Final, Literal +from typing import Any, Final, Literal, cast +from k1link.observatory.canonical_result import ( + is_admitted_observatory_recorded_result, +) from k1link.sessions.models import SessionSummary LABORATORY_SETUP_REGISTRY_SCHEMA: Final = ( @@ -22,6 +25,9 @@ LABORATORY_SETUP_REGISTRY_SCHEMA: Final = ( LABORATORY_SETUP_CATALOG_SCHEMA: Final = ( "missioncore.observatory-laboratory-setup-catalog/v1" ) +OBSERVATORY_CALCULATION_PROFILE_SCHEMA: Final = ( + "missioncore.observatory-calculation-profile/v1" +) _MAX_REGISTRY_BYTES: Final = 256 * 1024 _MAX_CONFIGURATION_BYTES: Final = 4 * 1024 * 1024 _IDENTIFIER: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$") @@ -282,6 +288,42 @@ class LaboratorySetupRegistry: return result.result_kind return None + def observatory_calculation_profile( + self, + summary: SessionSummary, + ) -> dict[str, object] | None: + """Project an exact preserved legacy profile without inferring identity.""" + + for setup in self.setups: + for result in setup.preserved_results: + if ( + result.access != "observatory" + or result.result_id != summary.session_id + ): + continue + if ( + setup.origin != "existing-result" + or setup.run_definition is not None + ): + return None + if not is_admitted_observatory_recorded_result( + summary, + expected_result_id=result.result_id, + expected_source_session_id=setup.source_session_id, + expected_result_kind=result.result_kind, + ): + return None + return { + "schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA, + "setup_id": setup.setup_id, + "display_name": setup.display_name, + "origin": setup.origin, + "definition_id": None, + "definition_version": None, + "definition_sha256": None, + } + return None + def _setup(value: object, *, repository_root: Path) -> LaboratorySetup: row = _object(value, "setup") @@ -330,7 +372,7 @@ def _setup(value: object, *, repository_root: Path) -> LaboratorySetup: setup_id=_identifier(row["setup_id"], "setup_id"), display_name=_text(row["display_name"], "display_name"), description=_text(row["description"], "description"), - origin=origin, + origin=cast(SetupOrigin, origin), source_session_id=_text(source["session_id"], "source session_id"), source_label=_text(source["label"], "source label"), required_modalities=modalities, @@ -427,7 +469,7 @@ def _preserved_result(value: object) -> _PreservedResult: result_id=result_id, result_kind=_identifier(row["result_kind"], "result_kind"), relation=_identifier(row["relation"], "result relation"), - access=access, + access=cast(ResultAccess, access), created_at_utc=_text(row["created_at_utc"], "created_at_utc"), ) diff --git a/src/k1link/observatory/source_admission.py b/src/k1link/observatory/source_admission.py index 4599b37..166e643 100644 --- a/src/k1link/observatory/source_admission.py +++ b/src/k1link/observatory/source_admission.py @@ -15,7 +15,7 @@ import re import secrets import stat from contextlib import suppress -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from typing import Final @@ -29,6 +29,7 @@ from k1link.sessions.media import ( ) from k1link.sessions.models import ( RecordedMediaArtifact, + ReplayArtifact, ReplayCommand, SessionArtifact, SessionDetail, @@ -40,6 +41,8 @@ PORTABLE_SOURCE_BUNDLE_SCHEMA: Final = "missioncore.portable-recorded-source-bun PORTABLE_SOURCE_CAPABILITY_SCHEMA: Final = "missioncore.portable-recorded-source-capability/v1" PORTABLE_SOURCE_ADAPTER_SCHEMA: Final = "missioncore.portable-source-adapter/v1" PORTABLE_SOURCE_DOCUMENT_DIRECTORY: Final = "observatory-portable-source-contracts" +PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID: Final = "raw-transport-index" +PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE: Final = "application/x-ndjson" _SHA256 = re.compile(r"^[a-f0-9]{64}$") _IDENTIFIER = re.compile(r"^[a-z][a-z0-9.-]{2,127}$") @@ -384,10 +387,14 @@ class RecordedK1SourceAdmissionService: camera_source=camera_source, recorded_media=recorded_media, ) + sealed_replay = _seal_replay_artifact_digests( + detail=detail, + replay=replay, + ) self._verify_replay( detail=detail, selected_sources=selected_sources, - replay=replay, + replay=sealed_replay, ) try: media = ( @@ -410,7 +417,7 @@ class RecordedK1SourceAdmissionService: detail=detail, catalog_sha256=catalog_sha256, selected_sources=selected_sources, - replay=replay, + replay=sealed_replay, media=media, ) source_bundle = _canonical_json(source_bundle_document) @@ -592,13 +599,31 @@ class RecordedK1SourceAdmissionService: raise PortableSourceAdmissionIntegrityError("spatial replay members are not unique") for replay_artifact in replay.artifacts: catalog_artifact = catalog_artifacts.get(replay_artifact.artifact_id) + exact_metadata_member = ( + catalog_artifact is not None + and replay_artifact.artifact_id + == PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID + and catalog_artifact.kind + == PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID + and replay_artifact.media_type + == PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE + and catalog_artifact.media_type + == PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE + ) + digest_matches_catalog = ( + replay_artifact.expected_sha256 == catalog_artifact.sha256 + if catalog_artifact is not None and catalog_artifact.sha256 is not None + else exact_metadata_member + and isinstance(replay_artifact.expected_sha256, str) + and _SHA256.fullmatch(replay_artifact.expected_sha256) is not None + ) if ( catalog_artifact is None or catalog_artifact.integrity_status not in _SEALED_ARTIFACT_STATES or replay_artifact.media_type != catalog_artifact.media_type or replay_artifact.file_byte_length != catalog_artifact.byte_length or not 1 <= replay_artifact.replay_byte_length <= replay_artifact.file_byte_length - or replay_artifact.expected_sha256 != catalog_artifact.sha256 + or not digest_matches_catalog ): raise PortableSourceAdmissionIntegrityError( "spatial replay member disagrees with the catalog" @@ -799,6 +824,130 @@ class RecordedK1SourceAdmissionService: } +def _seal_replay_artifact_digests( + *, + detail: SessionDetail, + replay: ReplayCommand, +) -> ReplayCommand: + """Seal the one legacy replay member whose catalog has no stored digest. + + Historical K1 catalogs explicitly register ``mqtt.metadata.jsonl`` as the + ``raw-transport-index`` replay artifact, but the catalog row predates a + persisted SHA-256 column value. Portable admission may derive that one + digest from the already confined ReplayCommand handle. No sibling-name or + directory discovery is permitted, and every other missing digest remains + an integrity failure in ``_verify_replay``. + """ + + catalog_artifacts = {artifact.artifact_id: artifact for artifact in detail.artifacts} + sealed: list[ReplayArtifact] = [] + sealed_metadata_count = 0 + for artifact in replay.artifacts: + if artifact.expected_sha256 is not None: + sealed.append(artifact) + continue + catalog_artifact = catalog_artifacts.get(artifact.artifact_id) + if ( + catalog_artifact is None + or artifact.artifact_id + != PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID + or catalog_artifact.kind + != PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID + or artifact.media_type != PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE + or catalog_artifact.media_type + != PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE + or catalog_artifact.sha256 is not None + or catalog_artifact.integrity_status not in _SEALED_ARTIFACT_STATES + or artifact.file_byte_length != catalog_artifact.byte_length + or artifact.replay_byte_length != catalog_artifact.byte_length + ): + sealed.append(artifact) + continue + sealed_metadata_count += 1 + if sealed_metadata_count != 1: + raise PortableSourceAdmissionIntegrityError( + "spatial replay metadata member is not unique" + ) + sha256 = _hash_confined_replay_artifact(replay=replay, artifact=artifact) + sealed.append(replace(artifact, expected_sha256=sha256)) + return replace(replay, artifacts=tuple(sealed)) + + +def _hash_confined_replay_artifact( + *, + replay: ReplayCommand, + artifact: ReplayArtifact, +) -> str: + descriptor = -1 + try: + allowed_root = replay.allowed_root.resolve(strict=True) + session_root = replay.session_root.resolve(strict=True) + source_metadata = artifact.path.lstat() + source_path = artifact.path.resolve(strict=True) + if ( + not allowed_root.is_dir() + or not session_root.is_dir() + or not session_root.is_relative_to(allowed_root) + or not source_path.is_relative_to(session_root) + or stat.S_ISLNK(source_metadata.st_mode) + or not stat.S_ISREG(source_metadata.st_mode) + or not 1 <= artifact.file_byte_length <= MAX_SAFE_INTEGER + ): + raise PortableSourceAdmissionIntegrityError( + "spatial replay metadata escapes its admitted session root" + ) + descriptor = os.open( + source_path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size != artifact.file_byte_length + or before.st_size != artifact.replay_byte_length + ): + raise PortableSourceAdmissionIntegrityError( + "spatial replay metadata is outside admitted bounds" + ) + digest = hashlib.sha256() + byte_length = 0 + while chunk := os.read(descriptor, 1024 * 1024): + byte_length += len(chunk) + if byte_length > artifact.file_byte_length: + raise PortableSourceAdmissionIntegrityError( + "spatial replay metadata grew while it was sealed" + ) + digest.update(chunk) + after = os.fstat(descriptor) + stable_identity = ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) == ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ) + if byte_length != artifact.file_byte_length or not stable_identity: + raise PortableSourceAdmissionIntegrityError( + "spatial replay metadata changed while it was sealed" + ) + return digest.hexdigest() + except PortableSourceAdmissionIntegrityError: + raise + except OSError as exc: + raise PortableSourceAdmissionIntegrityError( + "spatial replay metadata is unavailable" + ) from exc + finally: + if descriptor >= 0: + os.close(descriptor) + + def _read_canonical_camera_summary( source_path: Path, ) -> tuple[dict[str, object], int]: diff --git a/src/k1link/observatory/worker_agent.py b/src/k1link/observatory/worker_agent.py index bc550e4..1f84782 100644 --- a/src/k1link/observatory/worker_agent.py +++ b/src/k1link/observatory/worker_agent.py @@ -19,6 +19,7 @@ import re import threading from collections.abc import Callable, Mapping from dataclasses import dataclass +from datetime import datetime from typing import Annotated, Final, Literal, Protocol from uuid import uuid4 @@ -51,6 +52,7 @@ type WorkerCycleState = Literal[ "succeeded", "failed", "rejected", + "lease-lost", ] type RecordedJobWireState = Literal[ "accepted", @@ -122,6 +124,7 @@ class SealedObservatoryRecordedJob: job_id: str request_sha256: str identity_sha256: str + submission_receipt_sha256: str source_session_id: str source_catalog_sha256: str source_bundle_sha256: str @@ -140,6 +143,10 @@ class SealedObservatoryRecordedJob: checkpoint_policy: Literal["cooperative", "non-checkpointable"] allowed_checkpoints: tuple[str, ...] claim_generation: int + claim_claimed_at_utc: str | None + claim_expires_at_utc: str | None + claim_heartbeat_at_utc: str | None + claim_renewal_count: int restart_from_zero: bool @@ -213,6 +220,16 @@ class ObservatoryWorkerTransport(Protocol): claim_token: str, ) -> Mapping[str, object]: ... + def renew_claim( + self, + *, + claimant_id: str, + job_id: str, + claim_token: str, + claim_generation: int, + heartbeat_sequence: int, + ) -> Mapping[str, object]: ... + def succeed( self, *, @@ -308,6 +325,13 @@ class _TerminalPayload(_StrictPayload): message: str = Field(min_length=1, max_length=1_000) +class _ClaimLeasePayload(_StrictPayload): + claimed_at_utc: Timestamp + expires_at_utc: Timestamp + heartbeat_at_utc: Timestamp + renewal_count: int = Field(ge=0) + + class _RecordedJobPayload(_StrictPayload): schema_version: Literal["missioncore.observatory-recorded-job/v1"] job_id: str = Field(pattern=_JOB_ID_PATTERN) @@ -329,6 +353,7 @@ class _RecordedJobPayload(_StrictPayload): restart_from_zero: bool preemption_receipt_sha256: Sha256 | None claim_generation: int = Field(ge=0) + claim_lease: _ClaimLeasePayload | None result: _ResultPayload | None terminal: _TerminalPayload | None created_at_utc: Timestamp @@ -365,10 +390,20 @@ class ObservatoryWorkerAgent: transport: ObservatoryWorkerTransport, executors: ObservatoryWorkerExecutorRegistry, claim_request_id_factory: Callable[[], str] | None = None, + heartbeat_interval_seconds: float | None = None, + heartbeat_stop_timeout_seconds: float = 5.0, ) -> None: + if heartbeat_interval_seconds is not None and not ( + 0.01 <= heartbeat_interval_seconds <= 300.0 + ): + raise ValueError("Worker heartbeat interval is invalid") + if not 0.1 <= heartbeat_stop_timeout_seconds <= 300.0: + raise ValueError("Worker heartbeat stop timeout is invalid") self._transport = transport self._executors = executors self._claim_request_id_factory = claim_request_id_factory or _default_claim_request_id + self._heartbeat_interval_seconds = heartbeat_interval_seconds + self._heartbeat_stop_timeout_seconds = heartbeat_stop_timeout_seconds self._cycle_lock = threading.Lock() def run_once(self) -> ObservatoryWorkerCycleReport: @@ -443,11 +478,33 @@ class ObservatoryWorkerAgent: job_id=claim.job.job_id, ) + active_job = _seal_job(started) + heartbeat = _ClaimHeartbeat( + transport=self._transport, + job=active_job, + claim_token=claim.claim_token, + interval_seconds=( + self._heartbeat_interval_seconds + if self._heartbeat_interval_seconds is not None + else _default_heartbeat_interval(active_job) + ), + stop_timeout_seconds=self._heartbeat_stop_timeout_seconds, + ) + heartbeat.start() + try: - result = adapter.execute(claim.job) + result = adapter.execute(active_job) if not isinstance(result, ObservatoryWorkerExecutionResult): raise TypeError("executor returned an unknown result contract") except Exception as exc: + heartbeat.stop() + if heartbeat.failed: + return ObservatoryWorkerCycleReport( + state="lease-lost", + claim_request_id=claim_request_id, + job_id=claim.job.job_id, + failure_code="claim-heartbeat-lost", + ) failure_code = "executor-error" acknowledgement = self._transport.fail( claimant_id=WORKER_006_CONTOUR_ID, @@ -468,6 +525,15 @@ class ObservatoryWorkerAgent: failure_code=failure_code, ) + heartbeat.stop() + if heartbeat.failed: + return ObservatoryWorkerCycleReport( + state="lease-lost", + claim_request_id=claim_request_id, + job_id=claim.job.job_id, + failure_code="claim-heartbeat-lost", + ) + acknowledgement = self._transport.succeed( claimant_id=WORKER_006_CONTOUR_ID, job_id=claim.job.job_id, @@ -495,6 +561,92 @@ class ObservatoryWorkerAgent: ) +class _ClaimHeartbeat: + """Renew one exact generation while an executor owns Worker resources.""" + + def __init__( + self, + *, + transport: ObservatoryWorkerTransport, + job: SealedObservatoryRecordedJob, + claim_token: str, + interval_seconds: float, + stop_timeout_seconds: float, + ) -> None: + if interval_seconds <= 0: + raise ValueError("Worker heartbeat interval must be positive") + self._transport = transport + self._job = job + self._claim_token = claim_token + self._interval_seconds = interval_seconds + self._stop_timeout_seconds = stop_timeout_seconds + self._stop = threading.Event() + self._state_lock = threading.Lock() + self._failure: Exception | None = None + self._thread = threading.Thread( + target=self._run, + name=f"observatory-heartbeat-{job.job_id}", + daemon=True, + ) + + @property + def failed(self) -> bool: + with self._state_lock: + return self._failure is not None + + def start(self) -> None: + self._thread.start() + + def stop(self) -> None: + self._stop.set() + self._thread.join(timeout=self._stop_timeout_seconds) + if self._thread.is_alive(): + self._record_failure( + ObservatoryWorkerClaimRejectedError( + "Worker claim heartbeat did not stop within its bound" + ) + ) + + def _run(self) -> None: + sequence = self._job.claim_renewal_count + 1 + # Renew immediately once start has been acknowledged. Waiting a full + # interval here would assume that claim/start transport latency consumed + # none of the original lease window. + while not self._stop.is_set(): + try: + acknowledgement = self._transport.renew_claim( + claimant_id=WORKER_006_CONTOUR_ID, + job_id=self._job.job_id, + claim_token=self._claim_token, + claim_generation=self._job.claim_generation, + heartbeat_sequence=sequence, + ) + renewed = _validate_transition_acknowledgement( + acknowledgement, + expected_job=self._job, + expected_state=("claimed", "running"), + ) + if ( + renewed.claim_lease is None + or renewed.claim_lease.renewal_count != sequence + ): + raise ObservatoryWorkerClaimRejectedError( + "Worker heartbeat acknowledgement changed its sequence" + ) + sequence += 1 + except Exception as exc: + self._record_failure(exc) + self._stop.set() + return + if self._stop.wait(self._interval_seconds): + return + + def _record_failure(self, exc: Exception) -> None: + with self._state_lock: + if self._failure is None: + self._failure = exc + + def _validate_claim( payload: Mapping[str, object], *, @@ -519,6 +671,10 @@ def _validate_claim( raise ObservatoryWorkerClaimRejectedError( "Worker claim job is not in a claimed generation" ) + if claim.job.claim_lease is None: + raise ObservatoryWorkerClaimRejectedError( + "Worker claim has no renewable lease" + ) if claim.job.result is not None or claim.job.terminal is not None: raise ObservatoryWorkerClaimRejectedError( "Worker claim already carries a terminal outcome" @@ -556,6 +712,14 @@ def _seal_job(payload: _RecordedJobPayload) -> SealedObservatoryRecordedJob: and payload.checkpoint_policy.allowed_checkpoints ): raise ObservatoryWorkerClaimRejectedError("Worker claim checkpoint policy is inconsistent") + if payload.claim_lease is not None: + claimed_at = _parse_timestamp(payload.claim_lease.claimed_at_utc) + expires_at = _parse_timestamp(payload.claim_lease.expires_at_utc) + heartbeat_at = _parse_timestamp(payload.claim_lease.heartbeat_at_utc) + if not claimed_at <= heartbeat_at < expires_at: + raise ObservatoryWorkerClaimRejectedError( + "Worker claim lease chronology is inconsistent" + ) expected_request_sha256 = _sha256_document( { @@ -615,6 +779,7 @@ def _seal_job(payload: _RecordedJobPayload) -> SealedObservatoryRecordedJob: job_id=payload.job_id, request_sha256=payload.request_sha256, identity_sha256=payload.identity_sha256, + submission_receipt_sha256=payload.submission_receipt_sha256, source_session_id=payload.source.session_id, source_catalog_sha256=payload.source.catalog_sha256, source_bundle_sha256=payload.source.bundle_sha256, @@ -638,6 +803,18 @@ def _seal_job(payload: _RecordedJobPayload) -> SealedObservatoryRecordedJob: checkpoint_policy=payload.checkpoint_policy.mode, allowed_checkpoints=tuple(payload.checkpoint_policy.allowed_checkpoints), claim_generation=payload.claim_generation, + claim_claimed_at_utc=( + None if payload.claim_lease is None else payload.claim_lease.claimed_at_utc + ), + claim_expires_at_utc=( + None if payload.claim_lease is None else payload.claim_lease.expires_at_utc + ), + claim_heartbeat_at_utc=( + None if payload.claim_lease is None else payload.claim_lease.heartbeat_at_utc + ), + claim_renewal_count=( + 0 if payload.claim_lease is None else payload.claim_lease.renewal_count + ), restart_from_zero=payload.restart_from_zero, ) @@ -662,6 +839,12 @@ def _validate_transition_acknowledgement( raise ObservatoryWorkerClaimRejectedError( "Worker transition acknowledgement has an unexpected state" ) + if acknowledgement.state in {"claimed", "running", "preemption-pending"} and ( + acknowledgement.claim_lease is None + ): + raise ObservatoryWorkerClaimRejectedError( + "Worker transition acknowledgement lost the active claim lease" + ) if ( sealed.job_id != expected_job.job_id or sealed.identity_sha256 != expected_job.identity_sha256 @@ -677,6 +860,20 @@ def _default_claim_request_id() -> str: return f"worker-006:{uuid4().hex}" +def _default_heartbeat_interval(job: SealedObservatoryRecordedJob) -> float: + if job.claim_heartbeat_at_utc is None or job.claim_expires_at_utc is None: + raise ObservatoryWorkerClaimRejectedError( + "Worker claim has no heartbeat lease bounds" + ) + remaining = ( + _parse_timestamp(job.claim_expires_at_utc) + - _parse_timestamp(job.claim_heartbeat_at_utc) + ).total_seconds() + if remaining <= 0: + raise ObservatoryWorkerClaimRejectedError("Worker claim lease already expired") + return max(0.5, min(30.0, remaining / 3.0)) + + def _bounded_executor_failure(exc: Exception) -> str: detail = " ".join(str(exc).split()) message = f"Executor adapter raised {type(exc).__name__}." @@ -693,3 +890,7 @@ def _sha256_document(document: Mapping[str, object]) -> str: separators=(",", ":"), ).encode() return hashlib.sha256(payload).hexdigest() + + +def _parse_timestamp(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) diff --git a/src/k1link/observatory/worker_http_transport.py b/src/k1link/observatory/worker_http_transport.py new file mode 100644 index 0000000..bab5c48 --- /dev/null +++ b/src/k1link/observatory/worker_http_transport.py @@ -0,0 +1,1184 @@ +"""Bounded HTTP client for the portable Observatory Worker boundary. + +One object implements queue transitions plus the source-materializer and +result-uploader ports used by :mod:`portable_worker_runtime`. The Worker keeps +the bearer secret locally, caches only the active claim returned by Mission +Core, validates every downloaded/uploaded digest, and chooses all local paths +itself. Server payloads cannot select a command or filesystem destination. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import secrets +import stat +import threading +from collections.abc import Mapping +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal, cast +from urllib.parse import urlsplit + +import httpx + +from k1link.observatory.portable_artifact_transport import ( + PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA, + PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA, + PORTABLE_SOURCE_MATERIALIZATION_SCHEMA, +) +from k1link.observatory.portable_result_contract import ( + OBSERVATION_ONLY_AUTHORITY, + PortableResultPackageManifest, + canonical_json, + relative_artifact_path, +) +from k1link.observatory.portable_worker_runtime import ( + PortableWorkerResultDraft, + PortableWorkerSourceStage, +) +from k1link.observatory.source_admission import ( + PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID, + PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE, +) +from k1link.observatory.worker_agent import ( + WORKER_006_CONTOUR_ID, + ObservatoryWorkerExecutionResult, + ObservatoryWorkerTransport, + SealedObservatoryRecordedJob, +) + +WORKER_HTTP_MAX_JSON_BYTES: Final = 16 * 1024 * 1024 +WORKER_HTTP_COPY_CHUNK_BYTES: Final = 1024 * 1024 +WORKER_HTTP_DEFAULT_TIMEOUT_SECONDS: Final = 30.0 + +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_JOB_ID = re.compile(r"^observatory-run-[a-f0-9]{32}$") +_TOKEN = re.compile(r"^[A-Za-z0-9._:-]{32,512}$") +_CLAIM_TOKEN = re.compile(r"^[a-f0-9]{64}$") +_MEMBER_ID = re.compile(r"^[a-f0-9]{64}$") +_RESULT_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +_ROLE = re.compile(r"^[a-z][a-z0-9.-]{2,95}$") +_MAX_SAFE_INTEGER: Final = 9_007_199_254_740_991 +_CONTOUR_HEADER = "X-Mission-Core-Contour-Id" +_CLAIM_TOKEN_HEADER = "X-Mission-Core-Claim-Token" +_CLAIM_GENERATION_HEADER = "X-Mission-Core-Claim-Generation" +_CONTENT_SHA_HEADER = "X-Mission-Core-Content-Sha256" + + +class ObservatoryWorkerHttpError(RuntimeError): + """The authenticated Worker HTTP boundary failed closed.""" + + +@dataclass(frozen=True, slots=True) +class _ClaimContext: + job_id: str + claim_token: str + claim_generation: int + + +@dataclass(frozen=True, slots=True) +class _SourceMember: + member_id: str + kind: Literal[ + "source-bundle", + "source-capability", + "spatial-replay", + "spatial-replay-metadata", + "camera-init", + "camera-segment", + ] + media_type: str + byte_length: int + sha256: str + artifact_id: str | None + primary: bool + camera_epoch: int | None + camera_sequence: int | None + + +@dataclass(frozen=True, slots=True) +class _UploadMember: + member_id: str + role: str + media_type: str + byte_length: int + sha256: str + uploaded: bool + + +class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport): + """Authenticated queue transport and portable artifact gateway client.""" + + def __init__( + self, + *, + base_url: str, + bearer_token: str, + work_root: Path, + contour_id: str = WORKER_006_CONTOUR_ID, + timeout_seconds: float = WORKER_HTTP_DEFAULT_TIMEOUT_SECONDS, + transport: httpx.BaseTransport | None = None, + ) -> None: + self._base_url = _validated_base_url(base_url) + if _TOKEN.fullmatch(bearer_token) is None: + raise ValueError("Worker bearer credential format is invalid") + if contour_id != WORKER_006_CONTOUR_ID: + raise ValueError("portable Worker gateway is pinned to Worker 006") + if not 1 <= timeout_seconds <= 300: + raise ValueError("Worker HTTP timeout is invalid") + self._headers = { + "Authorization": f"Bearer {bearer_token}", + _CONTOUR_HEADER: contour_id, + "Accept": "application/json", + } + self._work_root = _secure_directory(work_root) + self._client = httpx.Client( + base_url=self._base_url, + headers=self._headers, + timeout=httpx.Timeout(timeout_seconds), + # Artifact streaming and the generation-fenced heartbeat overlap. + # One connection would starve lease renewal behind a long transfer. + limits=httpx.Limits(max_connections=2, max_keepalive_connections=2), + follow_redirects=False, + transport=transport, + ) + self._claim_lock = threading.Lock() + self._claims: dict[str, _ClaimContext] = {} + + def close(self) -> None: + self._client.close() + + def __enter__(self) -> ObservatoryWorkerHttpGateway: + return self + + def __exit__(self, *_args: object) -> None: + self.close() + + def claim_next( + self, + *, + claimant_id: str, + claim_request_id: str, + ) -> Mapping[str, object] | None: + self._require_claimant(claimant_id) + payload = self._json_request( + "POST", + "/api/v1/worker/observatory/recorded-jobs/claims", + json_body={ + "schema_version": "missioncore.observatory-worker-claim-request/v1", + "claim_request_id": claim_request_id, + }, + allow_empty=True, + ) + if payload is None: + return None + claim_token = _string(payload.get("claim_token"), "Worker claim token") + if _CLAIM_TOKEN.fullmatch(claim_token) is None: + raise ObservatoryWorkerHttpError("Worker claim token is invalid") + job = _object(payload.get("job"), "Worker claim job") + job_id = _string(job.get("job_id"), "Worker claim job id") + generation = _integer(job.get("claim_generation"), "Worker claim generation") + _job_id(job_id) + if generation < 1: + raise ObservatoryWorkerHttpError("Worker claim generation is invalid") + with self._claim_lock: + self._claims[job_id] = _ClaimContext(job_id, claim_token, generation) + return payload + + def start( + self, + *, + claimant_id: str, + job_id: str, + claim_token: str, + ) -> Mapping[str, object]: + self._require_claimant(claimant_id) + self._require_cached_claim(job_id, claim_token) + return self._required_json_request( + "POST", + self._job_path(job_id, "start"), + json_body={ + "schema_version": "missioncore.observatory-worker-start-request/v1", + "claim_token": claim_token, + }, + ) + + def renew_claim( + self, + *, + claimant_id: str, + job_id: str, + claim_token: str, + claim_generation: int, + heartbeat_sequence: int, + ) -> Mapping[str, object]: + self._require_claimant(claimant_id) + context = self._require_cached_claim(job_id, claim_token) + if context.claim_generation != claim_generation: + raise ObservatoryWorkerHttpError("Worker claim generation changed") + return self._required_json_request( + "POST", + self._job_path(job_id, "lease/renew"), + json_body={ + "schema_version": "missioncore.observatory-worker-renew-request/v1", + "claim_token": claim_token, + "claim_generation": claim_generation, + "heartbeat_sequence": heartbeat_sequence, + }, + ) + + def succeed( + self, + *, + claimant_id: str, + job_id: str, + claim_token: str, + result_id: str, + result_sha256: str, + ) -> Mapping[str, object]: + self._require_claimant(claimant_id) + self._require_cached_claim(job_id, claim_token) + payload = self._required_json_request( + "POST", + self._job_path(job_id, "succeed"), + json_body={ + "schema_version": "missioncore.observatory-worker-succeed-request/v1", + "claim_token": claim_token, + "result_id": result_id, + "result_sha256": result_sha256, + }, + ) + self._forget_claim(job_id) + return payload + + def fail( + self, + *, + claimant_id: str, + job_id: str, + claim_token: str, + error_code: str, + message: str, + ) -> Mapping[str, object]: + self._require_claimant(claimant_id) + self._require_cached_claim(job_id, claim_token) + payload = self._required_json_request( + "POST", + self._job_path(job_id, "fail"), + json_body={ + "schema_version": "missioncore.observatory-worker-fail-request/v1", + "claim_token": claim_token, + "error_code": error_code, + "message": message, + }, + ) + self._forget_claim(job_id) + return payload + + def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage: + context = self._require_job_context(job) + headers = self._claim_headers(context) + manifest = self._required_json_request( + "GET", + self._job_path(job.job_id, "source-materialization"), + headers=headers, + ) + members = _source_members(manifest, job) + root = _secure_directory( + self._work_root + / "sources" + / job.job_id + / f"generation-{job.claim_generation}" + / job.source_bundle_sha256 + ) + destinations: dict[Path, _SourceMember] = {} + for member in members: + destination = _source_destination(root, member) + if destination in destinations: + raise ObservatoryWorkerHttpError( + "source materialization members select the same local role" + ) + destinations[destination] = member + for destination, member in destinations.items(): + if _matches_file(destination, member.sha256, member.byte_length): + continue + self._download_member( + job_id=job.job_id, + context=context, + member=member, + destination=destination, + ) + manifest_path = root / "materialization-manifest.json" + _write_local_exact(manifest_path, canonical_json(manifest)) + return PortableWorkerSourceStage( + root=root, + source_bundle_sha256=job.source_bundle_sha256, + source_capability_manifest_sha256=job.source_capability_manifest_sha256, + source_adapter_sha256=job.source_adapter_sha256, + ) + + def publish( + self, + job: SealedObservatoryRecordedJob, + draft: PortableWorkerResultDraft, + ) -> ObservatoryWorkerExecutionResult: + context = self._require_job_context(job) + manifest_path = draft.root / "manifest.json" + manifest_payload = _read_local_file(manifest_path, 1024 * 1024) + try: + package = PortableResultPackageManifest.from_bytes(manifest_payload) + except Exception as exc: + raise ObservatoryWorkerHttpError( + "Worker result draft manifest is invalid" + ) from exc + if ( + package.manifest_sha256 != draft.result_sha256 + or package.result.get("result_id") != draft.result_id + or package.job + != { + "job_id": job.job_id, + "request_sha256": job.request_sha256, + "identity_sha256": job.identity_sha256, + "submission_receipt_sha256": job.submission_receipt_sha256, + "claim_generation": job.claim_generation, + } + ): + raise ObservatoryWorkerHttpError( + "Worker result draft belongs to another sealed job" + ) + headers = self._claim_headers(context) + path = self._job_path( + job.job_id, + f"result-packages/{draft.result_sha256}/manifest", + ) + plan = self._required_json_request( + "PUT", + path, + headers={**headers, "Content-Type": "application/json"}, + content=manifest_payload, + ) + if plan.get("package_identity_sha256") != package.identity_sha256: + raise ObservatoryWorkerHttpError( + "result upload plan uses another package identity" + ) + upload_members = _upload_members(plan, job, draft) + artifacts = {artifact.role: artifact for artifact in package.artifacts} + if ( + len(artifacts) != len(package.artifacts) + or len(upload_members) != len(artifacts) + or {member.role for member in upload_members} != set(artifacts) + ): + raise ObservatoryWorkerHttpError("Worker result artifact roles are not unique") + for member in upload_members: + artifact = artifacts.get(member.role) + expected_member_id = hashlib.sha256( + canonical_json( + { + "package_identity_sha256": package.identity_sha256, + "artifact": artifact.as_dict() if artifact is not None else None, + } + ) + ).hexdigest() + if artifact is None or ( + artifact.media_type, + artifact.byte_length, + artifact.sha256, + ) != (member.media_type, member.byte_length, member.sha256) or ( + member.member_id != expected_member_id + ): + raise ObservatoryWorkerHttpError( + "result upload plan differs from the local manifest" + ) + if member.uploaded: + continue + relative = relative_artifact_path(artifact.relative_path) + source = _confined_local_member(draft.root, relative.parts) + with source.open("rb") as stream: + updated = self._required_json_request( + "PUT", + self._job_path( + job.job_id, + ( + f"result-packages/{draft.result_sha256}/members/" + f"{member.member_id}" + ), + ), + headers={**headers, "Content-Type": member.media_type}, + content=stream, + ) + if updated.get("package_identity_sha256") != package.identity_sha256: + raise ObservatoryWorkerHttpError( + "result upload acknowledgement changed package identity" + ) + updated_members = _upload_members(updated, job, draft) + if not any( + item.member_id == member.member_id and item.uploaded + for item in updated_members + ): + raise ObservatoryWorkerHttpError( + "result upload acknowledgement did not seal its member" + ) + receipt = self._required_json_request( + "POST", + self._job_path( + job.job_id, + f"result-packages/{draft.result_sha256}/complete", + ), + headers=headers, + ) + receipt_identity: dict[str, object] = { + "schema_version": PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA, + "job_id": job.job_id, + "job_identity_sha256": job.identity_sha256, + "claim_generation": job.claim_generation, + "claim_token_sha256": hashlib.sha256( + context.claim_token.encode("ascii") + ).hexdigest(), + "result_id": draft.result_id, + "result_sha256": draft.result_sha256, + "package_identity_sha256": package.identity_sha256, + "member_count": len(package.artifacts), + "total_bytes": sum( + artifact.byte_length for artifact in package.artifacts + ), + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + expected_receipt = { + **receipt_identity, + "receipt_sha256": hashlib.sha256( + canonical_json(receipt_identity) + ).hexdigest(), + } + if receipt != expected_receipt: + raise ObservatoryWorkerHttpError( + "result completion receipt differs from the Worker draft" + ) + return ObservatoryWorkerExecutionResult( + result_id=draft.result_id, + result_sha256=draft.result_sha256, + ) + + def _download_member( + self, + *, + job_id: str, + context: _ClaimContext, + member: _SourceMember, + destination: Path, + ) -> None: + parent = _secure_directory(destination.parent) + temporary = parent / f".download-{secrets.token_hex(16)}" + digest = hashlib.sha256() + byte_length = 0 + try: + with self._client.stream( + "GET", + self._job_path(job_id, f"source-members/{member.member_id}"), + headers=self._claim_headers(context), + ) as response: + self._raise_for_status(response) + declared = response.headers.get("content-length") + if declared is not None and declared != str(member.byte_length): + raise ObservatoryWorkerHttpError( + "source member Content-Length changed" + ) + if response.headers.get(_CONTENT_SHA_HEADER) != member.sha256: + raise ObservatoryWorkerHttpError( + "source member digest header changed" + ) + descriptor = os.open( + temporary, + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + with os.fdopen(descriptor, "wb") as stream: + for chunk in response.iter_bytes(WORKER_HTTP_COPY_CHUNK_BYTES): + byte_length += len(chunk) + if byte_length > member.byte_length: + raise ObservatoryWorkerHttpError( + "source member exceeds its declared length" + ) + digest.update(chunk) + stream.write(chunk) + stream.flush() + os.fsync(stream.fileno()) + if byte_length != member.byte_length or digest.hexdigest() != member.sha256: + raise ObservatoryWorkerHttpError( + "source member content differs from its manifest" + ) + _publish_local_file(temporary, destination, member.sha256, member.byte_length) + finally: + with suppress(FileNotFoundError): + temporary.unlink() + + def _required_json_request( + self, + method: str, + path: str, + *, + json_body: object | None = None, + headers: Mapping[str, str] | None = None, + content: bytes | object | None = None, + ) -> dict[str, object]: + payload = self._json_request( + method, + path, + json_body=json_body, + headers=headers, + content=content, + ) + if payload is None: + raise ObservatoryWorkerHttpError("Worker endpoint returned no document") + return payload + + def _json_request( + self, + method: str, + path: str, + *, + json_body: object | None = None, + headers: Mapping[str, str] | None = None, + content: bytes | object | None = None, + allow_empty: bool = False, + ) -> dict[str, object] | None: + try: + with self._client.stream( + method, + path, + json=json_body, + headers=headers, + content=content, # type: ignore[arg-type] + ) as response: + if allow_empty and response.status_code == 204: + return None + self._raise_for_status(response) + payload = bytearray() + for chunk in response.iter_bytes(): + payload.extend(chunk) + if len(payload) > WORKER_HTTP_MAX_JSON_BYTES: + raise ObservatoryWorkerHttpError( + "Worker JSON response exceeds bounds" + ) + except ObservatoryWorkerHttpError: + raise + except httpx.HTTPError as exc: + raise ObservatoryWorkerHttpError("Worker HTTP transport is unavailable") from exc + if not payload: + if allow_empty: + return None + raise ObservatoryWorkerHttpError("Worker endpoint returned an empty response") + try: + decoded = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ObservatoryWorkerHttpError("Worker response is not valid JSON") from exc + return _object(decoded, "Worker response") + + def _raise_for_status(self, response: httpx.Response) -> None: + if response.is_redirect: + raise ObservatoryWorkerHttpError("Worker endpoint redirect was rejected") + if response.status_code < 200 or response.status_code >= 300: + raise ObservatoryWorkerHttpError( + f"Worker endpoint rejected the request with HTTP {response.status_code}" + ) + + def _require_job_context( + self, + job: SealedObservatoryRecordedJob, + ) -> _ClaimContext: + with self._claim_lock: + context = self._claims.get(job.job_id) + if context is None or context.claim_generation != job.claim_generation: + raise ObservatoryWorkerHttpError( + "Worker artifact operation has no matching active claim" + ) + return context + + def _require_cached_claim(self, job_id: str, claim_token: str) -> _ClaimContext: + _job_id(job_id) + with self._claim_lock: + context = self._claims.get(job_id) + if context is None or context.claim_token != claim_token: + raise ObservatoryWorkerHttpError("Worker claim context is stale") + return context + + def _forget_claim(self, job_id: str) -> None: + with self._claim_lock: + self._claims.pop(job_id, None) + + def _claim_headers(self, context: _ClaimContext) -> dict[str, str]: + return { + _CLAIM_TOKEN_HEADER: context.claim_token, + _CLAIM_GENERATION_HEADER: str(context.claim_generation), + } + + @staticmethod + def _require_claimant(claimant_id: str) -> None: + if claimant_id != WORKER_006_CONTOUR_ID: + raise ObservatoryWorkerHttpError("Worker claimant identity changed") + + @staticmethod + def _job_path(job_id: str, suffix: str) -> str: + _job_id(job_id) + if ( + not suffix + or suffix.startswith("/") + or ".." in suffix.split("/") + or any(not part for part in suffix.split("/")) + ): + raise ValueError("Worker endpoint suffix is invalid") + return f"/api/v1/worker/observatory/recorded-jobs/{job_id}/{suffix}" + + +def _source_members( + document: Mapping[str, object], + job: SealedObservatoryRecordedJob, +) -> tuple[_SourceMember, ...]: + if ( + set(document) + != { + "schema_version", + "job_id", + "job_identity_sha256", + "claim_generation", + "source", + "members", + "authority", + } + or document.get("schema_version") != PORTABLE_SOURCE_MATERIALIZATION_SCHEMA + or document.get("job_id") != job.job_id + or document.get("job_identity_sha256") != job.identity_sha256 + or document.get("claim_generation") != job.claim_generation + or document.get("authority") != OBSERVATION_ONLY_AUTHORITY + ): + raise ObservatoryWorkerHttpError( + "source materialization belongs to another sealed job" + ) + source = _object(document.get("source"), "source materialization identity") + if ( + set(source) + != {"session_id", "bundle_sha256", "capability_manifest_sha256"} + or source.get("session_id") != job.source_session_id + or source.get("bundle_sha256") != job.source_bundle_sha256 + or source.get("capability_manifest_sha256") + != job.source_capability_manifest_sha256 + ): + raise ObservatoryWorkerHttpError("source materialization identity changed") + values = document.get("members") + if not isinstance(values, list) or not 2 <= len(values) <= 100_000: + raise ObservatoryWorkerHttpError("source materialization members are invalid") + members = tuple(_source_member(value) for value in values) + if ( + len({member.member_id for member in members}) != len(members) + or tuple(member.member_id for member in members) + != tuple(sorted(member.member_id for member in members)) + or sum(member.byte_length for member in members) + > 2 * 1024 * 1024 * 1024 * 1024 + ): + raise ObservatoryWorkerHttpError("source materialization bounds changed") + for member in members: + expected_member_id = hashlib.sha256( + canonical_json( + { + "job_identity_sha256": job.identity_sha256, + "source_bundle_sha256": job.source_bundle_sha256, + "kind": member.kind, + "artifact_id": member.artifact_id, + "primary": member.primary, + "camera_epoch": member.camera_epoch, + "camera_sequence": member.camera_sequence, + "media_type": member.media_type, + "byte_length": member.byte_length, + "sha256": member.sha256, + } + ) + ).hexdigest() + if member.member_id != expected_member_id: + raise ObservatoryWorkerHttpError( + "source materialization member identity changed" + ) + bundle_members = tuple( + member for member in members if member.kind == "source-bundle" + ) + capability_members = tuple( + member for member in members if member.kind == "source-capability" + ) + if ( + len(bundle_members) != 1 + or bundle_members[0].sha256 != job.source_bundle_sha256 + or len(capability_members) != 1 + or capability_members[0].sha256 + != job.source_capability_manifest_sha256 + ): + raise ObservatoryWorkerHttpError("source materialization documents are incomplete") + if sum( + member.kind == "spatial-replay" and member.primary for member in members + ) != 1: + raise ObservatoryWorkerHttpError( + "source materialization primary replay is invalid" + ) + metadata_members = tuple( + member for member in members if member.kind == "spatial-replay-metadata" + ) + if len(metadata_members) > 1 or any( + member.artifact_id != PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID + or member.media_type != PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE + or member.primary + for member in metadata_members + ): + raise ObservatoryWorkerHttpError( + "source materialization replay metadata is invalid" + ) + camera_inits = tuple( + member for member in members if member.kind == "camera-init" + ) + camera_segments = tuple( + member for member in members if member.kind == "camera-segment" + ) + if ( + len(camera_inits) != 1 + or not camera_segments + or any( + member.artifact_id != camera_inits[0].artifact_id + or member.camera_epoch != camera_inits[0].camera_epoch + for member in camera_segments + ) + ): + raise ObservatoryWorkerHttpError( + "source materialization camera epoch is incomplete" + ) + return members + + +def _source_member(value: object) -> _SourceMember: + row = _object(value, "source materialization member") + expected = { + "member_id", + "kind", + "media_type", + "byte_length", + "sha256", + "artifact_id", + "primary", + "camera_epoch", + "camera_sequence", + } + if set(row) != expected: + raise ObservatoryWorkerHttpError("source materialization member fields changed") + kind = row["kind"] + if kind not in { + "source-bundle", + "source-capability", + "spatial-replay", + "spatial-replay-metadata", + "camera-init", + "camera-segment", + }: + raise ObservatoryWorkerHttpError("source materialization member kind changed") + member_id = _string(row["member_id"], "source member id") + sha256 = _string(row["sha256"], "source member sha256") + byte_length = _integer(row["byte_length"], "source member byte length") + if ( + _MEMBER_ID.fullmatch(member_id) is None + or _SHA256.fullmatch(sha256) is None + or not 0 <= byte_length <= 2 * 1024 * 1024 * 1024 * 1024 + or not isinstance(row["primary"], bool) + ): + raise ObservatoryWorkerHttpError("source materialization member is invalid") + artifact_id = row["artifact_id"] + camera_epoch = row["camera_epoch"] + camera_sequence = row["camera_sequence"] + if artifact_id is not None and ( + not isinstance(artifact_id, str) or _SOURCE_ID.fullmatch(artifact_id) is None + ): + raise ObservatoryWorkerHttpError("source artifact id is invalid") + for ordinal in (camera_epoch, camera_sequence): + if ordinal is not None and ( + not isinstance(ordinal, int) + or isinstance(ordinal, bool) + or not 1 <= ordinal <= _MAX_SAFE_INTEGER + ): + raise ObservatoryWorkerHttpError("source camera ordinal is invalid") + if kind in {"source-bundle", "source-capability"}: + if ( + artifact_id is not None + or row["primary"] + or camera_epoch is not None + or camera_sequence is not None + ): + raise ObservatoryWorkerHttpError( + "source document member metadata is invalid" + ) + elif kind in {"spatial-replay", "spatial-replay-metadata"}: + if ( + artifact_id is None + or camera_epoch is not None + or camera_sequence is not None + or (kind == "spatial-replay-metadata" and row["primary"]) + ): + raise ObservatoryWorkerHttpError( + "spatial source member metadata is invalid" + ) + elif kind == "camera-init": + if ( + artifact_id is None + or row["primary"] + or camera_epoch is None + or camera_sequence is not None + ): + raise ObservatoryWorkerHttpError( + "camera init member metadata is invalid" + ) + elif ( + artifact_id is None + or row["primary"] + or camera_epoch is None + or camera_sequence is None + ): + raise ObservatoryWorkerHttpError( + "camera segment member metadata is invalid" + ) + return _SourceMember( + member_id=member_id, + kind=cast( + Literal[ + "source-bundle", + "source-capability", + "spatial-replay", + "spatial-replay-metadata", + "camera-init", + "camera-segment", + ], + kind, + ), + media_type=_media_type(row["media_type"], "source member media type"), + byte_length=byte_length, + sha256=sha256, + artifact_id=artifact_id, + primary=row["primary"], + camera_epoch=cast(int | None, camera_epoch), + camera_sequence=cast(int | None, camera_sequence), + ) + + +def _source_destination(root: Path, member: _SourceMember) -> Path: + if member.kind == "source-bundle": + return root / "source-bundle.json" + if member.kind == "source-capability": + return root / "source-capability.json" + if member.kind == "spatial-replay": + if member.primary: + return root / "mqtt.raw.k1mqtt" + return _secure_directory(root / "spatial") / member.member_id + if member.kind == "spatial-replay-metadata": + return root / "mqtt.metadata.jsonl" + if member.camera_epoch is None: + raise ObservatoryWorkerHttpError("camera source member has no epoch") + epoch = _secure_directory(root / "camera" / f"epoch-{member.camera_epoch}") + if member.kind == "camera-init": + return epoch / "init.mp4" + if member.camera_sequence is None: + raise ObservatoryWorkerHttpError("camera segment has no sequence") + return _secure_directory(epoch / "segments") / f"{member.camera_sequence}.m4s" + + +def _upload_members( + document: Mapping[str, object], + job: SealedObservatoryRecordedJob, + draft: PortableWorkerResultDraft, +) -> tuple[_UploadMember, ...]: + if ( + set(document) + != { + "schema_version", + "job_id", + "claim_generation", + "result_id", + "result_sha256", + "package_identity_sha256", + "members", + "complete", + "authority", + } + or document.get("schema_version") != PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA + or document.get("job_id") != job.job_id + or document.get("claim_generation") != job.claim_generation + or document.get("result_id") != draft.result_id + or document.get("result_sha256") != draft.result_sha256 + or document.get("authority") != OBSERVATION_ONLY_AUTHORITY + or not isinstance(document.get("complete"), bool) + or not _is_digest(document.get("package_identity_sha256")) + ): + raise ObservatoryWorkerHttpError("result upload plan identity changed") + values = document.get("members") + if not isinstance(values, list) or not 1 <= len(values) <= 128: + raise ObservatoryWorkerHttpError("result upload members are invalid") + members: list[_UploadMember] = [] + for value in values: + row = _object(value, "result upload member") + if set(row) != { + "member_id", + "role", + "media_type", + "byte_length", + "sha256", + "uploaded", + }: + raise ObservatoryWorkerHttpError("result upload member fields changed") + member_id = _string(row["member_id"], "result member id") + sha256 = _string(row["sha256"], "result member sha256") + byte_length = _integer(row["byte_length"], "result member byte length") + if ( + _MEMBER_ID.fullmatch(member_id) is None + or _SHA256.fullmatch(sha256) is None + or not 0 <= byte_length <= 64 * 1024 * 1024 * 1024 + or not isinstance(row["uploaded"], bool) + ): + raise ObservatoryWorkerHttpError("result upload member is invalid") + members.append( + _UploadMember( + member_id=member_id, + role=_validated_role(row["role"]), + media_type=_media_type(row["media_type"], "result member media type"), + byte_length=byte_length, + sha256=sha256, + uploaded=row["uploaded"], + ) + ) + if ( + len({member.member_id for member in members}) != len(members) + or len({member.role for member in members}) != len(members) + or tuple(member.member_id for member in members) + != tuple(sorted(member.member_id for member in members)) + or sum(member.byte_length for member in members) + > 256 * 1024 * 1024 * 1024 + or document.get("complete") != all(member.uploaded for member in members) + ): + raise ObservatoryWorkerHttpError("result upload member inventory is invalid") + return tuple(members) + + +def _validated_base_url(value: str) -> str: + parsed = urlsplit(value) + if ( + parsed.query + or parsed.fragment + or parsed.username is not None + or parsed.password is not None + or parsed.path not in {"", "/"} + or parsed.scheme not in {"http", "https"} + or not parsed.hostname + ): + raise ValueError("Worker Mission Core base URL is invalid") + if parsed.scheme == "http" and parsed.hostname not in {"127.0.0.1", "localhost", "::1"}: + raise ValueError("unencrypted Worker transport requires a loopback tunnel") + return value.rstrip("/") + + +def _confined_local_member(root: Path, parts: tuple[str, ...]) -> Path: + try: + resolved_root = root.resolve(strict=True) + candidate = resolved_root.joinpath(*parts) + current = resolved_root + for part in parts: + current = current / part + metadata = current.lstat() + if stat.S_ISLNK(metadata.st_mode): + raise ObservatoryWorkerHttpError("result draft contains a symlink") + resolved = candidate.resolve(strict=True) + metadata = candidate.lstat() + except ObservatoryWorkerHttpError: + raise + except OSError as exc: + raise ObservatoryWorkerHttpError("result draft member is unavailable") from exc + if not resolved.is_relative_to(resolved_root) or not stat.S_ISREG(metadata.st_mode): + raise ObservatoryWorkerHttpError("result draft member escapes its root") + return resolved + + +def _secure_directory(path: Path) -> Path: + candidate = path.expanduser().absolute() + candidate.mkdir(mode=0o700, parents=True, exist_ok=True) + try: + metadata = candidate.lstat() + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise ObservatoryWorkerHttpError("Worker local artifact root is unavailable") from exc + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise ObservatoryWorkerHttpError("Worker local artifact root is unsafe") + return resolved + + +def _write_local_exact(path: Path, payload: bytes) -> None: + if path.exists(): + if _read_local_file(path, max(1, len(payload))) == payload: + return + raise ObservatoryWorkerHttpError("Worker local manifest identity collided") + parent = _secure_directory(path.parent) + temporary = parent / f".tmp-{secrets.token_hex(16)}" + descriptor = os.open( + temporary, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.link(temporary, path, follow_symlinks=False) + os.chmod(path, 0o400, follow_symlinks=False) + except FileExistsError: + if _read_local_file(path, max(1, len(payload))) != payload: + raise ObservatoryWorkerHttpError( + "Worker local manifest identity collided" + ) from None + finally: + with suppress(FileNotFoundError): + temporary.unlink() + + +def _publish_local_file( + temporary: Path, + destination: Path, + sha256: str, + byte_length: int, +) -> None: + if _matches_file(destination, sha256, byte_length): + return + if destination.exists(): + raise ObservatoryWorkerHttpError("Worker local member identity collided") + try: + os.link(temporary, destination, follow_symlinks=False) + os.chmod(destination, 0o400, follow_symlinks=False) + except FileExistsError: + if not _matches_file(destination, sha256, byte_length): + raise ObservatoryWorkerHttpError( + "Worker local member publication collided" + ) from None + + +def _matches_file(path: Path, sha256: str, byte_length: int) -> bool: + try: + digest, actual_bytes = _hash_local_regular_file(path, byte_length) + return actual_bytes == byte_length and digest == sha256 + except (OSError, ObservatoryWorkerHttpError): + return False + + +def _read_local_file(path: Path, maximum_bytes: int) -> bytes: + descriptor = -1 + try: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size < 0 + or before.st_size > maximum_bytes + ): + raise ObservatoryWorkerHttpError("Worker local file is outside bounds") + payload = bytearray() + while len(payload) < before.st_size: + chunk = os.read( + descriptor, + min(WORKER_HTTP_COPY_CHUNK_BYTES, before.st_size - len(payload)), + ) + if not chunk: + break + payload.extend(chunk) + after = os.fstat(descriptor) + if len(payload) != before.st_size or _stat_identity(before) != _stat_identity(after): + raise ObservatoryWorkerHttpError("Worker local file changed while read") + return bytes(payload) + except ObservatoryWorkerHttpError: + raise + except OSError as exc: + raise ObservatoryWorkerHttpError("Worker local file is unavailable") from exc + finally: + if descriptor >= 0: + os.close(descriptor) + + +def _hash_local_regular_file(path: Path, maximum_bytes: int) -> tuple[str, int]: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size < 0 + or before.st_size > maximum_bytes + ): + raise ObservatoryWorkerHttpError("Worker local file is outside bounds") + digest = hashlib.sha256() + byte_length = 0 + while chunk := os.read(descriptor, WORKER_HTTP_COPY_CHUNK_BYTES): + byte_length += len(chunk) + digest.update(chunk) + after = os.fstat(descriptor) + if byte_length != before.st_size or _stat_identity(before) != _stat_identity(after): + raise ObservatoryWorkerHttpError("Worker local file changed while hashed") + return digest.hexdigest(), byte_length + finally: + os.close(descriptor) + + +def _stat_identity(metadata: os.stat_result) -> tuple[int, int, int, int]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + metadata.st_mtime_ns, + ) + + +def _object(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise ObservatoryWorkerHttpError(f"{label} must be an object") + return value + + +def _string(value: object, label: str) -> str: + if not isinstance(value, str): + raise ObservatoryWorkerHttpError(f"{label} must be a string") + return value + + +def _media_type(value: object, label: str) -> str: + text = _string(value, label) + if ( + not 3 <= len(text) <= 255 + or "/" not in text + or text != text.strip() + or any(ord(character) < 32 or ord(character) > 126 for character in text) + ): + raise ObservatoryWorkerHttpError(f"{label} is invalid") + return text + + +def _validated_role(value: object) -> str: + role = _string(value, "result member role") + if _ROLE.fullmatch(role) is None: + raise ObservatoryWorkerHttpError("result member role is invalid") + return role + + +def _integer(value: object, label: str) -> int: + if not isinstance(value, int) or isinstance(value, bool): + raise ObservatoryWorkerHttpError(f"{label} must be an integer") + return value + + +def _job_id(value: str) -> None: + if _JOB_ID.fullmatch(value) is None: + raise ObservatoryWorkerHttpError("Worker job id is invalid") + + +def _is_digest(value: object) -> bool: + return isinstance(value, str) and _SHA256.fullmatch(value) is not None diff --git a/src/k1link/observatory/worker_service.py b/src/k1link/observatory/worker_service.py new file mode 100644 index 0000000..503386f --- /dev/null +++ b/src/k1link/observatory/worker_service.py @@ -0,0 +1,357 @@ +"""Install-time composition and bounded polling for portable Worker 006. + +The durable service owns transport cadence only. A reviewed Worker release +must inject an in-memory executor registry whose four-digest identities cover +every portable RunDefinition advertised as ready. Configuration cannot name +Python modules, commands, images, or executable paths, so neither Mission Core +nor an environment variable can turn an unsealed candidate into code. +""" + +from __future__ import annotations + +import os +import re +import stat +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from threading import Event +from typing import Final +from urllib.parse import urlsplit + +import httpx + +from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry +from k1link.observatory.worker_agent import ( + ObservatoryWorkerAgent, + ObservatoryWorkerCycleReport, + ObservatoryWorkerExecutorIdentity, + ObservatoryWorkerExecutorRegistry, + ObservatoryWorkerExecutorUnavailableError, +) +from k1link.observatory.worker_http_transport import ( + ObservatoryWorkerHttpError, + ObservatoryWorkerHttpGateway, +) + +OBSERVATORY_WORKER_BASE_URL_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_BASE_URL" +OBSERVATORY_WORKER_TOKEN_FILE_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE" +OBSERVATORY_WORKER_WORK_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT" +OBSERVATORY_WORKER_IDLE_POLL_SECONDS_ENV: Final = ( + "MISSIONCORE_OBSERVATORY_WORKER_IDLE_POLL_SECONDS" +) +OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS_ENV: Final = ( + "MISSIONCORE_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS" +) +OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES_ENV: Final = ( + "MISSIONCORE_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES" +) + +DEFAULT_OBSERVATORY_WORKER_BASE_URL: Final = "http://127.0.0.1:18080" +DEFAULT_OBSERVATORY_WORKER_IDLE_POLL_SECONDS: Final = 1.0 +DEFAULT_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS: Final = 5.0 +DEFAULT_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES: Final = 12 + +_TOKEN = re.compile(r"^[A-Za-z0-9._:-]{32,512}$") + + +class ObservatoryWorkerServiceError(RuntimeError): + """Worker 006 cannot start without its exact local operational boundary.""" + + +@dataclass(frozen=True, slots=True) +class ObservatoryWorkerServiceConfiguration: + """Path-only service configuration; it never contains a plaintext secret.""" + + base_url: str + bearer_token_file: Path + work_root: Path + idle_poll_seconds: float = DEFAULT_OBSERVATORY_WORKER_IDLE_POLL_SECONDS + transport_backoff_seconds: float = ( + DEFAULT_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS + ) + max_consecutive_transport_failures: int = ( + DEFAULT_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES + ) + + def __post_init__(self) -> None: + _validate_worker_base_url(self.base_url) + _absolute_path(self.bearer_token_file, "Worker bearer token file") + _absolute_path(self.work_root, "Worker work root") + if isinstance(self.idle_poll_seconds, bool) or not ( + 0.05 <= self.idle_poll_seconds <= 300.0 + ): + raise ValueError("Worker idle poll interval is invalid") + if isinstance(self.transport_backoff_seconds, bool) or not ( + 0.05 <= self.transport_backoff_seconds <= 300.0 + ): + raise ValueError("Worker transport backoff is invalid") + if ( + isinstance(self.max_consecutive_transport_failures, bool) + or not isinstance(self.max_consecutive_transport_failures, int) + or not 1 <= self.max_consecutive_transport_failures <= 10_000 + ): + raise ValueError("Worker transport failure bound is invalid") + + @classmethod + def from_environment( + cls, + environment: Mapping[str, str] | None = None, + ) -> ObservatoryWorkerServiceConfiguration: + values = os.environ if environment is None else environment + token_file = _required_path(values, OBSERVATORY_WORKER_TOKEN_FILE_ENV) + work_root = _required_path(values, OBSERVATORY_WORKER_WORK_ROOT_ENV) + return cls( + base_url=values.get( + OBSERVATORY_WORKER_BASE_URL_ENV, + DEFAULT_OBSERVATORY_WORKER_BASE_URL, + ), + bearer_token_file=token_file, + work_root=work_root, + idle_poll_seconds=_environment_float( + values, + OBSERVATORY_WORKER_IDLE_POLL_SECONDS_ENV, + DEFAULT_OBSERVATORY_WORKER_IDLE_POLL_SECONDS, + ), + transport_backoff_seconds=_environment_float( + values, + OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS_ENV, + DEFAULT_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS, + ), + max_consecutive_transport_failures=_environment_int( + values, + OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES_ENV, + DEFAULT_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES, + ), + ) + + +@dataclass(slots=True) +class InstalledObservatoryWorkerService: + """A composed gateway and agent owned by one installed Worker release.""" + + configuration: ObservatoryWorkerServiceConfiguration + gateway: ObservatoryWorkerHttpGateway + agent: ObservatoryWorkerAgent + + def close(self) -> None: + self.gateway.close() + + def run( + self, + *, + stop: Event, + on_cycle: Callable[[ObservatoryWorkerCycleReport], None] | None = None, + ) -> None: + """Poll until stopped, with a bounded consecutive transport-failure gate.""" + + consecutive_transport_failures = 0 + try: + while not stop.is_set(): + try: + report = self.agent.run_once() + except ObservatoryWorkerHttpError as exc: + consecutive_transport_failures += 1 + if ( + consecutive_transport_failures + >= self.configuration.max_consecutive_transport_failures + ): + raise ObservatoryWorkerServiceError( + "Worker transport exceeded its consecutive failure bound" + ) from exc + stop.wait(self.configuration.transport_backoff_seconds) + continue + consecutive_transport_failures = 0 + if on_cycle is not None: + on_cycle(report) + if report.state in { + "empty", + "deferred", + "rejected", + "lease-lost", + }: + stop.wait(self.configuration.idle_poll_seconds) + finally: + self.close() + + +def compose_installed_observatory_worker_service( + *, + configuration: ObservatoryWorkerServiceConfiguration, + definitions: PortableRunDefinitionRegistry, + executors: ObservatoryWorkerExecutorRegistry, + http_transport: httpx.BaseTransport | None = None, +) -> InstalledObservatoryWorkerService: + """Bind transport to an install-time registry after exact coverage checks. + + This is the fixed seam a reviewed Worker release calls. There is no + dynamic import/provider name in service configuration. Blocked catalog + candidates are ignored, while an empty ready set or any missing exact + local executor identity rejects service startup before the first claim. + """ + + require_ready_executor_coverage(definitions=definitions, executors=executors) + bearer_token = load_observatory_worker_bearer_token( + configuration.bearer_token_file + ) + try: + gateway = ObservatoryWorkerHttpGateway( + base_url=configuration.base_url, + bearer_token=bearer_token, + work_root=configuration.work_root, + transport=http_transport, + ) + finally: + # The immutable string remains owned by the gateway headers for the + # service lifetime; this local binding must not outlive composition. + del bearer_token + return InstalledObservatoryWorkerService( + configuration=configuration, + gateway=gateway, + agent=ObservatoryWorkerAgent(transport=gateway, executors=executors), + ) + + +def require_ready_executor_coverage( + *, + definitions: PortableRunDefinitionRegistry, + executors: ObservatoryWorkerExecutorRegistry, +) -> tuple[ObservatoryWorkerExecutorIdentity, ...]: + """Prove every server-advertisable definition has one local identity.""" + + ready = definitions.ready_recorded_definitions() + if not ready: + raise ObservatoryWorkerServiceError( + "no portable RunDefinition has a sealed ready executor" + ) + identities: list[ObservatoryWorkerExecutorIdentity] = [] + for definition in ready: + identity = ObservatoryWorkerExecutorIdentity( + release_sha256=definition.executor_release_sha256, + image_sha256=definition.executor_image_sha256, + model_manifest_sha256=definition.model_manifest_sha256, + resource_profile_sha256=definition.resource_profile_sha256, + ) + try: + executors.resolve(identity) + except ObservatoryWorkerExecutorUnavailableError as exc: + raise ObservatoryWorkerServiceError( + "a ready portable RunDefinition has no exact local executor identity" + ) from exc + identities.append(identity) + return tuple(identities) + + +def load_observatory_worker_bearer_token(path: Path) -> str: + """Read one private regular ASCII token without accepting links/newlines.""" + + candidate = path.expanduser().absolute() + descriptor: int | None = None + try: + descriptor = os.open( + candidate, + os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + raise ObservatoryWorkerServiceError( + "Worker bearer credential must be a regular file" + ) + if os.name != "posix": + raise ObservatoryWorkerServiceError( + "native Worker credential ACL verification is not available; " + "use the admitted POSIX Worker service runtime" + ) + if metadata.st_mode & 0o077: + raise ObservatoryWorkerServiceError( + "Worker bearer credential permissions are too broad" + ) + if not 32 <= metadata.st_size <= 512: + raise ObservatoryWorkerServiceError( + "Worker bearer credential format is invalid" + ) + with os.fdopen(descriptor, "rb") as stream: + descriptor = None + payload = stream.read(513) + except ObservatoryWorkerServiceError: + raise + except OSError as exc: + raise ObservatoryWorkerServiceError( + "Worker bearer credential is unavailable" + ) from exc + finally: + if descriptor is not None: + os.close(descriptor) + try: + token = payload.decode("ascii") + except UnicodeDecodeError as exc: + raise ObservatoryWorkerServiceError( + "Worker bearer credential is not ASCII" + ) from exc + if _TOKEN.fullmatch(token) is None: + raise ObservatoryWorkerServiceError( + "Worker bearer credential format is invalid" + ) + return token + + +def _validate_worker_base_url(value: str) -> None: + parsed = urlsplit(value) + if ( + value != value.strip() + or parsed.query + or parsed.fragment + or parsed.username is not None + or parsed.password is not None + or parsed.path not in {"", "/"} + or parsed.scheme not in {"http", "https"} + or not parsed.hostname + ): + raise ValueError("Worker Mission Core base URL is invalid") + if parsed.scheme == "http" and parsed.hostname not in { + "127.0.0.1", + "localhost", + "::1", + }: + raise ValueError("unencrypted Worker transport requires a loopback tunnel") + + +def _absolute_path(path: Path, label: str) -> None: + if not path.is_absolute() or str(path) != str(path).strip(): + raise ValueError(f"{label} must be an absolute path") + + +def _required_path(environment: Mapping[str, str], name: str) -> Path: + value = environment.get(name, "") + if not value or value != value.strip(): + raise ObservatoryWorkerServiceError(f"{name} is required") + path = Path(value) + _absolute_path(path, name) + return path + + +def _environment_float( + environment: Mapping[str, str], + name: str, + default: float, +) -> float: + value = environment.get(name, "") + if not value: + return default + try: + return float(value) + except ValueError as exc: + raise ObservatoryWorkerServiceError(f"{name} is invalid") from exc + + +def _environment_int( + environment: Mapping[str, str], + name: str, + default: int, +) -> int: + value = environment.get(name, "") + if not value: + return default + if not value.isascii() or not value.isdecimal(): + raise ObservatoryWorkerServiceError(f"{name} is invalid") + return int(value) diff --git a/src/k1link/observatory/worker_tunnel_launchd.py b/src/k1link/observatory/worker_tunnel_launchd.py new file mode 100644 index 0000000..3507ec7 --- /dev/null +++ b/src/k1link/observatory/worker_tunnel_launchd.py @@ -0,0 +1,142 @@ +"""Pure launchd plan for the Mac-owned Worker 006 reverse SSH tunnel.""" + +from __future__ import annotations + +import hashlib +import os +import plistlib +import stat +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +OBSERVATORY_WORKER_TUNNEL_LABEL: Final = ( + "com.nodedc.observatory-worker-tunnel.local" +) +OBSERVATORY_WORKER_TUNNEL_SCHEMA: Final = ( + "missioncore.observatory-worker-tunnel-plan/v1" +) +OBSERVATORY_WORKER_TUNNEL_PORT: Final = 18080 + + +class ObservatoryWorkerTunnelPlanError(RuntimeError): + """The reverse tunnel cannot be declared without weakening its boundary.""" + + +@dataclass(frozen=True, slots=True) +class ObservatoryWorkerTunnelLaunchAgentPlan: + agent_path: Path + data_directory: Path + desired_sha256: str + desired_payload: bytes + + def to_dict(self) -> dict[str, object]: + return { + "schema_version": OBSERVATORY_WORKER_TUNNEL_SCHEMA, + "label": OBSERVATORY_WORKER_TUNNEL_LABEL, + "agent_path": str(self.agent_path), + "data_directory": str(self.data_directory), + "desired_sha256": self.desired_sha256, + "transport": { + "owner": "mac-launchd", + "ssh_alias": "mission-gpu", + "worker_listener": "127.0.0.1:18080", + "mission_core_target": "127.0.0.1:8000", + "encrypted": True, + "worker_listener_loopback_only": True, + "bearer_credential_in_arguments": False, + }, + "changes": { + "durable_mutation_performed": False, + "requires_hash-gated_install": True, + }, + } + + +def plan_observatory_worker_tunnel_launch_agent( + *, + data_directory: Path, + agent_path: Path, + ssh_path: Path = Path("/usr/bin/ssh"), +) -> ObservatoryWorkerTunnelLaunchAgentPlan: + """Build, but never install, the exact reverse-loopback declaration.""" + + data_root = _private_directory(data_directory) + ssh = _exact_executable(ssh_path) + target_path = agent_path.expanduser().absolute() + log_path = data_root / "observatory-worker-tunnel.log" + arguments = [ + str(ssh), + "-o", + "BatchMode=yes", + "-o", + "ExitOnForwardFailure=yes", + "-o", + "ServerAliveInterval=15", + "-o", + "ServerAliveCountMax=3", + "-o", + "RequestTTY=no", + "-N", + "-T", + "-R", + "127.0.0.1:18080:127.0.0.1:8000", + "mission-gpu", + ] + desired = { + "Label": OBSERVATORY_WORKER_TUNNEL_LABEL, + "ProgramArguments": arguments, + "KeepAlive": True, + "RunAtLoad": True, + "AbandonProcessGroup": False, + "ProcessType": "Background", + "ThrottleInterval": 5, + "ExitTimeOut": 10, + "StandardOutPath": str(log_path), + "StandardErrorPath": str(log_path), + } + payload = plistlib.dumps(desired, fmt=plistlib.FMT_XML, sort_keys=True) + return ObservatoryWorkerTunnelLaunchAgentPlan( + agent_path=target_path, + data_directory=data_root, + desired_sha256=hashlib.sha256(payload).hexdigest(), + desired_payload=payload, + ) + + +def _private_directory(path: Path) -> Path: + candidate = path.expanduser().absolute() + try: + metadata = candidate.lstat() + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise ObservatoryWorkerTunnelPlanError( + "Mission Core data directory is unavailable" + ) from exc + if ( + resolved != candidate + or not stat.S_ISDIR(metadata.st_mode) + or stat.S_IMODE(metadata.st_mode) != 0o700 + or metadata.st_uid != os.getuid() + ): + raise ObservatoryWorkerTunnelPlanError( + "Mission Core data directory is not private and canonical" + ) + return resolved + + +def _exact_executable(path: Path) -> Path: + candidate = path.expanduser().absolute() + try: + metadata = candidate.lstat() + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise ObservatoryWorkerTunnelPlanError("SSH executable is unavailable") from exc + if ( + resolved != candidate + or stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or not os.access(candidate, os.X_OK) + ): + raise ObservatoryWorkerTunnelPlanError("SSH executable is not exact") + return resolved diff --git a/src/k1link/web/app.py b/src/k1link/web/app.py index 741cedd..f3a17ba 100644 --- a/src/k1link/web/app.py +++ b/src/k1link/web/app.py @@ -47,20 +47,38 @@ from k1link.observatory.m49_queue_binding import ( M49QueueBindingError, M49RecordedQueueBindingService, ) +from k1link.observatory.portable_queue_binding import ( + PortableQueueBindingError, + PortableRecordedQueueBindingService, +) +from k1link.observatory.portable_result_contract import ( + PortableCalculationProfileRegistry, + PortableResultContractValidatorRegistry, +) +from k1link.observatory.portable_result_publisher import ( + resolve_published_portable_calculation_profile, +) from k1link.observatory.portable_run_definitions import ( PortableRunDefinitionRegistry, PortableRunDefinitionRegistryError, ) from k1link.observatory.portable_setup_projection import ( - PORTABLE_LAB_V1_SETUP_ID, - PortableLabV1SetupProjector, PortableSetupProjectionError, + PortableSetupProjector, + portable_calculation_profile_registry, +) +from k1link.observatory.portable_worker_integration import ( + PortableObservatoryWorkerIntegration, + PortableWorkerIntegrationError, + PortableWorkerStorageRoots, + build_portable_observatory_worker_integration, + portable_result_validator_registry, ) from k1link.observatory.recorded_jobs import ( ObservatoryRecordedJobQueue, ObservatoryRecordedQueueError, + RecordedRunDefinitionRegistry, ) -from k1link.observatory.source_admission import RecordedK1SourceAdmissionService from k1link.sessions import ( MaterializedRecording, RecordedCameraFrameService, @@ -73,6 +91,7 @@ from k1link.sessions import ( SessionRecordingPreparationManager, SessionStore, ) +from k1link.sessions.models import SessionSummary from k1link.simulation.projects import SimulationProjectService, SimulationProjectStore from k1link.web.advanced_laboratory_api import build_advanced_laboratory_router from k1link.web.artifact_health_api import build_artifact_health_router @@ -261,6 +280,55 @@ plugin_catalog: DevicePluginCatalog = plugin_environment.catalog plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher session_store = SessionStore(REPOSITORY_ROOT) +OBSERVATORY_PORTABLE_DEFINITION_REGISTRY: PortableRunDefinitionRegistry | None +OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR: str | None +OBSERVATORY_PORTABLE_CALCULATION_PROFILES: PortableCalculationProfileRegistry | None +OBSERVATORY_PORTABLE_RESULT_VALIDATORS: PortableResultContractValidatorRegistry | None +try: + OBSERVATORY_PORTABLE_DEFINITION_REGISTRY = PortableRunDefinitionRegistry.from_file( + REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" + ) + OBSERVATORY_PORTABLE_CALCULATION_PROFILES = portable_calculation_profile_registry( + OBSERVATORY_PORTABLE_DEFINITION_REGISTRY + ) + OBSERVATORY_PORTABLE_RESULT_VALIDATORS = portable_result_validator_registry( + OBSERVATORY_PORTABLE_DEFINITION_REGISTRY + ) + OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR = None +except ( + PortableRunDefinitionRegistryError, + PortableWorkerIntegrationError, + OSError, + ValueError, +) as exc: + # Portable definitions are an optional observation-only slice. Registry + # drift cannot affect K1, Simulation, legacy LAB, or the exact M49 binding. + OBSERVATORY_PORTABLE_DEFINITION_REGISTRY = None + OBSERVATORY_PORTABLE_CALCULATION_PROFILES = None + OBSERVATORY_PORTABLE_RESULT_VALIDATORS = None + OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR = str(exc) + + +def _resolve_observatory_calculation_profile( + summary: SessionSummary, +) -> dict[str, object] | None: + if OBSERVATORY_LABORATORY_SETUP_REGISTRY is not None: + legacy = OBSERVATORY_LABORATORY_SETUP_REGISTRY.observatory_calculation_profile( + summary + ) + if legacy is not None: + return legacy + if ( + OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None + or OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None + ): + return None + return resolve_published_portable_calculation_profile( + summary, + definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY, + calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES, + ) + def _load_optional_observatory_worker_authentication( recorded_job_queue: ObservatoryRecordedJobQueue | None, @@ -299,9 +367,14 @@ try: REPOSITORY_ROOT / "config" / "observatory-m49-recorded-queue-binding.json" ), ) + recorded_definitions = list(OBSERVATORY_RECORDED_BINDING_SERVICE.definitions.definitions) + if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is not None: + recorded_definitions.extend( + OBSERVATORY_PORTABLE_DEFINITION_REGISTRY.ready_recorded_definitions() + ) OBSERVATORY_RECORDED_JOB_QUEUE = ObservatoryRecordedJobQueue( session_store.data_dir, - definitions=OBSERVATORY_RECORDED_BINDING_SERVICE.definitions, + definitions=RecordedRunDefinitionRegistry(tuple(recorded_definitions)), ) OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = None except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError) as exc: @@ -316,11 +389,14 @@ OBSERVATORY_WORKER_CLAIM_LEASE_READY = False OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = False OBSERVATORY_WORKER_PRODUCTION_API_ENABLED = False OBSERVATORY_WORKER_AUTHENTICATION: ObservatoryWorkerAuthentication | None +OBSERVATORY_WORKER_AUTHENTICATION_ERROR: str | None OBSERVATORY_WORKER_API_ERROR: str | None -OBSERVATORY_WORKER_AUTHENTICATION = None -OBSERVATORY_WORKER_API_ERROR = ( - "Worker pull API is hard-disabled until claim leases and a verified " - "Observatory result publisher are implemented and accepted" +( + OBSERVATORY_WORKER_AUTHENTICATION, + OBSERVATORY_WORKER_AUTHENTICATION_ERROR, +) = _load_optional_observatory_worker_authentication( + OBSERVATORY_RECORDED_JOB_QUEUE, + token_path=OBSERVATORY_WORKER_TOKEN_PATH, ) simulation_project_store = SimulationProjectStore(session_store.data_dir) simulation_project_service = SimulationProjectService(simulation_project_store) @@ -336,37 +412,122 @@ session_recording_materializer = SessionRecordingMaterializer( session_recorded_media_inspector = RecordedMediaInspector( session_store.data_dir / "recorded-media-preparations" ) -OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableLabV1SetupProjector | None +OBSERVATORY_PORTABLE_WORKER_INTEGRATION: PortableObservatoryWorkerIntegration | None +OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR: str | None +OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS: PortableWorkerStorageRoots | None = None +try: + if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None: + raise PortableWorkerIntegrationError( + OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR + or "portable definition registry is unavailable" + ) + if OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None: + raise PortableWorkerIntegrationError( + "portable calculation profile registry is unavailable" + ) + if OBSERVATORY_PORTABLE_RESULT_VALIDATORS is None: + raise PortableWorkerIntegrationError( + "portable result validator registry is unavailable" + ) + if OBSERVATORY_RECORDED_JOB_QUEUE is None: + raise PortableWorkerIntegrationError( + OBSERVATORY_RECORDED_JOB_QUEUE_ERROR + or "Observatory recorded-job queue is unavailable" + ) + if session_artifact_gateway is None: + raise PortableWorkerIntegrationError( + "central artifact store is not configured" + ) + if session_artifact_gateway.status().central_status != "ready": + raise PortableWorkerIntegrationError( + "central artifact store is unavailable" + ) + OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS = ( + PortableWorkerStorageRoots.from_environment( + artifact_store_root=session_artifact_gateway.store.root, + ) + ) + OBSERVATORY_PORTABLE_WORKER_INTEGRATION = ( + build_portable_observatory_worker_integration( + queue=OBSERVATORY_RECORDED_JOB_QUEUE, + session_store=session_store, + media_inspector=session_recorded_media_inspector, + definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY, + artifact_store=session_artifact_gateway.store, + calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES, + validators=OBSERVATORY_PORTABLE_RESULT_VALIDATORS, + source_cas_root=( + OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.source_cas_root + ), + result_staging_root=( + OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.result_staging_root + ), + ) + ) + OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = None +except (PortableWorkerIntegrationError, OSError, ValueError) as exc: + # Constructing this dormant foundation does not enable the Worker router. + # Failure remains isolated from K1, Simulation and legacy LAB. + OBSERVATORY_PORTABLE_WORKER_INTEGRATION = None + OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = str(exc) +OBSERVATORY_WORKER_API_ERROR = ( + "Worker pull API is hard-disabled pending sealed installed executors, " + "a configured Worker credential, explicit integration acceptance and the " + "production gate" + + ( + "" + if OBSERVATORY_WORKER_AUTHENTICATION_ERROR is None + else ( + "; authentication unavailable: " + f"{OBSERVATORY_WORKER_AUTHENTICATION_ERROR}" + ) + ) + + ( + "" + if OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR is None + else f"; integration unavailable: {OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR}" + ) +) +OBSERVATORY_WORKER_DISPATCH_READY = ( + OBSERVATORY_WORKER_PRODUCTION_API_ENABLED + and OBSERVATORY_WORKER_CLAIM_LEASE_READY + and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY + and OBSERVATORY_RECORDED_JOB_QUEUE is not None + and OBSERVATORY_WORKER_AUTHENTICATION is not None + and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None +) +OBSERVATORY_PORTABLE_BINDING_SERVICE: PortableRecordedQueueBindingService | None +OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableSetupProjector | None OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None try: - portable_definition_registry = PortableRunDefinitionRegistry.from_file( - REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" - ) - portable_lab_v1_definition = next( - definition - for definition in portable_definition_registry.definitions - if definition.setup_id == PORTABLE_LAB_V1_SETUP_ID - ) - portable_source_capability_service = RecordedK1SourceAdmissionService( + if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None: + raise PortableSetupProjectionError( + OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR + or "portable definition registry is unavailable" + ) + OBSERVATORY_PORTABLE_BINDING_SERVICE = PortableRecordedQueueBindingService( data_dir=session_store.data_dir, session_store=session_store, media_inspector=session_recorded_media_inspector, - requirements=portable_lab_v1_definition.to_source_admission_requirements(), + definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY, + queue=OBSERVATORY_RECORDED_JOB_QUEUE, ) - OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableLabV1SetupProjector( - registry=portable_definition_registry, - capability_probe=portable_source_capability_service, + OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableSetupProjector( + registry=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY, + capability_probe=OBSERVATORY_PORTABLE_BINDING_SERVICE, + dispatch_available=OBSERVATORY_WORKER_DISPATCH_READY, ) OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None except ( + PortableQueueBindingError, PortableRunDefinitionRegistryError, PortableSetupProjectionError, OSError, - StopIteration, ValueError, ) as exc: - # Portable LAB V1 is an optional observation-only slice. A drifted + # Portable setup execution is an optional observation-only slice. A drifted # registry cannot affect K1, Simulation, legacy LAB, or the exact M49 queue. + OBSERVATORY_PORTABLE_BINDING_SERVICE = None OBSERVATORY_PORTABLE_SETUP_PROJECTOR = None OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = str(exc) _ffmpeg = _resolve_media_tool("ffmpeg") @@ -829,6 +990,14 @@ app.include_router( perception_overlay_provider=session_perception_overlay_store, perception_media_provider=session_perception_epoch_store, point_color_renderers=plugin_environment.point_color_renderers, + lab_calculation_profile_resolver=( + None + if ( + OBSERVATORY_LABORATORY_SETUP_REGISTRY is None + and OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None + ) + else _resolve_observatory_calculation_profile + ), ) ) app.include_router( @@ -843,19 +1012,23 @@ app.include_router( recorded_job_queue_error=OBSERVATORY_RECORDED_JOB_QUEUE_ERROR, portable_setup_projector=OBSERVATORY_PORTABLE_SETUP_PROJECTOR, portable_setup_projector_error=OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR, + portable_binding_service=OBSERVATORY_PORTABLE_BINDING_SERVICE, ) ) -if ( - OBSERVATORY_WORKER_PRODUCTION_API_ENABLED - and OBSERVATORY_WORKER_CLAIM_LEASE_READY - and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY - and OBSERVATORY_RECORDED_JOB_QUEUE is not None - and OBSERVATORY_WORKER_AUTHENTICATION is not None -): +if OBSERVATORY_WORKER_DISPATCH_READY: + assert OBSERVATORY_RECORDED_JOB_QUEUE is not None + assert OBSERVATORY_WORKER_AUTHENTICATION is not None + assert OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None app.include_router( build_observatory_worker_router( OBSERVATORY_RECORDED_JOB_QUEUE, authentication=OBSERVATORY_WORKER_AUTHENTICATION, + artifact_transport=( + OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport + ), + result_publisher=( + OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher + ), ) ) app.include_router( diff --git a/src/k1link/web/observatory_api.py b/src/k1link/web/observatory_api.py index 67d91ee..f416f39 100644 --- a/src/k1link/web/observatory_api.py +++ b/src/k1link/web/observatory_api.py @@ -22,9 +22,19 @@ from k1link.observatory.m49_queue_binding import ( M49QueueBindingIntegrityError, M49RecordedQueueBindingService, ) +from k1link.observatory.portable_queue_binding import ( + PortableQueueBindingError, + PortableQueueBindingIntegrityError, + PortableQueueBindingStaleCheckError, + PortableRecordedQueueBindingService, +) +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinitionUnavailableError, +) from k1link.observatory.portable_setup_projection import ( PortableLabV1SetupProjector, PortableSetupProjectionError, + PortableSetupProjector, ) from k1link.observatory.recorded_jobs import ( ObservatoryRecordedJobQueue, @@ -33,6 +43,7 @@ from k1link.observatory.recorded_jobs import ( ObservatoryRecordedQueueError, ObservatoryRecordedQueueNotFoundError, ) +from k1link.observatory.source_admission import PortableSourceAdmissionError from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore from k1link.sessions.models import SessionSummary @@ -130,6 +141,8 @@ class ObservatoryRecordedRunSubmitRequest(_StrictApiModel): max_length=96, pattern=r"^[a-z][a-z0-9-]{2,95}$", ) + definition_sha256: str | None = Field(default=None, pattern=r"^[a-f0-9]{64}$") + check_sha256: str | None = Field(default=None, pattern=r"^[a-f0-9]{64}$") def build_observatory_router( @@ -142,8 +155,9 @@ def build_observatory_router( recorded_binding_service: M49RecordedQueueBindingService | None = None, recorded_job_queue: ObservatoryRecordedJobQueue | None = None, recorded_job_queue_error: str | None = None, - portable_setup_projector: PortableLabV1SetupProjector | None = None, + portable_setup_projector: PortableSetupProjector | PortableLabV1SetupProjector | None = None, portable_setup_projector_error: str | None = None, + portable_binding_service: PortableRecordedQueueBindingService | None = None, ) -> APIRouter: """Build bounded catalog-only mutations for typed Observatory projections.""" @@ -214,6 +228,126 @@ def build_observatory_router( available.add(result_id) return frozenset(available) + def portable_run_preflight( + source: SessionSummary, + request: ObservatoryRunPreflightRequest, + ) -> dict[str, object] | None: + projector = portable_setup_projector + if projector is None or not projector.has_setup(request.setup_id): + return None + try: + projected = projector.project(source, setup_id=request.setup_id) + except PortableSetupProjectionError as exc: + raise HTTPException( + status_code=503, + detail="Portable-каталог сетапов нарушил контракт целостности.", + ) from exc + definition = projected.get("run_definition") + compatibility = projected.get("source_compatibility") + executor = projected.get("executor") + if ( + not isinstance(definition, dict) + or not isinstance(compatibility, dict) + or not isinstance(executor, dict) + ): + raise HTTPException( + status_code=503, + detail="Portable-каталог сетапов нарушил контракт целостности.", + ) + expected_digest = definition.get("definition_sha256") + if request.definition_sha256 != expected_digest: + raise HTTPException( + status_code=409, + detail="Идентичность RunDefinition изменилась; обновите каталог.", + ) + compatible = compatibility.get("compatible") is True + executor_ready = executor.get("state") == "ready" and executor.get("ready") is True + checked = None + check_reason: str | None = None + if ( + compatible + and executor_ready + and portable_binding_service is not None + and recorded_job_queue is not None + and isinstance(expected_digest, str) + ): + try: + checked = portable_binding_service.check( + source_session_id=request.source_session_id, + setup_id=request.setup_id, + definition_sha256=expected_digest, + ) + except PortableRunDefinitionUnavailableError as exc: + check_reason = str(exc) + except (PortableQueueBindingError, PortableSourceAdmissionError, ValueError): + check_reason = ( + "Источник или исполняемый portable-релиз не прошёл проверку целостности." + ) + elif compatible and executor_ready: + check_reason = "Portable dispatch-контур или durable-очередь недоступны." + check_sha256 = None if checked is None else checked.check_sha256 + queueable = check_sha256 is not None + checks: list[dict[str, Any]] = [ + { + "check_id": "source-compatibility", + "outcome": "pass" if compatible else "fail", + "reason_code": "source-compatible" if compatible else "source-incompatible", + "message": str(compatibility.get("reason")), + }, + { + "check_id": "executor", + "outcome": "pass" if executor_ready else "fail", + "reason_code": ( + "executor-release-sealed" + if executor_ready + else str(executor.get("reason_code")) + ), + "message": ( + "Исполняемый portable-релиз и image запечатаны." + if executor_ready + else str(executor.get("reason")) + ), + }, + { + "check_id": "definition-check", + "outcome": "pass" if checked is not None else "fail", + "reason_code": ( + "portable-check-sealed" if checked is not None else "portable-check-unavailable" + ), + "message": ( + "Источник и RunDefinition связаны одноразовым check SHA." + if checked is not None + else check_reason + or "Portable-проверка недоступна до установки executor-релиза." + ), + }, + { + "check_id": "durable-queue", + "outcome": "pass" if queueable else "fail", + "reason_code": ( + "durable-queue-ready" if queueable else "durable-queue-unavailable" + ), + "message": ( + "Durable-очередь готова принять расчёт по check SHA." + if queueable + else "Расчёт нельзя поставить в очередь." + ), + }, + ] + return { + "schema_version": OBSERVATORY_RUN_PREFLIGHT_SCHEMA, + "source_session_id": request.source_session_id, + "setup_id": request.setup_id, + "definition_sha256": expected_digest, + "check_sha256": check_sha256, + "outcome": "queueable" if queueable else "blocked", + "submission_allowed": queueable, + "checks": checks, + "existing_result_ids": [], + "executor": executor, + "authority": projected.get("authority", dict(_OBSERVATION_ONLY_AUTHORITY)), + } + if portable_setup_projector is not None: @router.get("/api/v1/observatory/portable-laboratory-setups") @@ -230,7 +364,7 @@ def build_observatory_router( except PortableSetupProjectionError as exc: raise HTTPException( status_code=503, - detail="Portable-каталог LAB V1 нарушил контракт целостности.", + detail="Portable-каталог сетапов нарушил контракт целостности.", ) from exc elif portable_setup_projector_error is not None: @@ -246,10 +380,10 @@ def build_observatory_router( del source_session_id raise HTTPException( status_code=503, - detail="Portable-каталог LAB V1 недоступен.", + detail="Portable-каталог сетапов недоступен.", ) - if setup_registry is not None: + if setup_registry is not None or portable_setup_projector is not None: @router.get("/api/v1/observatory/laboratory-setups") def list_observatory_laboratory_setups( @@ -259,6 +393,11 @@ def build_observatory_router( pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$", ), ) -> dict[str, object]: + if setup_registry is None: + raise HTTPException( + status_code=503, + detail="Каталог legacy-сетапов Обсерватории недоступен.", + ) source = source_summary(source_session_id) return setup_registry.catalog( source, @@ -270,6 +409,11 @@ def build_observatory_router( request: ObservatoryRunPreflightRequest, ) -> dict[str, object]: source = source_summary(request.source_session_id) + portable = portable_run_preflight(source, request) + if portable is not None: + return portable + if setup_registry is None: + raise HTTPException(status_code=404, detail="Сетап лаборатории не найден.") try: setup_registry.setup(request.setup_id) except KeyError as exc: @@ -639,16 +783,18 @@ def build_observatory_router( detail="Подготовка расчётов Обсерватории недоступна.", ) - if ( - setup_registry is not None - and recorded_binding_service is not None - and recorded_job_queue is not None + if recorded_job_queue is not None and ( + recorded_binding_service is not None or portable_binding_service is not None ): @router.post("/api/v1/observatory/runs", status_code=202) def submit_observatory_recorded_run( request: ObservatoryRecordedRunSubmitRequest, ) -> dict[str, object]: + portable_request = ( + portable_setup_projector is not None + and portable_setup_projector.has_setup(request.setup_id) + ) try: existing_job = recorded_job_queue.get_by_idempotency_key(request.idempotency_key) except ObservatoryRecordedQueueNotFoundError: @@ -662,6 +808,10 @@ def build_observatory_router( if ( existing_job.source_session_id != request.source_session_id or existing_job.setup_id != request.setup_id + or ( + portable_request + and existing_job.definition_sha256 != request.definition_sha256 + ) ): raise HTTPException( status_code=409, @@ -670,6 +820,113 @@ def build_observatory_router( return existing_job.as_dict() source = source_summary(request.source_session_id) + if portable_request: + if portable_binding_service is None: + raise HTTPException( + status_code=503, + detail="Portable dispatch-контур недоступен.", + ) + if request.definition_sha256 is None or request.check_sha256 is None: + raise HTTPException( + status_code=409, + detail=( + "Для portable-расчёта требуются актуальные definition SHA " + "и check SHA из preflight." + ), + ) + assert portable_setup_projector is not None + try: + portable_projection = portable_setup_projector.project( + source, + setup_id=request.setup_id, + ) + except PortableSetupProjectionError as exc: + raise HTTPException( + status_code=503, + detail="Portable-каталог сетапов нарушил контракт целостности.", + ) from exc + projected_definition = portable_projection.get("run_definition") + projected_executor = portable_projection.get("executor") + if not isinstance(projected_definition, dict) or not isinstance( + projected_executor, dict + ): + raise HTTPException( + status_code=503, + detail="Portable-каталог сетапов нарушил контракт целостности.", + ) + if projected_definition.get("definition_sha256") != request.definition_sha256: + raise HTTPException( + status_code=409, + detail="Идентичность RunDefinition изменилась; повторите preflight.", + ) + if ( + projected_executor.get("state") != "ready" + or projected_executor.get("ready") is not True + ): + raise HTTPException( + status_code=409, + detail="Portable executor-релиз не установлен.", + ) + try: + job, _created = portable_binding_service.submit( + source_session_id=request.source_session_id, + setup_id=request.setup_id, + definition_sha256=request.definition_sha256, + expected_check_sha256=request.check_sha256, + idempotency_key=request.idempotency_key, + ) + return job.as_dict() + except PortableQueueBindingStaleCheckError as exc: + raise HTTPException( + status_code=409, + detail="Источник или RunDefinition изменились; повторите preflight.", + ) from exc + except PortableRunDefinitionUnavailableError as exc: + raise HTTPException( + status_code=409, + detail="Portable executor-релиз не установлен.", + ) from exc + except ( + PortableQueueBindingIntegrityError, + PortableSourceAdmissionError, + ) as exc: + raise HTTPException( + status_code=409, + detail="Portable-привязка источника не прошла проверку целостности.", + ) from exc + except ObservatoryRecordedQueueConflictError as exc: + try: + raced = recorded_job_queue.get_by_idempotency_key(request.idempotency_key) + except ObservatoryRecordedQueueNotFoundError: + raced = None + if ( + raced is not None + and raced.source_session_id == request.source_session_id + and raced.setup_id == request.setup_id + and raced.definition_sha256 == request.definition_sha256 + ): + return raced.as_dict() + raise HTTPException( + status_code=409, + detail="Ключ идемпотентности уже связан с другим расчётом.", + ) from exc + except ObservatoryRecordedQueueCapacityError as exc: + raise HTTPException( + status_code=503, + detail="Квота durable-очереди расчётов исчерпана.", + ) from exc + except ( + PortableQueueBindingError, + ObservatoryRecordedQueueError, + ValueError, + ) as exc: + raise HTTPException( + status_code=503, + detail="Portable dispatch-контур недоступен.", + ) from exc + + if setup_registry is None or recorded_binding_service is None: + raise HTTPException(status_code=404, detail="Сетап лаборатории не найден.") try: setup_registry.setup(request.setup_id) except KeyError as exc: diff --git a/src/k1link/web/observatory_worker_api.py b/src/k1link/web/observatory_worker_api.py index 5c12304..8878923 100644 --- a/src/k1link/web/observatory_worker_api.py +++ b/src/k1link/web/observatory_worker_api.py @@ -18,11 +18,23 @@ from dataclasses import dataclass from pathlib import Path from typing import Annotated, Final, Literal -from fastapi import APIRouter, Depends, Header, HTTPException, Response +from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response from fastapi import Path as ApiPath +from fastapi.responses import FileResponse from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from pydantic import BaseModel, ConfigDict, Field +from k1link.observatory.portable_artifact_transport import ( + MAX_RESULT_MANIFEST_BYTES, + PortableArtifactTransportError, + PortableArtifactTransportIntegrityError, + PortableArtifactTransportUnavailableError, + PortableObservatoryArtifactTransport, +) +from k1link.observatory.portable_result_contract import PortableResultPublisherError +from k1link.observatory.portable_result_publisher import ( + PortableObservatoryResultPublisher, +) from k1link.observatory.recorded_jobs import ( ObservatoryRecordedCheckpointError, ObservatoryRecordedJobQueue, @@ -38,6 +50,7 @@ from k1link.observatory.recorded_jobs import ( OBSERVATORY_WORKER_CLAIM_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-claim-request/v1" OBSERVATORY_WORKER_START_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-start-request/v1" +OBSERVATORY_WORKER_RENEW_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-renew-request/v1" OBSERVATORY_WORKER_CHECKPOINT_REQUEST_SCHEMA: Final = ( "missioncore.observatory-worker-checkpoint-request/v1" ) @@ -46,6 +59,10 @@ OBSERVATORY_WORKER_SUCCEED_REQUEST_SCHEMA: Final = ( ) OBSERVATORY_WORKER_FAIL_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-fail-request/v1" OBSERVATORY_WORKER_CONTOUR_HEADER: Final = "X-Mission-Core-Contour-Id" +OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER: Final = "X-Mission-Core-Claim-Token" +OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER: Final = ( + "X-Mission-Core-Claim-Generation" +) _IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$") _SHA256 = re.compile(r"^[a-f0-9]{64}$") @@ -138,6 +155,13 @@ class ObservatoryWorkerStartRequest(_StrictWorkerRequest): claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN) +class ObservatoryWorkerRenewRequest(_StrictWorkerRequest): + schema_version: Literal["missioncore.observatory-worker-renew-request/v1"] + claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN) + claim_generation: int = Field(ge=1) + heartbeat_sequence: int = Field(ge=1) + + class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest): schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"] claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN) @@ -174,6 +198,8 @@ def build_observatory_worker_router( queue: ObservatoryRecordedJobQueue, *, authentication: ObservatoryWorkerAuthentication, + artifact_transport: PortableObservatoryArtifactTransport | None = None, + result_publisher: PortableObservatoryResultPublisher | None = None, ) -> APIRouter: """Build the bounded Worker pull/state-transition router. @@ -182,6 +208,9 @@ def build_observatory_worker_router( hashed, and is compared to the configured digest in constant time. """ + if result_publisher is not None and artifact_transport is None: + raise ValueError("portable result publisher requires artifact transport") + def require_configured_worker( credentials: Annotated[ HTTPAuthorizationCredentials | None, @@ -245,6 +274,20 @@ def build_observatory_worker_router( ) -> dict[str, object]: return _queue_call(lambda: queue.start(job_id, claim_token=request.claim_token)).as_dict() + @router.post("/recorded-jobs/{job_id}/lease/renew") + def renew_job_claim( + request: ObservatoryWorkerRenewRequest, + job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)], + ) -> dict[str, object]: + return _queue_call( + lambda: queue.renew_claim( + job_id, + claim_token=request.claim_token, + claim_generation=request.claim_generation, + heartbeat_sequence=request.heartbeat_sequence, + ) + ).as_dict() + @router.post("/recorded-jobs/{job_id}/checkpoint") def checkpoint_job( request: ObservatoryWorkerCheckpointRequest, @@ -263,14 +306,39 @@ def build_observatory_worker_router( request: ObservatoryWorkerSucceedRequest, job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)], ) -> dict[str, object]: - return _queue_call( + if artifact_transport is not None: + _artifact_call( + lambda: artifact_transport.require_completed_for_success( + job_id=job_id, + result_id=request.result_id, + result_sha256=request.result_sha256, + claim_token=request.claim_token, + claimant_id=authentication.contour_id, + ) + ) + succeeded = _queue_call( lambda: queue.succeed( job_id, claim_token=request.claim_token, result_id=request.result_id, result_sha256=request.result_sha256, ) - ).as_dict() + ) + if artifact_transport is not None and result_publisher is not None: + package_root = _artifact_call( + lambda: artifact_transport.package_root_for_terminal(succeeded) + ) + try: + result_publisher.publish(job=succeeded, package_root=package_root) + except PortableResultPublisherError as exc: + raise HTTPException( + status_code=503, + detail=( + "Recorded result is sealed but its verified publication " + "requires reconciliation." + ), + ) from exc + return succeeded.as_dict() @router.post("/recorded-jobs/{job_id}/fail") def fail_job( @@ -286,6 +354,164 @@ def build_observatory_worker_router( ) ).as_dict() + if artifact_transport is not None: + + @router.get("/recorded-jobs/{job_id}/source-materialization") + def source_materialization( + job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)], + claim_token: Annotated[ + str, + Header( + alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER, + pattern=_CLAIM_TOKEN_PATTERN, + ), + ], + claim_generation: Annotated[ + int, + Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1), + ], + ) -> dict[str, object]: + return _artifact_call( + lambda: artifact_transport.source_manifest( + job_id=job_id, + claim_token=claim_token, + claim_generation=claim_generation, + claimant_id=authentication.contour_id, + ) + .as_dict() + ) + + @router.get("/recorded-jobs/{job_id}/source-members/{member_id}") + def source_member( + job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)], + member_id: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")], + claim_token: Annotated[ + str, + Header( + alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER, + pattern=_CLAIM_TOKEN_PATTERN, + ), + ], + claim_generation: Annotated[ + int, + Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1), + ], + ) -> FileResponse: + member, path = _artifact_call( + lambda: artifact_transport.materialize_source_member( + job_id=job_id, + member_id=member_id, + claim_token=claim_token, + claim_generation=claim_generation, + claimant_id=authentication.contour_id, + ) + ) + return FileResponse( + path, + media_type=member.media_type, + headers={ + "ETag": f'"{member.sha256}"', + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + "X-Mission-Core-Content-Sha256": member.sha256, + }, + ) + + @router.put( + "/recorded-jobs/{job_id}/result-packages/{result_sha256}/manifest" + ) + async def stage_result_manifest( + request: Request, + job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)], + result_sha256: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")], + claim_token: Annotated[ + str, + Header( + alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER, + pattern=_CLAIM_TOKEN_PATTERN, + ), + ], + claim_generation: Annotated[ + int, + Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1), + ], + ) -> dict[str, object]: + payload = await _read_bounded_body(request, MAX_RESULT_MANIFEST_BYTES) + return _artifact_call( + lambda: artifact_transport.stage_result_manifest( + job_id=job_id, + result_sha256=result_sha256, + manifest_payload=payload, + claim_token=claim_token, + claim_generation=claim_generation, + claimant_id=authentication.contour_id, + ) + .as_dict() + ) + + @router.put( + "/recorded-jobs/{job_id}/result-packages/{result_sha256}/members/{member_id}" + ) + async def upload_result_member( + request: Request, + job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)], + result_sha256: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")], + member_id: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")], + claim_token: Annotated[ + str, + Header( + alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER, + pattern=_CLAIM_TOKEN_PATTERN, + ), + ], + claim_generation: Annotated[ + int, + Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1), + ], + ) -> dict[str, object]: + try: + plan = await artifact_transport.upload_result_member( + job_id=job_id, + result_sha256=result_sha256, + member_id=member_id, + chunks=request.stream(), + claim_token=claim_token, + claim_generation=claim_generation, + claimant_id=authentication.contour_id, + ) + except Exception as exc: + _raise_artifact_or_queue_error(exc) + return plan.as_dict() + + @router.post( + "/recorded-jobs/{job_id}/result-packages/{result_sha256}/complete" + ) + def complete_result_package( + job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)], + result_sha256: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")], + claim_token: Annotated[ + str, + Header( + alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER, + pattern=_CLAIM_TOKEN_PATTERN, + ), + ], + claim_generation: Annotated[ + int, + Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1), + ], + ) -> dict[str, object]: + return _artifact_call( + lambda: artifact_transport.complete_result_upload( + job_id=job_id, + result_sha256=result_sha256, + claim_token=claim_token, + claim_generation=claim_generation, + claimant_id=authentication.contour_id, + ) + .as_dict() + ) + return router @@ -341,3 +567,66 @@ def _queue_call[T](operation: Callable[[], T]) -> T: status_code=503, detail="Recorded-job queue is unavailable.", ) from exc + + +def _artifact_call[T](operation: Callable[[], T]) -> T: + try: + return _queue_call(operation) + except PortableArtifactTransportIntegrityError as exc: + raise HTTPException( + status_code=409, + detail="Worker artifact identity was rejected.", + ) from exc + except PortableArtifactTransportUnavailableError as exc: + raise HTTPException( + status_code=409, + detail="Worker artifact member is unavailable for this claim.", + ) from exc + except PortableArtifactTransportError as exc: + raise HTTPException( + status_code=503, + detail="Worker artifact transport is unavailable.", + ) from exc + + +def _raise_artifact_or_queue_error(exc: Exception) -> None: + if isinstance(exc, PortableArtifactTransportIntegrityError): + raise HTTPException( + status_code=409, + detail="Worker artifact identity was rejected.", + ) from exc + if isinstance(exc, PortableArtifactTransportUnavailableError): + raise HTTPException( + status_code=409, + detail="Worker artifact member is unavailable for this claim.", + ) from exc + if isinstance(exc, PortableArtifactTransportError): + raise HTTPException( + status_code=503, + detail="Worker artifact transport is unavailable.", + ) from exc + _queue_call(lambda: _raise(exc)) + raise AssertionError("unreachable") + + +def _raise(exc: Exception) -> None: + raise exc + + +async def _read_bounded_body(request: Request, maximum_bytes: int) -> bytes: + content_length = request.headers.get("content-length") + if content_length is not None: + try: + declared = int(content_length) + except ValueError as exc: + raise HTTPException(status_code=400, detail="Content-Length is invalid.") from exc + if declared < 1 or declared > maximum_bytes: + raise HTTPException(status_code=413, detail="Request body is outside bounds.") + payload = bytearray() + async for chunk in request.stream(): + payload.extend(chunk) + if len(payload) > maximum_bytes: + raise HTTPException(status_code=413, detail="Request body is outside bounds.") + if not payload: + raise HTTPException(status_code=400, detail="Request body is empty.") + return bytes(payload) diff --git a/src/k1link/web/session_api.py b/src/k1link/web/session_api.py index ffc8f73..7d8d48c 100644 --- a/src/k1link/web/session_api.py +++ b/src/k1link/web/session_api.py @@ -41,6 +41,7 @@ from k1link.sessions.canonical_lab_spatial import ( CANONICAL_LAB_SPATIAL_PROFILE, canonical_lab_spatial_frame, ) +from k1link.sessions.models import SessionSummary from k1link.sessions.plugin_contract import RecordedPointColorRenderer from k1link.viewer.recorded import ( APPLICATION_ID as RECORDED_APPLICATION_ID, @@ -328,6 +329,9 @@ def build_session_router( perception_overlay_provider: RecordedPerceptionOverlayProvider | None = None, perception_media_provider: RecordedPerceptionMediaProvider | None = None, point_color_renderers: Mapping[str, RecordedPointColorRenderer] | None = None, + lab_calculation_profile_resolver: ( + Callable[[SessionSummary], Mapping[str, object] | None] | None + ) = None, allow_synchronous_recording_fallback: bool = False, replay_action_id: str = DEFAULT_REPLAY_ACTION_ID, ) -> APIRouter: @@ -336,12 +340,29 @@ def build_session_router( router = APIRouter(tags=["observation-sessions"]) recorded_media_inspector = media_inspector or RecordedMediaInspector() + def lab_catalog_document( + summary: SessionSummary, + contract: Literal["v1", "v2", "v3"], + ) -> dict[str, Any]: + lab = summary.lab + if lab is None: + raise ValueError("LAB catalog document requires a LAB summary") + document = lab.as_dict(include_replay_capability=contract in ("v2", "v3")) + if contract == "v3": + profile = ( + None + if lab_calculation_profile_resolver is None + else lab_calculation_profile_resolver(summary) + ) + document["calculation_profile"] = None if profile is None else dict(profile) + return document + @router.get("/api/v1/observation-sessions") def list_observation_sessions( limit: int = Query(default=20, ge=1, le=100), cursor: str | None = Query(default=None, max_length=128), scope: Literal["all", "source", "laboratory"] = "all", - lab_contract: Literal["v1", "v2"] = "v1", + lab_contract: Literal["v1", "v2", "v3"] = "v1", ) -> dict[str, Any]: try: _refresh_catalog(catalog_refresher) @@ -349,7 +370,7 @@ def build_session_router( limit=limit, cursor=cursor, scope=scope, - include_capability_projections=lab_contract == "v2", + include_capability_projections=lab_contract in ("v2", "v3"), ) return { "items": [ @@ -364,9 +385,7 @@ def build_session_router( "replayable": item.replayable, **( { - "lab": item.lab.as_dict( - include_replay_capability=lab_contract == "v2" - ) + "lab": lab_catalog_document(item, lab_contract) } if item.lab is not None else {} diff --git a/tests/test_m49_portable_executor_release.py b/tests/test_m49_portable_executor_release.py new file mode 100644 index 0000000..d08ea42 --- /dev/null +++ b/tests/test_m49_portable_executor_release.py @@ -0,0 +1,836 @@ +from __future__ import annotations + +import copy +import hashlib +import importlib.util +import json +import shutil +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import cast + +import numpy as np +import pytest + +import k1link.observatory.m49_portable_source as source_module +from k1link.observatory.m49_portable_executor import ( + M49_PORTABLE_RUNTIME_PHASES, + M49PortableBoundSourceStage, + M49PortableProfileRunnerAdapter, + M49PortableRunnerInstallation, + M49PortableSourceMaterializerAdapter, +) +from k1link.observatory.m49_portable_result import ( + M49PortableResultError, + validate_m49_portable_result, +) +from k1link.observatory.m49_portable_source import ( + M49PortableSourceError, + M49PortableSourceIdentity, + materialize_m49_portable_source, + read_m49_source_index, + validate_m49_portable_source_stage, +) +from k1link.observatory.portable_result_contract import ( + PortableResultPackageManifest, + PortableResultValidationContext, + canonical_json, +) +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinition, + PortableRunDefinitionRegistry, + canonical_sha256, +) +from k1link.observatory.portable_worker_runtime import ( + PortableWorkerRuntimePlan, + PortableWorkerSourceStage, +) +from k1link.observatory.recorded_jobs import ( + ObservatoryRecordedJob, + ObservatoryRecordedJobIntent, + ObservatoryRecordedJobQueue, + RecordedRunDefinitionRegistry, +) +from k1link.observatory.source_admission import ( + PORTABLE_SOURCE_BUNDLE_SCHEMA, + PORTABLE_SOURCE_CAPABILITY_SCHEMA, +) +from k1link.observatory.worker_agent import ( + ObservatoryWorkerExecutorIdentity, + SealedObservatoryRecordedJob, +) + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" +PROFILE_PATH = REPOSITORY_ROOT / "config" / "perception" / "m49-tgs-portable-v2.json" +RUNNER_PATH = ( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "observatory_portable" + / "run_m49_tgs_portable.cpp" +) +BUILDER_PATH = REPOSITORY_ROOT / "scripts" / "build_m49_portable_executor_release.py" +DOCKERFILE_PATH = ( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "observatory_portable" + / "Dockerfile.m49-portable-executor" +) +INSTALLER_PATH = ( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "observatory_portable" + / "Invoke-M49PortableExecutorCandidateInstall.ps1" +) +NOW = "2026-08-31T09:00:00.000Z" +SESSION_ID = "20260831T085500Z_viewer_live" +AUTHORITY = { + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, + "production_accepted": False, +} +RAW_PAYLOAD = b"exact-k1-raw-replay" +METADATA_PAYLOAD = b'{"received_monotonic_ns":1}\n' + +_builder_spec = importlib.util.spec_from_file_location( + "build_m49_portable_executor_release", BUILDER_PATH +) +assert _builder_spec is not None and _builder_spec.loader is not None +builder = importlib.util.module_from_spec(_builder_spec) +sys.modules[_builder_spec.name] = builder +_builder_spec.loader.exec_module(builder) + + +class _FakeLidarPack: + def __init__( + self, + *, + adapter_sha256: str, + raw_sha256: str, + metadata_sha256: str, + ) -> None: + del adapter_sha256 + self.pack_id = f"lidar-replay-pack-{'c' * 64}" + self.identity = { + "session_id": SESSION_ID, + "logical_content_sha256": "e" * 64, + "source_evidence": { + "raw": { + "byte_length": len(RAW_PAYLOAD), + "sha256": raw_sha256, + }, + "metadata": { + "byte_length": len(METADATA_PAYLOAD), + "sha256": metadata_sha256, + }, + }, + } + self.manifest = {"identity_sha256": "d" * 64} + self.arrays = { + "point_received_monotonic_ns": np.asarray([100_000_000, 1_100_000_000], dtype=np.int64), + "pose_received_monotonic_ns": np.asarray([50_000_000, 1_050_000_000], dtype=np.int64), + "pose_positions_map": np.zeros((2, 3), dtype=np.float64), + } + self._points = ( + np.asarray( + [[2.0, 0.0, 0.0], [3.0, 0.0, 1.0], [4.0, 0.0, 0.2]], + dtype=np.float64, + ), + np.asarray( + [[2.5, 0.0, 0.0], [3.5, 0.0, 1.0], [4.5, 0.0, 0.2]], + dtype=np.float64, + ), + ) + + def point_frame(self, index: int) -> SimpleNamespace: + return SimpleNamespace( + xyz_map=self._points[index], + intensity=np.asarray([10, 20, 30], dtype=np.uint8), + ) + + def close(self) -> None: + return None + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _m49_definition() -> PortableRunDefinition: + return PortableRunDefinitionRegistry.from_file(REGISTRY_PATH).resolve_setup( + "m49-tgs-portable-v2" + ) + + +def _materialized_source( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + *, + include_metadata_member: bool = True, +) -> tuple[Path, M49PortableSourceIdentity]: + definition = _m49_definition() + adapter_sha256 = definition.source_adapter.contract_sha256 + raw_sha256 = hashlib.sha256(RAW_PAYLOAD).hexdigest() + metadata_sha256 = hashlib.sha256(METADATA_PAYLOAD).hexdigest() + bundle = { + "schema_version": PORTABLE_SOURCE_BUNDLE_SCHEMA, + "source_session_id": SESSION_ID, + "source_catalog_sha256": "1" * 64, + "plugin_id": definition.source_requirements.plugin_id, + "archive_id": definition.source_requirements.archive_id, + "source_adapter": { + "id": definition.source_adapter.adapter_id, + "version": definition.source_adapter.version, + "sha256": adapter_sha256, + }, + "sources": [], + "spatial_replay": { + "timeline_origin_monotonic_ns": 0, + "members": [ + { + "artifact_id": "raw-transport-primary", + "media_type": "application/x-nodedc-k1mqtt", + "byte_length": len(RAW_PAYLOAD), + "replay_byte_length": len(RAW_PAYLOAD), + "sha256": raw_sha256, + }, + *( + [ + { + "artifact_id": "raw-transport-index", + "media_type": "application/x-ndjson", + "byte_length": len(METADATA_PAYLOAD), + "replay_byte_length": len(METADATA_PAYLOAD), + "sha256": metadata_sha256, + } + ] + if include_metadata_member + else [] + ), + ], + }, + "camera": { + "generation_sha256": "2" * 64, + "epoch": { + "timeline_start_seconds": 0.0, + "timeline_end_seconds": 3.0, + "segments": [ + {"sequence": 1, "end_time_seconds": 1.0}, + {"sequence": 2, "end_time_seconds": 2.0}, + {"sequence": 3, "end_time_seconds": 3.0}, + ], + }, + }, + "authority": AUTHORITY, + } + bundle_payload = canonical_json(bundle) + bundle_sha256 = hashlib.sha256(bundle_payload).hexdigest() + capability = { + "schema_version": PORTABLE_SOURCE_CAPABILITY_SCHEMA, + "source_session_id": SESSION_ID, + "source_catalog_sha256": "1" * 64, + "source_bundle_sha256": bundle_sha256, + "source_adapter_sha256": adapter_sha256, + "modalities": ["point-cloud", "trajectory", "video"], + "camera_profile": { + "generation_sha256": "2" * 64, + "frame_count": 3, + }, + "calibration": {}, + "authority": AUTHORITY, + } + capability_payload = canonical_json(capability) + capability_sha256 = hashlib.sha256(capability_payload).hexdigest() + bundle_path = tmp_path / "source-bundle.json" + capability_path = tmp_path / "source-capability.json" + bundle_path.write_bytes(bundle_payload) + capability_path.write_bytes(capability_payload) + expected = M49PortableSourceIdentity( + source_session_id=SESSION_ID, + source_catalog_sha256="1" * 64, + source_bundle_sha256=bundle_sha256, + source_capability_manifest_sha256=capability_sha256, + source_adapter_sha256=adapter_sha256, + raw_capture_sha256=raw_sha256, + metadata_sha256=metadata_sha256, + ) + fake = _FakeLidarPack( + adapter_sha256=adapter_sha256, + raw_sha256=raw_sha256, + metadata_sha256=metadata_sha256, + ) + monkeypatch.setattr(source_module, "LidarReplayPackV2", lambda _root: fake) + stage = materialize_m49_portable_source( + source_bundle_path=bundle_path, + source_capability_path=capability_path, + lidar_pack_root=tmp_path, + profile_path=PROFILE_PATH, + output_parent=tmp_path / "source-stages", + expected=expected, + ) + return stage.root, expected + + +def _ready_m49_registry(tmp_path: Path) -> PortableRunDefinitionRegistry: + base_registry = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH) + base = base_registry.resolve_setup("m49-tgs-portable-v2") + document = cast(dict[str, object], json.loads(REGISTRY_PATH.read_text(encoding="utf-8"))) + selected = copy.deepcopy( + next( + cast(dict[str, object], row) + for row in cast(list[object], document["definitions"]) + if cast(dict[str, object], row)["setup_id"] == "m49-tgs-portable-v2" + ) + ) + runner = { + "component_id": "m49-tgs-portable-runner-v1", + "kind": "runner", + "sha256": _sha256(RUNNER_PATH), + } + components = cast(list[object], selected["components"]) + components.append(runner) + components.sort(key=lambda value: cast(str, cast(dict[str, object], value)["component_id"])) + executor = { + "contour_id": "worker-006", + "state": "ready", + "release_id": "m49-tgs-portable-executor-v1", + "release_sha256": "3" * 64, + "image_sha256": "4" * 64, + "reason_code": None, + "reason": None, + } + selected["executor"] = executor + identity = copy.deepcopy(base.identity_document()) + identity["components"] = copy.deepcopy(components) + identity["executor"] = { + key: executor[key] + for key in ("contour_id", "state", "release_id", "release_sha256", "image_sha256") + } + selected["definition_sha256"] = canonical_sha256(identity) + path = tmp_path / "ready-m49-registry.json" + path.write_bytes( + canonical_json( + { + "schema_version": document["schema_version"], + "definitions": [selected], + } + ) + ) + return PortableRunDefinitionRegistry.from_file(path) + + +def _running_job( + tmp_path: Path, + definition: PortableRunDefinition, + expected: M49PortableSourceIdentity, +) -> tuple[ObservatoryRecordedJobQueue, ObservatoryRecordedJob, str]: + queue = ObservatoryRecordedJobQueue( + tmp_path / "queue", + definitions=RecordedRunDefinitionRegistry((definition.to_recorded_run_definition(),)), + clock=lambda: NOW, + ) + job, created = queue.submit( + ObservatoryRecordedJobIntent( + idempotency_key="m49-portable-executor-test-001", + source_session_id=SESSION_ID, + source_catalog_sha256=expected.source_catalog_sha256, + source_bundle_sha256=expected.source_bundle_sha256, + source_capability_manifest_sha256=(expected.source_capability_manifest_sha256), + setup_id=definition.setup_id, + definition_sha256=definition.definition_sha256, + ), + enqueue=True, + ) + assert created is True + claim = queue.claim_next(claimant_id="worker-006", claim_request_id="m49-portable-claim-001") + assert claim is not None + running = queue.start(job.job_id, claim_token=claim.claim_token) + return queue, running, claim.claim_token + + +def _sealed_worker_job(expected: M49PortableSourceIdentity) -> SealedObservatoryRecordedJob: + definition = _m49_definition() + return SealedObservatoryRecordedJob( + job_id=f"observatory-run-{'a' * 32}", + request_sha256="2" * 64, + identity_sha256="3" * 64, + submission_receipt_sha256="6" * 64, + source_session_id=SESSION_ID, + source_catalog_sha256=expected.source_catalog_sha256, + source_bundle_sha256=expected.source_bundle_sha256, + source_capability_manifest_sha256=(expected.source_capability_manifest_sha256), + source_adapter_id=definition.source_adapter.adapter_id, + source_adapter_version=definition.source_adapter.version, + source_adapter_sha256=expected.source_adapter_sha256, + setup_id=definition.setup_id, + definition_id=definition.definition_id, + definition_version=definition.version, + definition_sha256=definition.definition_sha256, + executor_release_id="m49-tgs-portable-executor-v1", + executor_identity=ObservatoryWorkerExecutorIdentity( + release_sha256="4" * 64, + image_sha256="5" * 64, + model_manifest_sha256=definition.model_manifest_sha256, + resource_profile_sha256=definition.resource_profile.profile_sha256, + ), + model_release_ids=(), + resource_profile_id=definition.resource_profile.profile_id, + checkpoint_policy=definition.resource_profile.checkpoint_policy, + allowed_checkpoints=definition.resource_profile.allowed_checkpoints, + claim_generation=1, + claim_claimed_at_utc=NOW, + claim_expires_at_utc="2026-08-31T09:05:00.000Z", + claim_heartbeat_at_utc=NOW, + claim_renewal_count=0, + restart_from_zero=False, + ) + + +def _seal_running_job(job: ObservatoryRecordedJob) -> SealedObservatoryRecordedJob: + return SealedObservatoryRecordedJob( + job_id=job.job_id, + request_sha256=job.request_sha256, + identity_sha256=job.identity_sha256, + submission_receipt_sha256=job.submission_receipt_sha256, + source_session_id=job.source_session_id, + source_catalog_sha256=job.source_catalog_sha256, + source_bundle_sha256=job.source_bundle_sha256, + source_capability_manifest_sha256=job.source_capability_manifest_sha256, + source_adapter_id=job.source_adapter_id, + source_adapter_version=job.source_adapter_version, + source_adapter_sha256=job.source_adapter_sha256, + setup_id=job.setup_id, + definition_id=job.definition_id, + definition_version=job.definition_version, + definition_sha256=job.definition_sha256, + executor_release_id=job.executor_release_id, + executor_identity=ObservatoryWorkerExecutorIdentity( + release_sha256=job.executor_release_sha256, + image_sha256=job.executor_image_sha256, + model_manifest_sha256=job.model_manifest_sha256, + resource_profile_sha256=job.resource_profile_sha256, + ), + model_release_ids=job.model_release_ids, + resource_profile_id=job.resource_profile_id, + checkpoint_policy=job.checkpoint_policy, + allowed_checkpoints=job.allowed_checkpoints, + claim_generation=job.claim_generation, + claim_claimed_at_utc=job.claimed_at_utc, + claim_expires_at_utc=job.claim_expires_at_utc, + claim_heartbeat_at_utc=job.claim_heartbeat_at_utc, + claim_renewal_count=job.claim_renewal_count, + restart_from_zero=job.restart_from_zero, + ) + + +def _worker_source_member( + job: SealedObservatoryRecordedJob, + *, + kind: str, + payload: bytes, + media_type: str, + artifact_id: str | None = None, + primary: bool = False, + camera_epoch: int | None = None, + camera_sequence: int | None = None, +) -> dict[str, object]: + sha256 = hashlib.sha256(payload).hexdigest() + identity = { + "job_identity_sha256": job.identity_sha256, + "source_bundle_sha256": job.source_bundle_sha256, + "kind": kind, + "artifact_id": artifact_id, + "primary": primary, + "camera_epoch": camera_epoch, + "camera_sequence": camera_sequence, + "media_type": media_type, + "byte_length": len(payload), + "sha256": sha256, + } + return { + "member_id": hashlib.sha256(canonical_json(identity)).hexdigest(), + "kind": kind, + "media_type": media_type, + "byte_length": len(payload), + "sha256": sha256, + "artifact_id": artifact_id, + "primary": primary, + "camera_epoch": camera_epoch, + "camera_sequence": camera_sequence, + } + + +def _runner_outputs(stage_root: Path, tmp_path: Path) -> tuple[Path, Path]: + stage = validate_m49_portable_source_stage(stage_root) + rows = read_m49_source_index( + stage.root / "sequence-index.ndjson", + expected_frame_count=stage.timeline_frame_count, + ) + output_root = tmp_path / "runner-outputs" + output_root.mkdir() + timing = [ + "timeline_frame_index\tsource_frame_index\tsession_seconds\tsample_available" + "\tavailable_slot\tinput_points\tground_points\tnonground_points\ttgs_ms" + "\tstage_wall_ms" + ] + for row in rows: + index = cast(int, row["timeline_frame_index"]) + if row["sample_available"] is not True: + timing.append( + f"{index}\t{row['source_frame_index']}\t" + f"{cast(float, row['session_seconds']):.6f}" + "\t0\t-1\t0\t0\t0\t0.000000\t0.010000" + ) + continue + source = np.fromfile(stage.root / cast(str, row["relative_path"]), dtype=" 1.0) & (ranges < 80.0)] + ground = eligible[:1] + nonground = eligible[1:2] + (output_root / f"{index}_ground.bin").write_bytes(ground.tobytes()) + (output_root / f"{index}_nonground.bin").write_bytes(nonground.tobytes()) + timing.append( + f"{index}\t{row['source_frame_index']}\t" + f"{cast(float, row['session_seconds']):.6f}" + f"\t1\t{row['available_slot']}\t{source.shape[0]}\t{ground.shape[0]}" + f"\t{nonground.shape[0]}\t1.250000\t1.500000" + ) + timing_path = tmp_path / "tgs-timing.tsv" + timing_path.write_text("\n".join(timing) + "\n", encoding="utf-8") + return output_root, timing_path + + +def test_dynamic_source_materializer_is_source_derived_and_tamper_evident( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + stage_root, _expected = _materialized_source(tmp_path, monkeypatch) + stage = validate_m49_portable_source_stage(stage_root) + assert stage.timeline_frame_count == 3 + assert stage.available_lidar_frame_count == 2 + rows = read_m49_source_index(stage.root / "sequence-index.ndjson", expected_frame_count=3) + assert [row["sample_available"] for row in rows] == [False, True, True] + assert [row["source_frame_index"] for row in rows] == [0, 1, 2] + assert all( + "RAVNOVES" not in path.read_text(errors="ignore") + for path in ( + stage.root / "manifest.json", + stage.root / "sequence-index.ndjson", + stage.root / "schedule.tsv", + ) + ) + + schedule = stage.root / "schedule.tsv" + schedule.write_text(schedule.read_text() + "0\t0\t0\t-1\t0\n", encoding="utf-8") + with pytest.raises(M49PortableSourceError): + validate_m49_portable_source_stage(stage.root) + + +def test_source_materializer_rejects_unadmitted_adjacent_metadata( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with pytest.raises(M49PortableSourceError, match="admitted raw and host-time"): + _materialized_source( + tmp_path, + monkeypatch, + include_metadata_member=False, + ) + + +def test_fixed_worker_stage_consumer_requires_manifested_metadata_member( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + direct_root = tmp_path / "direct" + direct_root.mkdir() + _direct_stage, expected = _materialized_source(direct_root, monkeypatch) + job = _sealed_worker_job(expected) + worker_root = tmp_path / "worker-source" + (worker_root / "camera" / "epoch-1" / "segments").mkdir(parents=True) + bundle = (direct_root / "source-bundle.json").read_bytes() + capability = (direct_root / "source-capability.json").read_bytes() + init = b"exact-init" + segments = (b"segment-1", b"segment-2", b"segment-3") + (worker_root / "source-bundle.json").write_bytes(bundle) + (worker_root / "source-capability.json").write_bytes(capability) + (worker_root / "mqtt.raw.k1mqtt").write_bytes(RAW_PAYLOAD) + (worker_root / "mqtt.metadata.jsonl").write_bytes(METADATA_PAYLOAD) + (worker_root / "camera" / "epoch-1" / "init.mp4").write_bytes(init) + for index, payload in enumerate(segments, start=1): + (worker_root / "camera" / "epoch-1" / "segments" / f"{index}.m4s").write_bytes(payload) + members = [ + _worker_source_member( + job, + kind="source-bundle", + payload=bundle, + media_type="application/json", + ), + _worker_source_member( + job, + kind="source-capability", + payload=capability, + media_type="application/json", + ), + _worker_source_member( + job, + kind="spatial-replay", + payload=RAW_PAYLOAD, + media_type="application/x-nodedc-k1mqtt", + artifact_id="raw-transport-primary", + primary=True, + ), + _worker_source_member( + job, + kind="spatial-replay-metadata", + payload=METADATA_PAYLOAD, + media_type="application/x-ndjson", + artifact_id="raw-transport-index", + ), + _worker_source_member( + job, + kind="camera-init", + payload=init, + media_type="video/mp4", + artifact_id="camera-primary", + camera_epoch=1, + ), + *[ + _worker_source_member( + job, + kind="camera-segment", + payload=payload, + media_type="video/iso.segment", + artifact_id="camera-primary", + camera_epoch=1, + camera_sequence=index, + ) + for index, payload in enumerate(segments, start=1) + ], + ] + members.sort(key=lambda row: cast(str, row["member_id"])) + materialization = { + "schema_version": "missioncore.observatory-portable-source-materialization/v1", + "job_id": job.job_id, + "job_identity_sha256": job.identity_sha256, + "claim_generation": job.claim_generation, + "source": { + "session_id": job.source_session_id, + "bundle_sha256": job.source_bundle_sha256, + "capability_manifest_sha256": job.source_capability_manifest_sha256, + }, + "members": members, + "authority": AUTHORITY, + } + (worker_root / "materialization-manifest.json").write_bytes(canonical_json(materialization)) + stage = PortableWorkerSourceStage( + root=worker_root, + source_bundle_sha256=job.source_bundle_sha256, + source_capability_manifest_sha256=job.source_capability_manifest_sha256, + source_adapter_sha256=job.source_adapter_sha256, + ) + built_from: list[Path] = [] + + def fake_build(capture: Path, output: Path, *, session_id: str) -> Path: + assert session_id == SESSION_ID + assert capture == worker_root / "mqtt.raw.k1mqtt" + assert output.name == "lidar-replay-packs" + built_from.append(capture) + return worker_root + + monkeypatch.setattr(source_module, "build_lidar_replay_pack_v2", fake_build) + + class _DeliveredSource: + def materialize(self, requested: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage: + assert requested == job + return stage + + materializer = M49PortableSourceMaterializerAdapter( + upstream=_DeliveredSource(), + profile_path=PROFILE_PATH, + output_parent=tmp_path / "worker-output", + ) + materialized = materializer.materialize(job) + assert built_from == [worker_root / "mqtt.raw.k1mqtt"] + assert isinstance(materialized, M49PortableBoundSourceStage) + assert materialized.m49_stage.timeline_frame_count == 3 + assert materialized.m49_stage.available_lidar_frame_count == 2 + + +def test_result_v2_assembler_and_exact_validator_round_trip( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + stage_root, expected = _materialized_source(tmp_path, monkeypatch) + registry = _ready_m49_registry(tmp_path) + definition = registry.resolve_setup("m49-tgs-portable-v2") + queue, running, claim_token = _running_job(tmp_path, definition, expected) + sealed = _seal_running_job(running) + prepared = tmp_path / "prepared-runner" + prepared.mkdir() + output_root, timing_path = _runner_outputs(stage_root, prepared) + binary = tmp_path / "run_m49_tgs_portable" + binary.write_bytes(b"\x7fELFtest-only") + binary.chmod(0o755) + build_seal = builder.seal_m49_compiled_runner_build( + source_root=REPOSITORY_ROOT, + source_revision="f" * 40, + binary_path=binary, + manifest_path=tmp_path / "runner-build-seal.json", + ) + installation = M49PortableRunnerInstallation( + profile_path=PROFILE_PATH, + runner_binary_path=binary, + runner_build_seal_path=build_seal.manifest_path, + runner_build_seal_sha256=build_seal.manifest_sha256, + output_parent=tmp_path / "executor-output", + ) + + def fake_invoke( + *, + binary: Path, + sequence: Path, + schedule: Path, + output: Path, + timing: Path, + workspace: Path, + timeout_seconds: int, + ) -> None: + assert binary == installation.runner_binary_path + assert sequence == Path(stage_root) / "tgs/sequence/velodyne" + assert schedule == Path(stage_root) / "schedule.tsv" + assert workspace.parent == installation.output_parent + assert timeout_seconds == installation.timeout_seconds + shutil.copytree(output_root, output) + shutil.copyfile(timing_path, timing) + + source_stage = validate_m49_portable_source_stage(stage_root) + bound_source = M49PortableBoundSourceStage( + root=source_stage.root, + source_bundle_sha256=sealed.source_bundle_sha256, + source_capability_manifest_sha256=sealed.source_capability_manifest_sha256, + source_adapter_sha256=sealed.source_adapter_sha256, + job=sealed, + m49_stage=source_stage, + ) + runner = M49PortableProfileRunnerAdapter( + definition=definition, + installation=installation, + created_at_utc=lambda: NOW, + invoker=fake_invoke, + ) + draft = runner.run( + PortableWorkerRuntimePlan( + job_id=sealed.job_id, + adapter_id="m49-tgs-worker006-portable-v2", + candidate_sha256="f" * 64, + setup_id=sealed.setup_id, + definition_sha256=sealed.definition_sha256, + source_bundle_sha256=sealed.source_bundle_sha256, + source_capability_manifest_sha256=sealed.source_capability_manifest_sha256, + result_contract_sha256=definition.result_contract.contract_sha256, + phases=M49_PORTABLE_RUNTIME_PHASES, + ), + bound_source, + ) + package = PortableResultPackageManifest.from_bytes((draft.root / "manifest.json").read_bytes()) + assert draft.root.name == package.manifest_sha256 + assert draft.result_id.startswith("m49-tgs-portable-review-") + succeeded = queue.succeed( + running.job_id, + claim_token=claim_token, + result_id=draft.result_id, + result_sha256=draft.result_sha256, + ) + artifact_paths = { + artifact.role: draft.root.joinpath(*Path(artifact.relative_path).parts) + for artifact in package.artifacts + } + result_document = cast( + dict[str, object], + json.loads(artifact_paths["result-document"].read_bytes()), + ) + context = PortableResultValidationContext( + manifest=package, + job=succeeded, + definition=definition, + result_document=result_document, + artifact_paths=artifact_paths, + ) + validate_m49_portable_result(context) + + states = np.load(artifact_paths["costmap-states"], mmap_mode="r", allow_pickle=False) + assert np.all(states[0] == 0) + assert set(np.unique(states[1])).issubset({0, 1, 2, 3}) + assert 3 in states[1] + point_accounting = cast(dict[str, object], result_document["point_accounting"]) + assert point_accounting["unaccounted"] == 0 + + drifted = copy.deepcopy(result_document) + cast(dict[str, object], drifted["point_accounting"])["eligible"] = 999 + with pytest.raises(M49PortableResultError): + validate_m49_portable_result( + PortableResultValidationContext( + manifest=package, + job=succeeded, + definition=definition, + result_document=drifted, + artifact_paths=artifact_paths, + ) + ) + + +def test_executor_release_candidate_is_deterministic_blocked_and_tamper_evident( + tmp_path: Path, +) -> None: + source_root = tmp_path / "source" + for relative in builder.M49_RELEASE_SOURCES: + target = source_root / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(REPOSITORY_ROOT / relative, target) + first = builder.build_m49_executor_release_candidate( + source_root=source_root, + output_directory=tmp_path / "out-a", + source_revision="f" * 40, + source_state="uncommitted-candidate", + ) + second = builder.build_m49_executor_release_candidate( + source_root=source_root, + output_directory=tmp_path / "out-b", + source_revision="f" * 40, + source_state="uncommitted-candidate", + ) + assert first.archive.read_bytes() == second.archive.read_bytes() + assert first.archive_sha256 == second.archive_sha256 + assert first.manifest["state"] == "blocked" + assert tuple(first.manifest["blockers"]) == builder.M49_RELEASE_BLOCKERS + assert first.manifest["executor_image_sha256"] is None + assert first.manifest["compiled_runner"] is None + assert builder.verify_m49_executor_release_candidate(first.archive) == first + + dockerfile = DOCKERFILE_PATH.read_text(encoding="utf-8") + assert "FROM ndc/mission-core-m49-t3-travel:20260826" in dockerfile + assert builder.M49_TRAVEL_IMAGE_SHA256 in dockerfile + assert "--network" not in dockerfile + assert "com.nodedc.authority=\"observation-only\"" in dockerfile + installer = INSTALLER_PATH.read_text(encoding="utf-8") + assert "--pull=false --no-cache --network none" in installer + assert "--network none --read-only" in installer + assert "committed-source-snapshot-missing" in installer + assert "MissionCore-M49TgsFullShadow" in installer + + changed = source_root / "src" / "k1link" / "observatory" / "m49_portable_result.py" + changed.write_bytes(changed.read_bytes() + b"\n") + drifted = builder.build_m49_executor_release_candidate( + source_root=source_root, + output_directory=tmp_path / "out-c", + source_revision="f" * 40, + source_state="uncommitted-candidate", + ) + assert drifted.candidate_sha256 != first.candidate_sha256 diff --git a/tests/test_observatory_portable_artifact_transport.py b/tests/test_observatory_portable_artifact_transport.py new file mode 100644 index 0000000..b495343 --- /dev/null +++ b/tests/test_observatory_portable_artifact_transport.py @@ -0,0 +1,600 @@ +from __future__ import annotations + +import asyncio +import copy +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +import pytest + +from k1link.observatory.portable_artifact_transport import ( + PortableArtifactTransportIntegrityError, + PortableArtifactTransportUnavailableError, + PortableObservatoryArtifactTransport, +) +from k1link.observatory.portable_result_contract import ( + OBSERVATION_ONLY_AUTHORITY, + RESULT_DOCUMENT_ROLE, + PortableResultArtifact, + PortableResultPackageManifest, + canonical_json, +) +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinition, + PortableRunDefinitionRegistry, + canonical_sha256, +) +from k1link.observatory.recorded_jobs import ( + ObservatoryRecordedJob, + ObservatoryRecordedJobIntent, + ObservatoryRecordedJobQueue, + ObservatoryRecordedQueueStaleClaimError, + RecordedRunDefinitionRegistry, +) +from k1link.observatory.source_admission import ( + PORTABLE_SOURCE_BUNDLE_SCHEMA, + PORTABLE_SOURCE_CAPABILITY_SCHEMA, + PORTABLE_SOURCE_DOCUMENT_DIRECTORY, +) +from k1link.sessions.media import ( + RecordedMediaEpoch, + RecordedMediaManifest, + RecordedMediaSegment, +) +from k1link.sessions.models import ( + RecordedMediaArtifact, + ReplayArtifact, + ReplayCommand, + SessionDetail, + SessionSummary, +) + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" +NOW = "2026-08-31T10:00:00.000Z" +SOURCE_SESSION_ID = "20260831T095500Z_viewer_live" +RESULT_ID = "portable-result-transport-001" + + +def _ready_registry(tmp_path: Path) -> PortableRunDefinitionRegistry: + registry = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH) + base = registry.definitions[0] + document = cast( + dict[str, object], json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) + ) + rows = cast(list[object], document["definitions"]) + selected = copy.deepcopy(cast(dict[str, object], rows[0])) + selected["executor"] = { + "contour_id": "worker-006", + "state": "ready", + "release_id": "portable-transport-test-executor", + "release_sha256": "1" * 64, + "image_sha256": "2" * 64, + "reason_code": None, + "reason": None, + } + identity = copy.deepcopy(base.identity_document()) + identity["executor"] = { + "contour_id": "worker-006", + "state": "ready", + "release_id": "portable-transport-test-executor", + "release_sha256": "1" * 64, + "image_sha256": "2" * 64, + } + selected["definition_sha256"] = canonical_sha256(identity) + path = tmp_path / "definitions.json" + path.write_bytes( + canonical_json( + { + "schema_version": document["schema_version"], + "definitions": [selected], + } + ) + ) + return PortableRunDefinitionRegistry.from_file(path) + + +@dataclass +class _Store: + data_dir: Path + detail: SessionDetail + catalog_sha256: str + replay: ReplayCommand + recorded_media: tuple[RecordedMediaArtifact, ...] + + def get_session_with_catalog_snapshot( + self, session_id: str + ) -> tuple[SessionDetail, str]: + assert session_id == SOURCE_SESSION_ID + return self.detail, self.catalog_sha256 + + def prepare_replay(self, session_id: str) -> ReplayCommand: + assert session_id == SOURCE_SESSION_ID + return self.replay + + def list_recorded_media( + self, session_id: str + ) -> tuple[RecordedMediaArtifact, ...]: + assert session_id == SOURCE_SESSION_ID + return self.recorded_media + + +@dataclass +class _Inspector: + manifest: RecordedMediaManifest + + def restore_prepared( + self, + artifact: RecordedMediaArtifact, + replay: ReplayCommand, + ) -> RecordedMediaManifest: + assert artifact.artifact_id == self.manifest.artifact_id + assert replay.session_id == self.manifest.session_id + return self.manifest + + +@dataclass +class _Fixture: + service: PortableObservatoryArtifactTransport + queue: ObservatoryRecordedJobQueue + definition: PortableRunDefinition + job: ObservatoryRecordedJob + claim_token: str + raw_path: Path + + +def _fixture(tmp_path: Path) -> _Fixture: + registry = _ready_registry(tmp_path) + definition = registry.definitions[0] + data_dir = tmp_path / "data" + source_root = tmp_path / "source" + raw_path = source_root / "mqtt.raw.k1mqtt" + metadata_path = source_root / "mqtt.metadata.jsonl" + init_path = source_root / "camera" / "epoch-1" / "init.mp4" + segment_path = source_root / "camera" / "epoch-1" / "segments" / "1.m4s" + segment_path.parent.mkdir(parents=True) + raw_path.parent.mkdir(parents=True, exist_ok=True) + raw_path.write_bytes(b"sealed-raw-replay") + metadata_path.write_bytes(b'{"offset":0,"topic":"/points"}\n') + init_path.write_bytes(b"sealed-init") + segment_path.write_bytes(b"sealed-segment") + raw_sha = hashlib.sha256(raw_path.read_bytes()).hexdigest() + metadata_sha = hashlib.sha256(metadata_path.read_bytes()).hexdigest() + init_sha = hashlib.sha256(init_path.read_bytes()).hexdigest() + segment_sha = hashlib.sha256(segment_path.read_bytes()).hexdigest() + catalog_sha = "3" * 64 + generation_sha = "4" * 64 + replay = ReplayCommand( + session_id=SOURCE_SESSION_ID, + plugin_id=definition.source_requirements.plugin_id, + allowed_root=source_root, + session_root=source_root, + primary_artifact_id="raw-transport-primary", + artifacts=( + ReplayArtifact( + artifact_id="raw-transport-primary", + path=raw_path, + media_type="application/x-nodedc-k1mqtt", + file_byte_length=raw_path.stat().st_size, + replay_byte_length=raw_path.stat().st_size, + expected_sha256=raw_sha, + ), + ReplayArtifact( + artifact_id="raw-transport-index", + path=metadata_path, + media_type="application/x-ndjson", + file_byte_length=metadata_path.stat().st_size, + replay_byte_length=metadata_path.stat().st_size, + expected_sha256=None, + ), + ), + timeline_origin_epoch_ns=1, + timeline_origin_monotonic_ns=2, + speed=1.0, + loop=False, + ) + recorded_media = RecordedMediaArtifact( + session_id=SOURCE_SESSION_ID, + public_source_id="recorded.camera.right", + artifact_id="recorded-video-right", + source_path=source_root / "camera", + byte_length=init_path.stat().st_size + segment_path.stat().st_size, + ) + epoch = RecordedMediaEpoch( + ordinal=1, + path=init_path.parent, + init_path=init_path, + init_byte_length=init_path.stat().st_size, + init_sha256=init_sha, + media_type='video/mp4; codecs="avc1.641028"', + timeline_start_seconds=0.0, + timeline_end_seconds=0.1, + segments=( + RecordedMediaSegment( + sequence=1, + path=segment_path, + byte_length=segment_path.stat().st_size, + sha256=segment_sha, + random_access=True, + end_time_seconds=0.1, + ), + ), + ) + manifest = RecordedMediaManifest( + session_id=SOURCE_SESSION_ID, + public_source_id=recorded_media.public_source_id, + artifact_id=recorded_media.artifact_id, + synchronization="host-arrival-best-effort", + generation_sha256=generation_sha, + timeline_start_seconds=0.0, + timeline_end_seconds=0.1, + byte_length=recorded_media.byte_length, + epochs=(epoch,), + ) + detail = SessionDetail( + summary=SessionSummary( + session_id=SOURCE_SESSION_ID, + display_name="Portable source", + status="ready", + started_at_utc=NOW, + completed_at_utc=NOW, + duration_seconds=0.1, + modalities=("point-cloud", "trajectory", "video"), + source_count=3, + total_bytes=100, + replayable=True, + origin=definition.source_requirements.archive_id, + ), + sources=(), + artifacts=(), + plugin_id=definition.source_requirements.plugin_id, + archive_id=definition.source_requirements.archive_id, + ) + source_adapter = { + "id": definition.source_adapter.adapter_id, + "version": definition.source_adapter.version, + "sha256": definition.source_adapter.contract_sha256, + } + bundle = { + "schema_version": PORTABLE_SOURCE_BUNDLE_SCHEMA, + "source_session_id": SOURCE_SESSION_ID, + "source_catalog_sha256": catalog_sha, + "plugin_id": definition.source_requirements.plugin_id, + "archive_id": definition.source_requirements.archive_id, + "source_adapter": source_adapter, + "sources": [], + "spatial_replay": { + "primary_artifact_id": replay.primary_artifact_id, + "members": [ + { + "artifact_id": "raw-transport-primary", + "media_type": "application/x-nodedc-k1mqtt", + "byte_length": raw_path.stat().st_size, + "replay_byte_length": raw_path.stat().st_size, + "sha256": raw_sha, + }, + { + "artifact_id": "raw-transport-index", + "media_type": "application/x-ndjson", + "byte_length": metadata_path.stat().st_size, + "replay_byte_length": metadata_path.stat().st_size, + "sha256": metadata_sha, + }, + ], + "timeline_origin_epoch_ns": 1, + "timeline_origin_monotonic_ns": 2, + }, + "camera": { + "artifact_id": recorded_media.artifact_id, + "public_source_id": recorded_media.public_source_id, + "generation_sha256": generation_sha, + "synchronization": "host-arrival-best-effort", + "epoch": { + "ordinal": 1, + "media_type": epoch.media_type, + "init": { + "byte_length": epoch.init_byte_length, + "sha256": init_sha, + }, + "timeline_start_seconds": 0.0, + "timeline_end_seconds": 0.1, + "segments": [ + { + "sequence": 1, + "byte_length": segment_path.stat().st_size, + "sha256": segment_sha, + "random_access": True, + "end_time_seconds": 0.1, + } + ], + }, + }, + "authority": OBSERVATION_ONLY_AUTHORITY, + } + bundle_payload = canonical_json(bundle) + bundle_sha = hashlib.sha256(bundle_payload).hexdigest() + capability = { + "schema_version": PORTABLE_SOURCE_CAPABILITY_SCHEMA, + "source_session_id": SOURCE_SESSION_ID, + "source_catalog_sha256": catalog_sha, + "source_bundle_sha256": bundle_sha, + "source_adapter_sha256": definition.source_adapter.contract_sha256, + "modalities": [], + "camera_profile": {}, + "calibration": {}, + "authority": OBSERVATION_ONLY_AUTHORITY, + } + capability_payload = canonical_json(capability) + capability_sha = hashlib.sha256(capability_payload).hexdigest() + documents = data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY + documents.mkdir(parents=True) + (documents / f"{bundle_sha}.json").write_bytes(bundle_payload) + (documents / f"{capability_sha}.json").write_bytes(capability_payload) + queue = ObservatoryRecordedJobQueue( + data_dir, + definitions=RecordedRunDefinitionRegistry( + (definition.to_recorded_run_definition(),) + ), + clock=lambda: NOW, + ) + job, created = queue.submit( + ObservatoryRecordedJobIntent( + idempotency_key="portable-artifact-transport-001", + source_session_id=SOURCE_SESSION_ID, + source_catalog_sha256=catalog_sha, + source_bundle_sha256=bundle_sha, + source_capability_manifest_sha256=capability_sha, + setup_id=definition.setup_id, + definition_sha256=definition.definition_sha256, + ), + enqueue=True, + ) + assert created + claim = queue.claim_next( + claimant_id="worker-006", + claim_request_id="portable-artifact-claim-001", + ) + assert claim is not None + running = queue.start(job.job_id, claim_token=claim.claim_token) + store = _Store( + data_dir=data_dir, + detail=detail, + catalog_sha256=catalog_sha, + replay=replay, + recorded_media=(recorded_media,), + ) + service = PortableObservatoryArtifactTransport( + queue=queue, + session_store=store, # type: ignore[arg-type] + media_inspector=_Inspector(manifest), # type: ignore[arg-type] + definitions=registry, + ) + return _Fixture(service, queue, definition, running, claim.claim_token, raw_path) + + +def _result_package( + tmp_path: Path, + fixture: _Fixture, +) -> tuple[PortableResultPackageManifest, bytes]: + result_document = { + "schema_version": fixture.definition.result_contract.result_schema, + "result_id": RESULT_ID, + "result_kind": fixture.definition.result_contract.result_kind, + "authority": OBSERVATION_ONLY_AUTHORITY, + } + payload = canonical_json(result_document) + artifact = PortableResultArtifact( + role=RESULT_DOCUMENT_ROLE, + relative_path="artifacts/result.json", + media_type="application/json", + byte_length=len(payload), + sha256=hashlib.sha256(payload).hexdigest(), + ) + package = PortableResultPackageManifest.create( + job=fixture.job, + definition=fixture.definition, + result_id=RESULT_ID, + created_at_utc=NOW, + artifacts=(artifact,), + ) + return package, payload + + +def test_source_members_are_claim_bound_and_materialized_through_cas( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + manifest = fixture.service.source_manifest( + job_id=fixture.job.job_id, + claim_token=fixture.claim_token, + claim_generation=fixture.job.claim_generation, + claimant_id="worker-006", + ) + + assert {member.kind for member in manifest.members} == { + "source-bundle", + "source-capability", + "spatial-replay", + "spatial-replay-metadata", + "camera-init", + "camera-segment", + } + assert "path" not in json.dumps(manifest.as_dict(), sort_keys=True) + raw = next(member for member in manifest.members if member.kind == "spatial-replay") + admitted, cas_path = fixture.service.materialize_source_member( + job_id=fixture.job.job_id, + member_id=raw.member_id, + claim_token=fixture.claim_token, + claim_generation=fixture.job.claim_generation, + claimant_id="worker-006", + ) + + assert admitted.sha256 == raw.sha256 + assert cas_path.read_bytes() == b"sealed-raw-replay" + assert cas_path != fixture.raw_path + assert cas_path.name == raw.sha256 + metadata = next( + member + for member in manifest.members + if member.kind == "spatial-replay-metadata" + ) + admitted_metadata, metadata_cas_path = fixture.service.materialize_source_member( + job_id=fixture.job.job_id, + member_id=metadata.member_id, + claim_token=fixture.claim_token, + claim_generation=fixture.job.claim_generation, + claimant_id="worker-006", + ) + assert admitted_metadata.artifact_id == "raw-transport-index" + assert metadata_cas_path.read_bytes() == b'{"offset":0,"topic":"/points"}\n' + assert metadata_cas_path.name == metadata.sha256 + + +def test_source_member_rejects_tampering_and_stale_generation(tmp_path: Path) -> None: + fixture = _fixture(tmp_path) + manifest = fixture.service.source_manifest( + job_id=fixture.job.job_id, + claim_token=fixture.claim_token, + claim_generation=fixture.job.claim_generation, + claimant_id="worker-006", + ) + raw = next(member for member in manifest.members if member.kind == "spatial-replay") + fixture.raw_path.write_bytes(b"changed") + + with pytest.raises( + PortableArtifactTransportIntegrityError, + match="admitted regular file|changed", + ): + fixture.service.materialize_source_member( + job_id=fixture.job.job_id, + member_id=raw.member_id, + claim_token=fixture.claim_token, + claim_generation=fixture.job.claim_generation, + claimant_id="worker-006", + ) + with pytest.raises(ObservatoryRecordedQueueStaleClaimError): + fixture.service.source_manifest( + job_id=fixture.job.job_id, + claim_token=fixture.claim_token, + claim_generation=fixture.job.claim_generation + 1, + claimant_id="worker-006", + ) + + +def test_result_upload_is_atomic_resumable_and_required_before_success( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + package, result_payload = _result_package(tmp_path, fixture) + plan = fixture.service.stage_result_manifest( + job_id=fixture.job.job_id, + result_sha256=package.manifest_sha256, + manifest_payload=package.canonical_bytes, + claim_token=fixture.claim_token, + claim_generation=fixture.job.claim_generation, + claimant_id="worker-006", + ) + assert plan.complete is False + with pytest.raises(PortableArtifactTransportUnavailableError, match="incomplete"): + fixture.service.complete_result_upload( + job_id=fixture.job.job_id, + result_sha256=package.manifest_sha256, + claim_token=fixture.claim_token, + claim_generation=fixture.job.claim_generation, + claimant_id="worker-006", + ) + + async def chunks() -> object: + yield result_payload[:7] + yield result_payload[7:] + + uploaded = asyncio.run( + fixture.service.upload_result_member( + job_id=fixture.job.job_id, + result_sha256=package.manifest_sha256, + member_id=plan.members[0].member_id, + chunks=chunks(), # type: ignore[arg-type] + claim_token=fixture.claim_token, + claim_generation=fixture.job.claim_generation, + claimant_id="worker-006", + ) + ) + assert uploaded.complete is True + repeated = fixture.service.stage_result_manifest( + job_id=fixture.job.job_id, + result_sha256=package.manifest_sha256, + manifest_payload=package.canonical_bytes, + claim_token=fixture.claim_token, + claim_generation=fixture.job.claim_generation, + claimant_id="worker-006", + ) + assert repeated.members[0].uploaded is True + receipt = fixture.service.complete_result_upload( + job_id=fixture.job.job_id, + result_sha256=package.manifest_sha256, + claim_token=fixture.claim_token, + claim_generation=fixture.job.claim_generation, + claimant_id="worker-006", + ) + assert receipt.result_id == RESULT_ID + package_root = fixture.service.require_completed_for_success( + job_id=fixture.job.job_id, + result_id=RESULT_ID, + result_sha256=package.manifest_sha256, + claim_token=fixture.claim_token, + claimant_id="worker-006", + ) + succeeded = fixture.queue.succeed( + fixture.job.job_id, + claim_token=fixture.claim_token, + result_id=RESULT_ID, + result_sha256=package.manifest_sha256, + ) + assert fixture.service.package_root_for_terminal(succeeded) == package_root + assert (package_root / "artifacts" / "result.json").read_bytes() == result_payload + + +def test_result_upload_rejects_wrong_digest_and_never_publishes_partial_member( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + package, result_payload = _result_package(tmp_path, fixture) + plan = fixture.service.stage_result_manifest( + job_id=fixture.job.job_id, + result_sha256=package.manifest_sha256, + manifest_payload=package.canonical_bytes, + claim_token=fixture.claim_token, + claim_generation=fixture.job.claim_generation, + claimant_id="worker-006", + ) + + async def bad_chunks() -> object: + yield b"x" * len(result_payload) + + with pytest.raises( + PortableArtifactTransportIntegrityError, + match="differs from its manifest", + ): + asyncio.run( + fixture.service.upload_result_member( + job_id=fixture.job.job_id, + result_sha256=package.manifest_sha256, + member_id=plan.members[0].member_id, + chunks=bad_chunks(), # type: ignore[arg-type] + claim_token=fixture.claim_token, + claim_generation=fixture.job.claim_generation, + claimant_id="worker-006", + ) + ) + repeated = fixture.service.stage_result_manifest( + job_id=fixture.job.job_id, + result_sha256=package.manifest_sha256, + manifest_payload=package.canonical_bytes, + claim_token=fixture.claim_token, + claim_generation=fixture.job.claim_generation, + claimant_id="worker-006", + ) + assert repeated.members[0].uploaded is False diff --git a/tests/test_observatory_portable_lab_v1_executor.py b/tests/test_observatory_portable_lab_v1_executor.py new file mode 100644 index 0000000..3a64373 --- /dev/null +++ b/tests/test_observatory_portable_lab_v1_executor.py @@ -0,0 +1,1265 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +from dataclasses import replace +from pathlib import Path + +import pytest + +from k1link.compute.jobs import CameraComputeJob +from k1link.observatory.portable_lab_v1_executor import ( + PORTABLE_LAB_V1_RESULT_SCHEMA, + PortableLabV1MaterializedSource, + PortableLabV1OrchestrationPlan, + PortableLabV1PlanError, + PortableLabV1ReleaseCandidate, + PortableLabV1ReleaseInspection, + PortableLabV1ResultError, + PortableLabV1SourceError, + PortableLabV1SourceInput, + assemble_lab_v1_result_v2, + build_portable_ddrnet_effective_config, + materialize_lab_v1_source_input, + package_lab_v1_result, + validate_lab_v1_result_v2, +) +from k1link.observatory.portable_lab_v1_worker import ( + PORTABLE_LAB_V1_RUNTIME_PHASES, + PORTABLE_SOURCE_MATERIALIZATION_SCHEMA, + PortableLabV1BoundSourceStage, + PortableLabV1ProfileRunnerAdapter, + PortableLabV1RunnerInstallation, + PortableLabV1WorkerError, + materialize_lab_v1_source_from_worker_stage, +) +from k1link.observatory.portable_result_contract import ( + OBSERVATION_ONLY_AUTHORITY, + PortableResultPackageManifest, + PortableResultValidationContext, + canonical_json, +) +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinition, + PortableRunDefinitionRegistry, + canonical_sha256, +) +from k1link.observatory.portable_worker_runtime import ( + PortableWorkerRuntimeJobRejectedError, + PortableWorkerRuntimePlan, + PortableWorkerSourceStage, +) +from k1link.observatory.recorded_jobs import ( + ObservatoryRecordedJob, + ObservatoryRecordedJobIntent, + ObservatoryRecordedJobQueue, + RecordedRunDefinition, + RecordedRunDefinitionRegistry, +) +from k1link.observatory.worker_agent import ( + ObservatoryWorkerExecutorIdentity, + SealedObservatoryRecordedJob, +) + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +DEFINITIONS_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" +RELEASE_PATH = ( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "observatory_portable" + / "lab-v1-eomt-ddrnet-executor-candidate.json" +) +PORTABLE_DDRNET_CONFIG = ( + REPOSITORY_ROOT + / "config" + / "perception" + / "lab-v1-eomt-ddrnet-portable-v2.json" +) +NOW = "2026-08-31T09:00:00.000Z" +SOURCE_SESSION_ID = "20260831T083000Z_viewer_live" +CATALOG_SHA256 = "a" * 64 +CAMERA_GENERATION_SHA256 = "b" * 64 +CAMERA_INIT_SHA256 = "e2279963e16d84c91d68e7dbb1f7efed840533387dfeb844b7398bff45fbde38" +CAMERA_SEGMENT_SHA256 = "c" * 64 +EXECUTOR_RELEASE_SHA256 = "d" * 64 +EXECUTOR_IMAGE_SHA256 = "e" * 64 +SPATIAL_REPLAY_PAYLOAD = b"sealed-spatial-replay" +SPATIAL_METADATA_PAYLOAD = b'{"offset":0,"topic":"/camera"}\n' + + +def _definition() -> PortableRunDefinition: + registry = PortableRunDefinitionRegistry.from_file(DEFINITIONS_PATH) + return next( + definition + for definition in registry.definitions + if definition.setup_id == "lab-v1-eomt-ddrnet-portable-v1" + ) + + +def _canonical(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def _source_documents(definition: PortableRunDefinition) -> tuple[bytes, bytes]: + source_adapter = { + "id": definition.source_adapter.adapter_id, + "version": definition.source_adapter.version, + "sha256": definition.source_adapter.contract_sha256, + } + bundle = { + "schema_version": "missioncore.portable-recorded-source-bundle/v1", + "source_session_id": SOURCE_SESSION_ID, + "source_catalog_sha256": CATALOG_SHA256, + "plugin_id": definition.source_requirements.plugin_id, + "archive_id": definition.source_requirements.archive_id, + "source_adapter": source_adapter, + "sources": [], + "spatial_replay": {}, + "camera": { + "artifact_id": "camera-recording", + "public_source_id": "sensor.camera.right", + "generation_sha256": CAMERA_GENERATION_SHA256, + "synchronization": "host-arrival-best-effort", + "epoch": { + "ordinal": 1, + "media_type": definition.source_requirements.recorded_media_type, + "init": {"byte_length": 4, "sha256": CAMERA_INIT_SHA256}, + "timeline_start_seconds": 1.0, + "timeline_end_seconds": 2.0, + "segments": [ + { + "sequence": 1, + "byte_length": 7, + "sha256": CAMERA_SEGMENT_SHA256, + "random_access": True, + "end_time_seconds": 2.0, + } + ], + }, + }, + "authority": OBSERVATION_ONLY_AUTHORITY, + } + bundle_bytes = _canonical(bundle) + bundle_sha256 = hashlib.sha256(bundle_bytes).hexdigest() + capability = { + "schema_version": "missioncore.portable-recorded-source-capability/v1", + "source_session_id": SOURCE_SESSION_ID, + "source_catalog_sha256": CATALOG_SHA256, + "source_bundle_sha256": bundle_sha256, + "source_adapter_sha256": definition.source_adapter.contract_sha256, + "modalities": [ + { + "modality": modality, + "source_id": ( + "sensor.camera.right" if modality == "video" else f"sensor.{modality}" + ), + "semantic_channel_id": ( + definition.source_requirements.camera_semantic_channel_id + if modality == "video" + else f"recorded.{modality}" + ), + "seekable": True, + } + for modality in definition.source_requirements.required_modalities + ], + "camera_profile": { + "media_type": definition.source_requirements.recorded_media_type, + "init_sha256": CAMERA_INIT_SHA256, + "width": 800, + "height": 600, + "profile_attestation": "exact-isobmff-init-sha256", + "generation_sha256": CAMERA_GENERATION_SHA256, + "frame_count": 1, + "timeline_start_seconds": 1.0, + "timeline_end_seconds": 2.0, + }, + "calibration": { + "slot": definition.source_requirements.calibration_slot, + "sha256": definition.source_requirements.calibration_identity_sha256, + "binding": "external-rig-profile", + }, + "authority": OBSERVATION_ONLY_AUTHORITY, + } + capability_bytes = _canonical(capability) + return bundle_bytes, capability_bytes + + +def _camera_job(tmp_path: Path) -> CameraComputeJob: + root = tmp_path / "recorded-camera-111111111111111111111111" + root.mkdir() + files = [ + { + "path": "input/camera/sensor.camera.right/epoch-1/summary.json", + "byte_length": 2, + "sha256": "1" * 64, + }, + { + "path": "input/camera/sensor.camera.right/epoch-1/index.jsonl", + "byte_length": 3, + "sha256": "2" * 64, + }, + { + "path": "input/camera/sensor.camera.right/epoch-1/init.mp4", + "byte_length": 4, + "sha256": CAMERA_INIT_SHA256, + }, + { + "path": "input/camera/sensor.camera.right/epoch-1/segments/1.m4s", + "byte_length": 7, + "sha256": CAMERA_SEGMENT_SHA256, + }, + ] + input_document = { + "kind": "canonical-camera-epoch", + "session_id": SOURCE_SESSION_ID, + "source_id": "sensor.camera.right", + "codec_epoch": 1, + "synchronization": "host-arrival-best-effort", + "media_type": "video/mp4; codecs=\"avc1.641028\"", + "timeline": { + "basis": "session-time-seconds", + "start_seconds": 1.0, + "end_seconds": 2.0, + }, + "segment_count": 1, + "byte_length": 16, + "archive_summary_sha256": "1" * 64, + "archive_index_sha256": "2" * 64, + "files": files, + } + input_sha256 = hashlib.sha256(_canonical(input_document)).hexdigest() + manifest = { + "schema_version": "missioncore.compute-job/v1", + "job_id": root.name, + "profile": "recorded-camera-perception/v1", + "input_sha256": input_sha256, + "input": input_document, + "result_contract": { + "schema_version": "missioncore.compute-result/v1", + "timestamp_basis": "session-time-seconds", + }, + } + (root / "job.json").write_bytes(_canonical(manifest)) + return CameraComputeJob( + job_id=root.name, + job_root=root, + manifest_path=root / "job.json", + session_id=SOURCE_SESSION_ID, + source_id="sensor.camera.right", + codec_epoch=1, + input_sha256=input_sha256, + input_byte_length=16, + segment_count=1, + timeline_start_seconds=1.0, + timeline_end_seconds=2.0, + ) + + +def _queue_job( + tmp_path: Path, + definition: PortableRunDefinition, + bundle_sha256: str, + capability_sha256: str, +) -> tuple[ObservatoryRecordedJobQueue, ObservatoryRecordedJob, str]: + recorded = RecordedRunDefinition( + setup_id=definition.setup_id, + definition_id=definition.definition_id, + definition_version=definition.version, + definition_sha256=definition.definition_sha256, + source_adapter_id=definition.source_adapter.adapter_id, + source_adapter_version=definition.source_adapter.version, + source_adapter_sha256=definition.source_adapter.contract_sha256, + executor_release_id="lab-v1-portable-test-release", + executor_release_sha256=EXECUTOR_RELEASE_SHA256, + executor_image_sha256=EXECUTOR_IMAGE_SHA256, + model_release_ids=definition.learned_models, + model_manifest_sha256=definition.model_manifest_sha256, + resource_profile_id=definition.resource_profile.profile_id, + resource_profile_sha256=definition.resource_profile.profile_sha256, + checkpoint_policy=definition.resource_profile.checkpoint_policy, + allowed_checkpoints=definition.resource_profile.allowed_checkpoints, + ) + queue = ObservatoryRecordedJobQueue( + tmp_path / "queue", + definitions=RecordedRunDefinitionRegistry((recorded,)), + clock=lambda: NOW, + ) + job, created = queue.submit( + ObservatoryRecordedJobIntent( + idempotency_key="portable-lab-v1-test-001", + source_session_id=SOURCE_SESSION_ID, + source_catalog_sha256=CATALOG_SHA256, + source_bundle_sha256=bundle_sha256, + source_capability_manifest_sha256=capability_sha256, + setup_id=definition.setup_id, + definition_sha256=definition.definition_sha256, + ), + enqueue=True, + ) + assert created is True + claim = queue.claim_next( + claimant_id="worker-006", + claim_request_id="portable-lab-v1-claim-001", + ) + assert claim is not None + running = queue.start(job.job_id, claim_token=claim.claim_token) + return queue, running, claim.claim_token + + +def _sealed(job: ObservatoryRecordedJob) -> SealedObservatoryRecordedJob: + return SealedObservatoryRecordedJob( + job_id=job.job_id, + request_sha256=job.request_sha256, + identity_sha256=job.identity_sha256, + submission_receipt_sha256=job.submission_receipt_sha256, + source_session_id=job.source_session_id, + source_catalog_sha256=job.source_catalog_sha256, + source_bundle_sha256=job.source_bundle_sha256, + source_capability_manifest_sha256=job.source_capability_manifest_sha256, + source_adapter_id=job.source_adapter_id, + source_adapter_version=job.source_adapter_version, + source_adapter_sha256=job.source_adapter_sha256, + setup_id=job.setup_id, + definition_id=job.definition_id, + definition_version=job.definition_version, + definition_sha256=job.definition_sha256, + executor_release_id=job.executor_release_id, + executor_identity=ObservatoryWorkerExecutorIdentity( + release_sha256=job.executor_release_sha256, + image_sha256=job.executor_image_sha256, + model_manifest_sha256=job.model_manifest_sha256, + resource_profile_sha256=job.resource_profile_sha256, + ), + model_release_ids=job.model_release_ids, + resource_profile_id=job.resource_profile_id, + checkpoint_policy=job.checkpoint_policy, + allowed_checkpoints=job.allowed_checkpoints, + claim_generation=job.claim_generation, + claim_claimed_at_utc=job.claimed_at_utc, + claim_expires_at_utc=job.claim_expires_at_utc, + claim_heartbeat_at_utc=job.claim_heartbeat_at_utc, + claim_renewal_count=job.claim_renewal_count, + restart_from_zero=job.restart_from_zero, + ) + + +def _contracts( + tmp_path: Path, + *, + admit_plan_for_contract_test: bool = False, +) -> tuple[ + PortableRunDefinition, + ObservatoryRecordedJobQueue, + ObservatoryRecordedJob, + str, + PortableLabV1SourceInput, + PortableLabV1ReleaseCandidate, + PortableLabV1ReleaseInspection, + PortableLabV1OrchestrationPlan, +]: + definition = _definition() + bundle, capability = _source_documents(definition) + queue, running, claim_token = _queue_job( + tmp_path, + definition, + hashlib.sha256(bundle).hexdigest(), + hashlib.sha256(capability).hexdigest(), + ) + sealed = _sealed(running) + source = materialize_lab_v1_source_input( + job=sealed, + definition=definition, + camera_job=_camera_job(tmp_path), + source_bundle_bytes=bundle, + capability_bytes=capability, + ) + release = PortableLabV1ReleaseCandidate.from_file( + RELEASE_PATH, + repository_root=REPOSITORY_ROOT, + ) + inspection = release.inspect() + plan_inspection = ( + PortableLabV1ReleaseInspection( + candidate_sha256=release.candidate_sha256, + matched_assets=tuple(asset.asset_id for asset in release.assets), + blockers=(), + ready=True, + ) + if admit_plan_for_contract_test + else inspection + ) + legacy_config = json.loads(PORTABLE_DDRNET_CONFIG.read_text(encoding="utf-8")) + plan = PortableLabV1OrchestrationPlan.create( + job=sealed, + definition=definition, + source=source, + release=release, + release_inspection=plan_inspection, + legacy_ddrnet_config=legacy_config, + ) + return definition, queue, running, claim_token, source, release, inspection, plan + + +def _write(path: Path, payload: bytes) -> dict[str, object]: + path.write_bytes(payload) + return { + "path": path.name, + "byte_length": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + + +def _component_outputs( + tmp_path: Path, + definition: PortableRunDefinition, + plan: PortableLabV1OrchestrationPlan, +) -> tuple[Path, Path]: + eomt_root = tmp_path / "eomt" + eomt_root.mkdir() + kinds = ( + ("panoptic-overlay-video", "video/mp4", "perception.mp4"), + ("panoptic-mask-archive", "application/gzip", "semantic-masks.tar.gz"), + ("panoptic-frame-metadata", "application/x-ndjson", "frames.jsonl"), + ("worker-gpu-telemetry", "application/x-ndjson", "gpu-telemetry.jsonl"), + ("perception-run-report", "application/json", "run-report.json"), + ) + eomt_artifacts = [] + for index, (kind, media_type, name) in enumerate(kinds): + record = _write(eomt_root / name, f"eomt-{index}".encode()) + eomt_artifacts.append({"kind": kind, "media_type": media_type, **record}) + _write( + eomt_root / "decode-repair.json", + _canonical( + { + "schema_version": "missioncore.recorded-video-decode-repair/v1", + "decoder": "fixture", + "packets_requested": 1, + "frames_decoded": 1, + "repaired_frame_count": 0, + "repairs": [], + } + ), + ) + eomt_model = next( + model for model in definition.models if model.release_id == "eomt-cityscapes-large-1024-v1" + ) + eomt_identity = { + "schema_version": "missioncore.recorded-perception-identity/v2", + "job_id": plan.source_input.camera_job_id, + "input_sha256": plan.source_input.camera_input_sha256, + "calibration": {}, + "configuration": {"pipeline": "recorded-semantic-eomt-fisheye-mask/v1"}, + "models": { + "semantic": { + "id": eomt_model.model_id, + "revision": eomt_model.revision, + "architecture": eomt_model.architecture, + } + }, + "publication": {}, + } + eomt_identity_sha256 = hashlib.sha256(_canonical(eomt_identity)).hexdigest() + eomt = { + "schema_version": "missioncore.recorded-perception-result/v2", + "result_id": f"result-{eomt_identity_sha256}", + "identity_sha256": eomt_identity_sha256, + "identity": eomt_identity, + "created_at_utc": NOW, + "job_id": plan.source_input.camera_job_id, + "input_sha256": plan.source_input.camera_input_sha256, + "session_id": SOURCE_SESSION_ID, + "source_id": "sensor.camera.right", + "codec_epoch": 1, + "timestamp_basis": "session-time-seconds", + "timeline_start_seconds": 1.0, + "timeline_end_seconds": 2.0, + "frames_processed": 1, + "ground_truth": False, + "artifacts": eomt_artifacts, + } + (eomt_root / "result.json").write_text( + json.dumps(eomt, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + ddrnet_root = tmp_path / "ddrnet" + ddrnet_root.mkdir() + archive = _write(ddrnet_root / "semantic-masks.zip", b"fixture-zip") + _write( + ddrnet_root / "decode-repair.json", + _canonical( + { + "schema_version": "missioncore.recorded-video-decode-repair/v1", + "decoder": "fixture", + "packets_requested": 1, + "frames_decoded": 1, + "repaired_frame_count": 0, + "repairs": [], + } + ), + ) + ddrnet_model = next( + model + for model in definition.models + if model.release_id == "lab-v1-ddrnet-39-goose-fine-64-v1" + ) + checkpoint = ddrnet_model.artifacts[0] + effective_source = plan.effective_ddrnet_config["ravnoves"] + candidate = { + "candidate_id": "goose-ddrnet-class-512", + "candidate_key": "ddrnet", + "loaded_model_name": "ddrnet_39", + "architecture_probe_failures": [], + "checkpoint_size_bytes": checkpoint.byte_length, + "checkpoint_sha256": checkpoint.sha256, + } + ddr_source = { + "source_id": effective_source["source_id"], # type: ignore[index] + "input_count": 1, + "ground_truth_available": False, + "mapping_sha256": "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f", + } + video_semantics = { + "base_m4_result_id": None, + "mask_archive": { + **archive, + "media_type": "application/zip", + "frame_count": 1, + "width": 800, + "height": 600, + "encoding": "uint8-class-id-png", + "sequence_binding": "sequence-0-to-masks/frame-000001.png", + }, + "taxonomy": {"schema_version": "fixture", "classes": []}, + "aggregate_prediction_pixels": [0] * 64, + "center_crop_xyxy": [100, 0, 700, 600], + "outside_crop_state": "undefined", + } + authority = { + "navigation_accepted": False, + "safety_accepted": False, + "actuation_accepted": False, + "camera_semantics_can_clear_rigid_geometry": False, + } + provenance = { + "config_sha256": plan.effective_ddrnet_config_sha256, + "policy_sha256": "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35", + "provider_map_sha256": "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352", + "hostname": "fixture", + "pid": 1, + } + ddrnet = { + "schema_version": "missioncore.lab-v1-goose-vegetation-run/v1", + "lab_id": "LAB-V1", + "worker_id": "worker-006", + "mode": "ravnoves-video", + "candidate": candidate, + "source": ddr_source, + "video_semantics": video_semantics, + "preprocessing": ["fixture"], + "metrics": {}, + "timing": {}, + "resource": {}, + "visual_cases": [], + "authority": authority, + "provenance": provenance, + } + identity_value = { + "schema_version": ddrnet["schema_version"], + "candidate": candidate, + "source": ddr_source, + "video_semantics": video_semantics, + "preprocessing": ddrnet["preprocessing"], + "metrics": ddrnet["metrics"], + "timing": ddrnet["timing"], + "resource": ddrnet["resource"], + "visual_cases": ddrnet["visual_cases"], + "authority": authority, + "config_sha256": provenance["config_sha256"], + "policy_sha256": provenance["policy_sha256"], + "provider_map_sha256": provenance["provider_map_sha256"], + } + ddrnet["result_id"] = ( + "lab-v1-ravnoves-video-ddrnet-" + hashlib.sha256(_canonical(identity_value)).hexdigest() + ) + (ddrnet_root / "result.json").write_text( + json.dumps(ddrnet, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return eomt_root, ddrnet_root + + +def _regenerated_definition_for_camera( + init_payload: bytes, + *, + version: int, +) -> PortableRunDefinition: + """Regenerate an exact definition identity without relying on v1's init bytes.""" + + definition = _definition() + source_requirements = replace( + definition.source_requirements, + recorded_media_init_sha256=hashlib.sha256(init_payload).hexdigest(), + ) + source_adapter = replace( + definition.source_adapter, + contract_sha256=canonical_sha256( + definition.source_adapter.identity_document(source_requirements) + ), + ) + identity = definition.identity_document() + identity["version"] = version + identity["source_requirements"] = source_requirements.as_dict() + identity["source_adapter"] = source_adapter.as_dict() + return replace( + definition, + version=version, + source_requirements=source_requirements, + source_adapter=source_adapter, + definition_sha256=canonical_sha256(identity), + ) + + +def _source_documents_for_camera( + definition: PortableRunDefinition, + *, + init_payload: bytes, + segment_payload: bytes, +) -> tuple[bytes, bytes]: + bundle_bytes, capability_bytes = _source_documents(definition) + bundle = json.loads(bundle_bytes) + camera_epoch = bundle["camera"]["epoch"] + camera_epoch["init"] = { + "byte_length": len(init_payload), + "sha256": hashlib.sha256(init_payload).hexdigest(), + } + camera_epoch["segments"] = [ + { + "sequence": 1, + "byte_length": len(segment_payload), + "sha256": hashlib.sha256(segment_payload).hexdigest(), + "random_access": True, + "end_time_seconds": 2.0, + } + ] + bundle["spatial_replay"] = { + "primary_artifact_id": "raw-primary", + "members": [ + { + "artifact_id": "raw-primary", + "media_type": "application/x-nodedc-k1mqtt", + "byte_length": len(SPATIAL_REPLAY_PAYLOAD), + "replay_byte_length": len(SPATIAL_REPLAY_PAYLOAD), + "sha256": hashlib.sha256(SPATIAL_REPLAY_PAYLOAD).hexdigest(), + }, + { + "artifact_id": "raw-transport-index", + "media_type": "application/x-ndjson", + "byte_length": len(SPATIAL_METADATA_PAYLOAD), + "replay_byte_length": len(SPATIAL_METADATA_PAYLOAD), + "sha256": hashlib.sha256(SPATIAL_METADATA_PAYLOAD).hexdigest(), + }, + ], + "timeline_origin_epoch_ns": 1, + "timeline_origin_monotonic_ns": 2, + } + bundle_bytes = _canonical(bundle) + capability = json.loads(capability_bytes) + capability["source_bundle_sha256"] = hashlib.sha256(bundle_bytes).hexdigest() + capability["camera_profile"]["init_sha256"] = hashlib.sha256( + init_payload + ).hexdigest() + capability_bytes = _canonical(capability) + return bundle_bytes, capability_bytes + + +def _worker_source_member( + job: SealedObservatoryRecordedJob, + *, + kind: str, + payload: bytes, + media_type: str, + artifact_id: str | None = None, + primary: bool = False, + camera_epoch: int | None = None, + camera_sequence: int | None = None, +) -> dict[str, object]: + payload_sha256 = hashlib.sha256(payload).hexdigest() + identity = { + "job_identity_sha256": job.identity_sha256, + "source_bundle_sha256": job.source_bundle_sha256, + "kind": kind, + "artifact_id": artifact_id, + "primary": primary, + "camera_epoch": camera_epoch, + "camera_sequence": camera_sequence, + "media_type": media_type, + "byte_length": len(payload), + "sha256": payload_sha256, + } + return { + "member_id": hashlib.sha256(canonical_json(identity)).hexdigest(), + "kind": kind, + "media_type": media_type, + "byte_length": len(payload), + "sha256": payload_sha256, + "artifact_id": artifact_id, + "primary": primary, + "camera_epoch": camera_epoch, + "camera_sequence": camera_sequence, + } + + +def _shared_worker_source_stage( + tmp_path: Path, + *, + job: SealedObservatoryRecordedJob, + bundle_bytes: bytes, + capability_bytes: bytes, + init_payload: bytes, + segment_payload: bytes, +) -> PortableWorkerSourceStage: + stage = tmp_path / "shared-worker-source" + (stage / "camera" / "epoch-1" / "segments").mkdir(parents=True) + members = [ + _worker_source_member( + job, + kind="source-bundle", + payload=bundle_bytes, + media_type="application/json", + ), + _worker_source_member( + job, + kind="source-capability", + payload=capability_bytes, + media_type="application/json", + ), + _worker_source_member( + job, + kind="spatial-replay", + payload=SPATIAL_REPLAY_PAYLOAD, + media_type="application/x-nodedc-k1mqtt", + artifact_id="raw-primary", + primary=True, + ), + _worker_source_member( + job, + kind="spatial-replay-metadata", + payload=SPATIAL_METADATA_PAYLOAD, + media_type="application/x-ndjson", + artifact_id="raw-transport-index", + ), + _worker_source_member( + job, + kind="camera-init", + payload=init_payload, + media_type='video/mp4; codecs="avc1.641028"', + artifact_id="camera-recording", + camera_epoch=1, + ), + _worker_source_member( + job, + kind="camera-segment", + payload=segment_payload, + media_type="video/iso.segment", + artifact_id="camera-recording", + camera_epoch=1, + camera_sequence=1, + ), + ] + members.sort(key=lambda row: str(row["member_id"])) + manifest = { + "schema_version": PORTABLE_SOURCE_MATERIALIZATION_SCHEMA, + "job_id": job.job_id, + "job_identity_sha256": job.identity_sha256, + "claim_generation": job.claim_generation, + "source": { + "session_id": job.source_session_id, + "bundle_sha256": job.source_bundle_sha256, + "capability_manifest_sha256": ( + job.source_capability_manifest_sha256 + ), + }, + "members": members, + "authority": OBSERVATION_ONLY_AUTHORITY, + } + (stage / "source-bundle.json").write_bytes(bundle_bytes) + (stage / "source-capability.json").write_bytes(capability_bytes) + (stage / "materialization-manifest.json").write_bytes(_canonical(manifest)) + (stage / "mqtt.raw.k1mqtt").write_bytes(SPATIAL_REPLAY_PAYLOAD) + (stage / "mqtt.metadata.jsonl").write_bytes(SPATIAL_METADATA_PAYLOAD) + (stage / "camera" / "epoch-1" / "init.mp4").write_bytes(init_payload) + (stage / "camera" / "epoch-1" / "segments" / "1.m4s").write_bytes( + segment_payload + ) + return PortableWorkerSourceStage( + root=stage, + source_bundle_sha256=job.source_bundle_sha256, + source_capability_manifest_sha256=( + job.source_capability_manifest_sha256 + ), + source_adapter_sha256=job.source_adapter_sha256, + ) + + +def _release_for_definition( + definition: PortableRunDefinition, +) -> PortableLabV1ReleaseCandidate: + release = PortableLabV1ReleaseCandidate.from_file( + RELEASE_PATH, + repository_root=REPOSITORY_ROOT, + ) + portable_config_bytes = PORTABLE_DDRNET_CONFIG.read_bytes() + assets = tuple( + sorted( + ( + replace( + asset, + asset_id="ddrnet-portable-config", + sha256=hashlib.sha256(portable_config_bytes).hexdigest(), + byte_length=len(portable_config_bytes), + repository_path=( + "config/perception/lab-v1-eomt-ddrnet-portable-v2.json" + ), + ) + if asset.asset_id == "ddrnet-legacy-config" + else asset + for asset in release.assets + ), + key=lambda asset: asset.asset_id, + ) + ) + identity = release.identity_document() + identity["definition_version"] = definition.version + identity["definition_sha256"] = definition.definition_sha256 + identity["executor_image_sha256"] = "f" * 64 + identity["assets"] = [asset.as_dict() for asset in assets] + identity["declared_blockers"] = [] + return replace( + release, + definition_version=definition.version, + definition_sha256=definition.definition_sha256, + executor_image_sha256="f" * 64, + assets=assets, + declared_blockers=(), + candidate_sha256=canonical_sha256(identity), + ) + + +def test_release_candidate_matches_repository_but_stays_honestly_blocked( + tmp_path: Path, +) -> None: + release = PortableLabV1ReleaseCandidate.from_file( + RELEASE_PATH, + repository_root=REPOSITORY_ROOT, + ) + release.bind_definition(_definition()) + inspection = release.inspect() + + repository_assets = { + asset.asset_id for asset in release.assets if asset.repository_path is not None + } + assert set(inspection.matched_assets) == repository_assets + assert inspection.ready is False + assert "executor-image-unsealed" in inspection.blockers + assert "commit-bound-source-unavailable" in inspection.blockers + assert "combined-executor-image-unsealed" in inspection.blockers + assert "asset-ddrnet-checkpoint-missing" in inspection.blockers + assert "asset-eomt-model-weights-missing" in inspection.blockers + with pytest.raises(Exception, match="not sealable"): + release.seal(inspection) + forged_ready = PortableLabV1ReleaseInspection( + candidate_sha256=release.candidate_sha256, + matched_assets=tuple(asset.asset_id for asset in release.assets), + blockers=(), + ready=True, + ) + with pytest.raises( + PortableLabV1WorkerError, + match="release inspection is not fully admitted", + ): + PortableLabV1RunnerInstallation( + release=release, + inspection=forged_ready, + portable_ddrnet_config_path=PORTABLE_DDRNET_CONFIG, + output_parent=tmp_path / "must-not-install", + ) + + +def test_source_materialization_and_effective_config_are_recording_independent( + tmp_path: Path, +) -> None: + definition, _queue, _job, _token, source, _release, _inspection, plan = _contracts( + tmp_path + ) + effective = build_portable_ddrnet_effective_config( + json.loads(PORTABLE_DDRNET_CONFIG.read_text(encoding="utf-8")), + source=source, + ) + + assert source.frame_count == 1 + assert effective == plan.effective_ddrnet_config + assert effective["ravnoves"]["expected_frame_count"] == 1 # type: ignore[index] + assert effective["ravnoves"]["source_sha256"] == source.camera_input_sha256 # type: ignore[index] + assert effective["ravnoves"]["base_m4_result_id"] is None # type: ignore[index] + assert plan.executable is False + with pytest.raises(PortableLabV1PlanError, match="release is blocked"): + plan.require_executable() + assert definition.result_contract.result_schema == PORTABLE_LAB_V1_RESULT_SCHEMA + + +def test_plan_rejects_rebound_source_and_incomplete_ready_inspection( + tmp_path: Path, +) -> None: + definition, _queue, running, _token, source, release, inspection, _plan = ( + _contracts(tmp_path) + ) + sealed = _sealed(running) + legacy_config = json.loads(PORTABLE_DDRNET_CONFIG.read_text(encoding="utf-8")) + + with pytest.raises(PortableLabV1PlanError, match="differs from the sealed job"): + PortableLabV1OrchestrationPlan.create( + job=sealed, + definition=definition, + source=replace(source, source_catalog_sha256="f" * 64), + release=release, + release_inspection=inspection, + legacy_ddrnet_config=legacy_config, + ) + + incomplete = PortableLabV1ReleaseInspection( + candidate_sha256=release.candidate_sha256, + matched_assets=tuple(asset.asset_id for asset in release.assets[:-1]), + blockers=(), + ready=True, + ) + with pytest.raises(PortableLabV1PlanError, match="every exact asset"): + PortableLabV1OrchestrationPlan.create( + job=sealed, + definition=definition, + source=source, + release=release, + release_inspection=incomplete, + legacy_ddrnet_config=legacy_config, + ) + + +def test_result_v2_assembly_package_and_exact_validator(tmp_path: Path) -> None: + definition, queue, running, claim_token, _source, _release, _inspection, plan = ( + _contracts(tmp_path, admit_plan_for_contract_test=True) + ) + eomt_root, ddrnet_root = _component_outputs(tmp_path, definition, plan) + assembly = assemble_lab_v1_result_v2( + plan=plan, + definition=definition, + eomt_result_root=eomt_root, + ddrnet_result_root=ddrnet_root, + output_parent=tmp_path / "assemblies", + ) + + draft = package_lab_v1_result( + assembly=assembly, + plan=plan, + job=running, + definition=definition, + created_at_utc=NOW, + output_parent=tmp_path / "packages", + ) + succeeded = queue.succeed( + running.job_id, + claim_token=claim_token, + result_id=draft.result_id, + result_sha256=draft.result_sha256, + ) + package = PortableResultPackageManifest.from_bytes( + (draft.root / "manifest.json").read_bytes() + ) + paths = { + artifact.role: draft.root / artifact.relative_path for artifact in package.artifacts + } + result_document = json.loads(paths["result-document"].read_text(encoding="utf-8")) + context = PortableResultValidationContext( + manifest=package, + job=succeeded, + definition=definition, + result_document=result_document, + artifact_paths=paths, + ) + + validate_lab_v1_result_v2(context) + assert draft.result_id == assembly.result_id + assert draft.result_sha256 == package.manifest_sha256 + assert result_document["components"]["eomt"]["frames_processed"] == 1 + assert result_document["components"]["ddrnet"]["frames_processed"] == 1 + + changed = json.loads(paths["ddrnet-result-document"].read_text(encoding="utf-8")) + changed["source"]["input_count"] = 2 + paths["ddrnet-result-document"].write_bytes(canonical_json(changed)) + with pytest.raises(PortableLabV1ResultError, match="artifact content changed"): + validate_lab_v1_result_v2(context) + + +def test_result_assembler_rejects_component_model_drift(tmp_path: Path) -> None: + definition, _queue, _job, _token, _source, _release, _inspection, plan = ( + _contracts(tmp_path, admit_plan_for_contract_test=True) + ) + eomt_root, ddrnet_root = _component_outputs(tmp_path, definition, plan) + ddrnet = json.loads((ddrnet_root / "result.json").read_text(encoding="utf-8")) + ddrnet["candidate"]["checkpoint_sha256"] = "f" * 64 + (ddrnet_root / "result.json").write_text(json.dumps(ddrnet), encoding="utf-8") + + with pytest.raises(PortableLabV1ResultError, match="DDRNet component contract"): + assemble_lab_v1_result_v2( + plan=plan, + definition=definition, + eomt_result_root=eomt_root, + ddrnet_result_root=ddrnet_root, + output_parent=tmp_path / "assemblies", + ) + + +def test_release_manifest_is_digest_fenced() -> None: + document = json.loads(RELEASE_PATH.read_text(encoding="utf-8")) + release = PortableLabV1ReleaseCandidate.from_file( + RELEASE_PATH, + repository_root=REPOSITORY_ROOT, + ) + + assert document["executor_image_sha256"] is None + with pytest.raises(Exception, match="identity digest changed"): + replace(release, candidate_sha256="f" * 64) + with pytest.raises(Exception, match="identity digest changed"): + replace(release, declared_blockers=()) + + +def test_shared_worker_stage_materializes_a_version_bound_camera_job( + tmp_path: Path, +) -> None: + init_payload = b"portable-init" + segment_payload = b"portable-segment" + definition = _regenerated_definition_for_camera(init_payload, version=2) + bundle_bytes, capability_bytes = _source_documents_for_camera( + definition, + init_payload=init_payload, + segment_payload=segment_payload, + ) + _queue, running, _claim_token = _queue_job( + tmp_path, + definition, + hashlib.sha256(bundle_bytes).hexdigest(), + hashlib.sha256(capability_bytes).hexdigest(), + ) + sealed = _sealed(running) + worker_stage = _shared_worker_source_stage( + tmp_path, + job=sealed, + bundle_bytes=bundle_bytes, + capability_bytes=capability_bytes, + init_payload=init_payload, + segment_payload=segment_payload, + ) + + materialized = materialize_lab_v1_source_from_worker_stage( + worker_stage=worker_stage, + job=sealed, + definition=definition, + output_parent=tmp_path / "lab-v1-sources", + ) + repeated = materialize_lab_v1_source_from_worker_stage( + worker_stage=worker_stage, + job=sealed, + definition=definition, + output_parent=tmp_path / "lab-v1-sources", + ) + + assert definition.version == 2 + assert materialized.descriptor.observatory_job_id == sealed.job_id + assert materialized.descriptor.source_bundle_sha256 == sealed.source_bundle_sha256 + assert materialized.descriptor.camera_job_id == repeated.descriptor.camera_job_id + assert materialized.descriptor.camera_input_sha256 == ( + repeated.descriptor.camera_input_sha256 + ) + assert ( + materialized.camera_job_root + / "input" + / "camera" + / "sensor.camera.right" + / "epoch-1" + / "init.mp4" + ).read_bytes() == init_payload + assert ( + materialized.camera_job_root + / "input" + / "camera" + / "sensor.camera.right" + / "epoch-1" + / "segments" + / "1.m4s" + ).read_bytes() == segment_payload + + manifest_path = worker_stage.root / "materialization-manifest.json" + changed = json.loads(manifest_path.read_bytes()) + changed["claim_generation"] = sealed.claim_generation + 1 + manifest_path.write_bytes(_canonical(changed)) + with pytest.raises( + PortableLabV1SourceError, + match="belongs to another sealed LAB V1 job", + ): + materialize_lab_v1_source_from_worker_stage( + worker_stage=worker_stage, + job=sealed, + definition=definition, + output_parent=tmp_path / "other-lab-v1-sources", + ) + + +def test_profile_runner_sequences_exact_components_and_packages_sealed_job( + tmp_path: Path, +) -> None: + init_payload = b"runner-portable-init" + segment_payload = b"runner-portable-segment" + definition = _regenerated_definition_for_camera(init_payload, version=2) + bundle_bytes, capability_bytes = _source_documents_for_camera( + definition, + init_payload=init_payload, + segment_payload=segment_payload, + ) + _queue, running, _claim_token = _queue_job( + tmp_path, + definition, + hashlib.sha256(bundle_bytes).hexdigest(), + hashlib.sha256(capability_bytes).hexdigest(), + ) + sealed = _sealed(running) + worker_stage = _shared_worker_source_stage( + tmp_path, + job=sealed, + bundle_bytes=bundle_bytes, + capability_bytes=capability_bytes, + init_payload=init_payload, + segment_payload=segment_payload, + ) + materialized = materialize_lab_v1_source_from_worker_stage( + worker_stage=worker_stage, + job=sealed, + definition=definition, + output_parent=tmp_path / "lab-v1-sources", + ) + bound_source = PortableLabV1BoundSourceStage( + root=materialized.root, + source_bundle_sha256=sealed.source_bundle_sha256, + source_capability_manifest_sha256=( + sealed.source_capability_manifest_sha256 + ), + source_adapter_sha256=sealed.source_adapter_sha256, + job=sealed, + lab_source=materialized, + ) + release = _release_for_definition(definition) + ready_inspection = PortableLabV1ReleaseInspection( + candidate_sha256=release.candidate_sha256, + matched_assets=tuple(asset.asset_id for asset in release.assets), + blockers=(), + ready=True, + ) + plan = PortableLabV1OrchestrationPlan.create( + job=sealed, + definition=definition, + source=materialized.descriptor, + release=release, + release_inspection=ready_inspection, + legacy_ddrnet_config=json.loads( + PORTABLE_DDRNET_CONFIG.read_text(encoding="utf-8") + ), + ) + plan.require_executable() + installation = PortableLabV1RunnerInstallation( + release=release, + inspection=ready_inspection, + portable_ddrnet_config_path=PORTABLE_DDRNET_CONFIG, + output_parent=tmp_path / "portable-worker-output", + ) + fixture_parent = tmp_path / "component-fixtures" + fixture_parent.mkdir() + fixture_eomt, fixture_ddrnet = _component_outputs( + fixture_parent, + definition, + plan, + ) + calls: list[str] = [] + + def run_eomt( + *, + source: PortableLabV1MaterializedSource, + plan: PortableLabV1OrchestrationPlan, + output_root: Path, + ) -> None: + assert source.descriptor.camera_input_sha256 == plan.source_input.camera_input_sha256 + calls.append("eomt") + shutil.copytree(fixture_eomt, output_root) + + def run_ddrnet( + *, + source: PortableLabV1MaterializedSource, + plan: PortableLabV1OrchestrationPlan, + effective_config_path: Path, + eomt_result_root: Path, + output_root: Path, + ) -> None: + assert source.descriptor.camera_input_sha256 == plan.source_input.camera_input_sha256 + assert calls == ["eomt"] + assert (eomt_result_root / "result.json").is_file() + assert hashlib.sha256(effective_config_path.read_bytes()).hexdigest() == ( + plan.effective_ddrnet_config_sha256 + ) + calls.append("ddrnet") + shutil.copytree(fixture_ddrnet, output_root) + + runner = PortableLabV1ProfileRunnerAdapter( + definition=definition, + installation=installation, + created_at_utc=lambda: NOW, + eomt_runner=run_eomt, + ddrnet_runner=run_ddrnet, + ) + runtime_plan = PortableWorkerRuntimePlan( + job_id=sealed.job_id, + adapter_id="lab-v1-eomt-ddrnet-worker006-v1", + candidate_sha256="f" * 64, + setup_id=sealed.setup_id, + definition_sha256=sealed.definition_sha256, + source_bundle_sha256=sealed.source_bundle_sha256, + source_capability_manifest_sha256=( + sealed.source_capability_manifest_sha256 + ), + result_contract_sha256=definition.result_contract.contract_sha256, + phases=PORTABLE_LAB_V1_RUNTIME_PHASES, + ) + with pytest.raises( + PortableWorkerRuntimeJobRejectedError, + match="runtime plan differs", + ): + runner.run(replace(runtime_plan, phases=("source-delivery",)), bound_source) + assert calls == [] + + draft = runner.run(runtime_plan, bound_source) + package = PortableResultPackageManifest.from_bytes( + (draft.root / "manifest.json").read_bytes() + ) + + assert calls == ["eomt", "ddrnet"] + assert draft.result_id.startswith("lab-v1-eomt-ddrnet-") + assert draft.result_sha256 == package.manifest_sha256 + assert draft.result_contract_sha256 == definition.result_contract.contract_sha256 + assert package.job["submission_receipt_sha256"] == sealed.submission_receipt_sha256 + assert package.job["claim_generation"] == sealed.claim_generation + assert not tuple(installation.output_parent.glob(".lab-v1-portable-run-*")) diff --git a/tests/test_observatory_portable_queue_binding.py b/tests/test_observatory_portable_queue_binding.py index a3b0913..989b1c2 100644 --- a/tests/test_observatory_portable_queue_binding.py +++ b/tests/test_observatory_portable_queue_binding.py @@ -367,6 +367,34 @@ def test_not_installed_definition_fails_before_source_or_queue_writes( assert not (tmp_path / "observatory-recorded-jobs.sqlite3").exists() +def test_not_installed_model_free_definition_remains_capability_probeable( + tmp_path: Path, +) -> None: + registry = _blocked_registry() + _write_probe_summary(tmp_path, segment_count=17) + service, store, queue, inspector = _service( + tmp_path, + registry=registry, + with_queue=False, + ) + definition = registry.resolve_setup("m49-tgs-portable-v2") + + capability = service.probe( + source_session_id=SESSION_ID, + setup_id=definition.setup_id, + definition_sha256=definition.definition_sha256, + ) + + assert capability.source_session_id == SESSION_ID + assert capability.camera_segment_count == 17 + assert capability.source_adapter_sha256 == definition.source_adapter.contract_sha256 + assert store.catalog_reads == 1 + assert store.prepare_replay_calls == 0 + assert inspector.restore_calls == 0 + assert queue is None + assert not (tmp_path / PORTABLE_SOURCE_DOCUMENT_DIRECTORY).exists() + + def test_probe_is_bounded_and_does_not_enter_replay_or_media_inspector( tmp_path: Path, ) -> None: diff --git a/tests/test_observatory_portable_result_publisher.py b/tests/test_observatory_portable_result_publisher.py new file mode 100644 index 0000000..3f7a31b --- /dev/null +++ b/tests/test_observatory_portable_result_publisher.py @@ -0,0 +1,595 @@ +from __future__ import annotations + +import copy +import hashlib +import json +from collections.abc import Callable +from dataclasses import replace +from pathlib import Path +from typing import cast + +import pytest + +from k1link.artifact_gateway import CentralArtifactStore +from k1link.observatory.portable_result_contract import ( + OBSERVATORY_CALCULATION_PROFILE_SCHEMA, + PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA, + RESULT_DOCUMENT_ROLE, + PortableCalculationProfilePolicy, + PortableCalculationProfileRegistry, + PortableResultArtifact, + PortableResultContractValidatorRegistration, + PortableResultContractValidatorRegistry, + PortableResultPackageIntegrityError, + PortableResultPackageManifest, + PortableResultPublicationBlockedError, + PortableResultValidationContext, +) +from k1link.observatory.portable_result_publisher import ( + PortableObservatoryResultPublisher, + resolve_published_portable_calculation_profile, +) +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinition, + PortableRunDefinitionRegistry, + canonical_sha256, +) +from k1link.observatory.recorded_jobs import ( + ObservatoryRecordedJob, + ObservatoryRecordedJobIntent, + ObservatoryRecordedJobQueue, + RecordedRunDefinitionRegistry, +) +from k1link.observatory.source_admission import ( + PORTABLE_SOURCE_BUNDLE_SCHEMA, + PORTABLE_SOURCE_CAPABILITY_SCHEMA, + PORTABLE_SOURCE_DOCUMENT_DIRECTORY, +) +from k1link.sessions import ( + ObservationArchiveSource, + ObservationSessionCandidate, + SessionStore, +) + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" +NOW = "2026-08-31T08:00:00.000Z" +SOURCE_SESSION_ID = "20260831T075500Z_viewer_live" +RESULT_ID = "portable-lab-result-001" +AUTHORITY = { + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, + "production_accepted": False, +} + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def _ready_registry(tmp_path: Path) -> PortableRunDefinitionRegistry: + base_definition = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH).definitions[0] + document = cast( + dict[str, object], + json.loads(REGISTRY_PATH.read_text(encoding="utf-8")), + ) + rows = cast(list[object], document["definitions"]) + selected = copy.deepcopy(cast(dict[str, object], rows[0])) + selected["executor"] = { + "contour_id": "worker-006", + "state": "ready", + "release_id": "lab-v1-portable-executor-v1", + "release_sha256": "1" * 64, + "image_sha256": "2" * 64, + "reason_code": None, + "reason": None, + } + identity = copy.deepcopy(base_definition.identity_document()) + identity["executor"] = { + "contour_id": "worker-006", + "state": "ready", + "release_id": "lab-v1-portable-executor-v1", + "release_sha256": "1" * 64, + "image_sha256": "2" * 64, + } + selected["definition_sha256"] = canonical_sha256(identity) + path = tmp_path / "portable-definitions.json" + path.write_bytes( + _canonical_json( + { + "schema_version": document["schema_version"], + "definitions": [selected], + } + ) + ) + return PortableRunDefinitionRegistry.from_file(path) + + +def _source_store( + tmp_path: Path, + definition: PortableRunDefinition, +) -> tuple[SessionStore, str, str, str]: + repository = tmp_path / "repository" + archive_root = tmp_path / "source-archive" + session_root = archive_root / SOURCE_SESSION_ID + session_root.mkdir(parents=True) + candidate = ObservationSessionCandidate( + session_id=SOURCE_SESSION_ID, + display_name="Portable result source", + status="ready", + started_at_utc="2026-08-31T07:55:00.000Z", + completed_at_utc="2026-08-31T07:59:00.000Z", + duration_seconds=240.0, + modalities=(), + replayable=False, + total_bytes=0, + allowed_root=archive_root, + session_root=session_root, + primary_replay_artifact_id=None, + timeline_origin_epoch_ns=None, + timeline_origin_monotonic_ns=None, + sources=(), + artifacts=(), + ) + archive = ObservationArchiveSource( + plugin_id=definition.source_requirements.plugin_id, + archive_id=definition.source_requirements.archive_id, + root=archive_root, + discover=lambda _root: (candidate,), + ) + store = SessionStore(repository, data_dir=tmp_path / "mission-core-data") + assert store.reconcile_archive(archive) == (SOURCE_SESSION_ID,) + _detail, catalog_sha256 = store.get_session_with_catalog_snapshot(SOURCE_SESSION_ID) + + source_adapter = { + "id": definition.source_adapter.adapter_id, + "version": definition.source_adapter.version, + "sha256": definition.source_adapter.contract_sha256, + } + bundle = { + "schema_version": PORTABLE_SOURCE_BUNDLE_SCHEMA, + "source_session_id": SOURCE_SESSION_ID, + "source_catalog_sha256": catalog_sha256, + "plugin_id": definition.source_requirements.plugin_id, + "archive_id": definition.source_requirements.archive_id, + "source_adapter": source_adapter, + "sources": [], + "spatial_replay": {}, + "camera": {}, + "authority": AUTHORITY, + } + bundle_bytes = _canonical_json(bundle) + bundle_sha256 = hashlib.sha256(bundle_bytes).hexdigest() + capability = { + "schema_version": PORTABLE_SOURCE_CAPABILITY_SCHEMA, + "source_session_id": SOURCE_SESSION_ID, + "source_catalog_sha256": catalog_sha256, + "source_bundle_sha256": bundle_sha256, + "source_adapter_sha256": definition.source_adapter.contract_sha256, + "modalities": [], + "camera_profile": {}, + "calibration": {}, + "authority": AUTHORITY, + } + capability_bytes = _canonical_json(capability) + capability_sha256 = hashlib.sha256(capability_bytes).hexdigest() + source_documents = store.data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY + source_documents.mkdir() + (source_documents / f"{bundle_sha256}.json").write_bytes(bundle_bytes) + (source_documents / f"{capability_sha256}.json").write_bytes(capability_bytes) + return store, catalog_sha256, bundle_sha256, capability_sha256 + + +def _running_job( + tmp_path: Path, + *, + definition: PortableRunDefinition, + catalog_sha256: str, + bundle_sha256: str, + capability_sha256: str, +) -> tuple[ObservatoryRecordedJobQueue, ObservatoryRecordedJob, str]: + recorded = definition.to_recorded_run_definition() + queue = ObservatoryRecordedJobQueue( + tmp_path / "mission-core-data", + definitions=RecordedRunDefinitionRegistry((recorded,)), + clock=lambda: NOW, + ) + job, created = queue.submit( + ObservatoryRecordedJobIntent( + idempotency_key="portable-result-publication-001", + source_session_id=SOURCE_SESSION_ID, + source_catalog_sha256=catalog_sha256, + source_bundle_sha256=bundle_sha256, + source_capability_manifest_sha256=capability_sha256, + setup_id=definition.setup_id, + definition_sha256=definition.definition_sha256, + ), + enqueue=True, + ) + assert created is True + claim = queue.claim_next( + claimant_id="worker-006", + claim_request_id="portable-result-claim-001", + ) + assert claim is not None + running = queue.start(job.job_id, claim_token=claim.claim_token) + assert running.state == "running" + return queue, running, claim.claim_token + + +def _package( + tmp_path: Path, + *, + job: ObservatoryRecordedJob, + definition: PortableRunDefinition, + result_id: str = RESULT_ID, + accepted: bool = True, +) -> tuple[Path, PortableResultPackageManifest]: + result_document = { + "schema_version": definition.result_contract.result_schema, + "result_id": result_id, + "result_kind": definition.result_contract.result_kind, + "accepted": accepted, + "authority": AUTHORITY, + } + result_bytes = _canonical_json(result_document) + artifact = PortableResultArtifact( + role=RESULT_DOCUMENT_ROLE, + relative_path="artifacts/result.json", + media_type="application/json", + byte_length=len(result_bytes), + sha256=hashlib.sha256(result_bytes).hexdigest(), + ) + package = PortableResultPackageManifest.create( + job=job, + definition=definition, + result_id=result_id, + created_at_utc=NOW, + artifacts=(artifact,), + ) + root = tmp_path / "packages" / package.manifest_sha256 + (root / "artifacts").mkdir(parents=True) + (root / "manifest.json").write_bytes(package.canonical_bytes) + (root / "artifacts" / "result.json").write_bytes(result_bytes) + return root, package + + +def _profile(definition: PortableRunDefinition) -> PortableCalculationProfilePolicy: + return PortableCalculationProfilePolicy( + setup_id=definition.setup_id, + definition_id=definition.definition_id, + definition_version=definition.version, + definition_sha256=definition.definition_sha256, + lab_id="LAB V1", + display_name="LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39", + ) + + +def _validator(context: PortableResultValidationContext) -> None: + expected = { + "schema_version": context.definition.result_contract.result_schema, + "result_id": context.job.result_id, + "result_kind": context.definition.result_contract.result_kind, + "accepted": True, + "authority": AUTHORITY, + } + if dict(context.result_document) != expected: + raise ValueError("result contract payload was not accepted") + + +def _publisher( + tmp_path: Path, + *, + store: SessionStore, + registry: PortableRunDefinitionRegistry, + profile: PortableCalculationProfilePolicy | None, + validator: Callable[[PortableResultValidationContext], None] | None, +) -> PortableObservatoryResultPublisher: + definition = registry.definitions[0] + registrations = ( + () + if validator is None + else ( + PortableResultContractValidatorRegistration( + contract_sha256=definition.result_contract.contract_sha256, + validator=validator, + ), + ) + ) + return PortableObservatoryResultPublisher( + session_store=store, + artifact_store=CentralArtifactStore(tmp_path / "central-artifacts", create=True), + definitions=registry, + calculation_profiles=PortableCalculationProfileRegistry( + () if profile is None else (profile,) + ), + validators=PortableResultContractValidatorRegistry(registrations), + ) + + +def _fixture( + tmp_path: Path, + *, + result_id: str = RESULT_ID, + accepted: bool = True, +) -> tuple[ + PortableRunDefinitionRegistry, + PortableRunDefinition, + SessionStore, + ObservatoryRecordedJob, + Path, +]: + registry = _ready_registry(tmp_path) + definition = registry.definitions[0] + store, catalog_sha, bundle_sha, capability_sha = _source_store(tmp_path, definition) + queue, running, claim_token = _running_job( + tmp_path, + definition=definition, + catalog_sha256=catalog_sha, + bundle_sha256=bundle_sha, + capability_sha256=capability_sha, + ) + package_root, package = _package( + tmp_path, + job=running, + definition=definition, + result_id=result_id, + accepted=accepted, + ) + succeeded = queue.succeed( + running.job_id, + claim_token=claim_token, + result_id=result_id, + result_sha256=package.manifest_sha256, + ) + return registry, definition, store, succeeded, package_root + + +def test_verified_package_publishes_immutable_binding_and_profile_provenance( + tmp_path: Path, +) -> None: + registry, definition, store, job, package_root = _fixture(tmp_path) + publisher = _publisher( + tmp_path, + store=store, + registry=registry, + profile=_profile(definition), + validator=_validator, + ) + + first = publisher.publish(job=job, package_root=package_root) + second = publisher.publish(job=job, package_root=package_root) + + assert second.binding == first.binding + assert second.artifact_manifest == first.artifact_manifest + assert first.binding.session_id == RESULT_ID + assert first.binding.source_session_id == SOURCE_SESSION_ID + assert first.binding.config_sha256 == definition.definition_sha256 + assert first.binding.replay_capability is None + assert first.binding.provenance["calculation_profile"] == { + "schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA, + "setup_id": definition.setup_id, + "display_name": "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39", + "origin": "archived-definition", + "definition_id": definition.definition_id, + "definition_version": definition.version, + "definition_sha256": definition.definition_sha256, + } + package_provenance = cast(dict[str, object], first.binding.provenance["result_package"]) + assert package_provenance["manifest_sha256"] == job.result_sha256 + assert package_provenance["artifact_manifest_id"] == first.artifact_manifest.manifest_id + assert store.get_lab_instance(RESULT_ID) == first.binding + assert store.get_session(SOURCE_SESSION_ID).summary.lab is None + + summary = store.get_session(RESULT_ID).summary + assert summary.display_name == ( + "Portable result source · полный маршрут и воспроизведение" + ) + profiles = PortableCalculationProfileRegistry((_profile(definition),)) + assert resolve_published_portable_calculation_profile( + summary, + definitions=registry, + calculation_profiles=profiles, + ) == _profile(definition).as_dict() + + assert summary.lab is not None + drifted_provenance = copy.deepcopy(summary.lab.provenance) + drifted_provenance["calculation_profile_sha256"] = "0" * 64 + drifted = replace( + summary, + lab=replace(summary.lab, provenance=drifted_provenance), + ) + assert ( + resolve_published_portable_calculation_profile( + drifted, + definitions=registry, + calculation_profiles=profiles, + ) + is None + ) + + +def test_unknown_result_contract_fails_before_artifacts_or_catalog_are_published( + tmp_path: Path, +) -> None: + registry, definition, store, job, package_root = _fixture(tmp_path) + publisher = _publisher( + tmp_path, + store=store, + registry=registry, + profile=_profile(definition), + validator=None, + ) + + with pytest.raises( + PortableResultPublicationBlockedError, + match="validator is not installed", + ): + publisher.publish(job=job, package_root=package_root) + + assert store.get_lab_instance(RESULT_ID) is None + assert not tuple((tmp_path / "central-artifacts").glob("manifests/sha256/*/*")) + + +def test_missing_definition_bound_profile_is_not_inferred_from_result_or_ui( + tmp_path: Path, +) -> None: + registry, _definition, store, job, package_root = _fixture(tmp_path) + publisher = _publisher( + tmp_path, + store=store, + registry=registry, + profile=None, + validator=_validator, + ) + + with pytest.raises( + PortableResultPublicationBlockedError, + match="profile policy is not registered", + ): + publisher.publish(job=job, package_root=package_root) + + assert store.get_lab_instance(RESULT_ID) is None + + +def test_changed_artifact_bytes_fail_closed_before_catalog_publication( + tmp_path: Path, +) -> None: + registry, definition, store, job, package_root = _fixture(tmp_path) + (package_root / "artifacts" / "result.json").write_bytes(b"x" * 8) + publisher = _publisher( + tmp_path, + store=store, + registry=registry, + profile=_profile(definition), + validator=_validator, + ) + + with pytest.raises( + PortableResultPackageIntegrityError, + match="artifact content changed", + ): + publisher.publish(job=job, package_root=package_root) + + assert store.get_lab_instance(RESULT_ID) is None + + +def test_contract_validator_rejection_never_becomes_a_lab_result(tmp_path: Path) -> None: + registry, definition, store, job, package_root = _fixture( + tmp_path, + accepted=False, + ) + publisher = _publisher( + tmp_path, + store=store, + registry=registry, + profile=_profile(definition), + validator=_validator, + ) + + with pytest.raises( + PortableResultPackageIntegrityError, + match="failed its exact contract validator", + ): + publisher.publish(job=job, package_root=package_root) + + assert store.get_lab_instance(RESULT_ID) is None + + +def test_missing_persisted_source_contract_is_a_publication_blocker(tmp_path: Path) -> None: + registry, definition, store, job, package_root = _fixture(tmp_path) + source_documents = store.data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY + for path in source_documents.iterdir(): + path.unlink() + publisher = _publisher( + tmp_path, + store=store, + registry=registry, + profile=_profile(definition), + validator=_validator, + ) + + with pytest.raises( + PortableResultPublicationBlockedError, + match="source document is unavailable", + ): + publisher.publish(job=job, package_root=package_root) + + assert store.get_lab_instance(RESULT_ID) is None + + +def test_package_manifest_rejects_noncanonical_or_authority_elevating_documents( + tmp_path: Path, +) -> None: + registry = _ready_registry(tmp_path) + definition = registry.definitions[0] + store, catalog_sha, bundle_sha, capability_sha = _source_store(tmp_path, definition) + _queue, running, _claim_token = _running_job( + tmp_path, + definition=definition, + catalog_sha256=catalog_sha, + bundle_sha256=bundle_sha, + capability_sha256=capability_sha, + ) + _root, manifest = _package( + tmp_path, + job=running, + definition=definition, + ) + elevated = manifest.as_dict() + authority = cast(dict[str, object], elevated["authority"]) + authority["commands_enabled"] = True + identity = { + "schema_version": PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA, + **{ + key: value + for key, value in elevated.items() + if key not in {"schema_version", "identity_sha256"} + }, + } + elevated["identity_sha256"] = canonical_sha256(identity) + + with pytest.raises( + PortableResultPackageIntegrityError, + match="not observation-only", + ): + PortableResultPackageManifest.from_bytes(_canonical_json(elevated)) + with pytest.raises( + PortableResultPackageIntegrityError, + match="not canonical JSON", + ): + PortableResultPackageManifest.from_bytes( + json.dumps(manifest.as_dict(), indent=2).encode() + ) + + +def test_legacy_canonical_result_namespace_cannot_be_republished( + tmp_path: Path, +) -> None: + legacy_result_id = "lab-v1-vegetation-shadow-" + "a" * 64 + registry, definition, store, job, package_root = _fixture( + tmp_path, + result_id=legacy_result_id, + ) + publisher = _publisher( + tmp_path, + store=store, + registry=registry, + profile=_profile(definition), + validator=_validator, + ) + + with pytest.raises( + PortableResultPublicationBlockedError, + match="legacy canonical result namespace", + ): + publisher.publish(job=job, package_root=package_root) + + assert store.get_lab_instance(legacy_result_id) is None diff --git a/tests/test_observatory_portable_run_definitions.py b/tests/test_observatory_portable_run_definitions.py index 8cfec7a..be8962a 100644 --- a/tests/test_observatory_portable_run_definitions.py +++ b/tests/test_observatory_portable_run_definitions.py @@ -1,6 +1,7 @@ from __future__ import annotations import copy +import hashlib import json from dataclasses import FrozenInstanceError, replace from pathlib import Path @@ -18,8 +19,10 @@ from k1link.observatory.portable_run_definitions import ( REPOSITORY_ROOT = Path(__file__).resolve().parents[1] REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" -DEFINITION_SHA256 = "57bf8f0859e10e54e30322c9a8aa28b427699f6fe6b5267e279ec3390fa78466" +DEFINITION_SHA256 = "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2" MODEL_MANIFEST_SHA256 = "3fd2d43af73bd73f89d9ffae95d8770cfdeb46033ec967509124fac6ae4afe56" +M49_DEFINITION_SHA256 = "73611f24d70319ea1edca428726d6538a3cbad012a415cc0c1a7ecb7d9b4d910" +M49_MODEL_MANIFEST_SHA256 = "489a43448f720a9b5c7993dc8279d167b77191a586f0d87b6d38b81cf728e2f1" def _registry() -> PortableRunDefinitionRegistry: @@ -96,6 +99,54 @@ def test_source_requirements_map_exactly_to_admission_contract() -> None: assert admission.adapter_sha256 == definition.source_adapter.contract_sha256 +def test_m49_portable_v2_is_model_free_and_contains_no_exact_source_binding() -> None: + definition = _registry().resolve_setup("m49-tgs-portable-v2") + + assert definition.definition_sha256 == M49_DEFINITION_SHA256 + assert definition.models == () + assert definition.learned_models == () + assert definition.model_manifest_sha256 == M49_MODEL_MANIFEST_SHA256 + assert definition.resource_profile.accelerator_id == "cpu-only" + assert definition.executor.state == "not-installed" + assert definition.executor.release_id is None + assert definition.executor.release_sha256 is None + assert definition.executor.image_sha256 is None + identity = json.dumps(definition.identity_document(), sort_keys=True) + assert "RAVNOVES00" not in identity + assert "20260720T065719Z_viewer_live" not in identity + assert "4489" not in identity + assert "3928" not in identity + + profile_path = REPOSITORY_ROOT / "config" / "perception" / "m49-tgs-portable-v2.json" + profile = json.loads(profile_path.read_text(encoding="utf-8")) + components = {component.component_id: component for component in definition.components} + assert hashlib.sha256(profile_path.read_bytes()).hexdigest() == ( + components["m49-tgs-portable-profile-v2"].sha256 + ) + assert profile["source_binding"] == { + "mode": "admitted-k1-recording", + "camera_timeline": "dynamic", + "lidar_replay": "dynamic", + "trajectory": "dynamic", + "frame_counts": "source-derived", + "filesystem_paths": "executor-resolved", + } + + with pytest.raises(PortableRunDefinitionRegistryError, match="runner"): + replace( + definition, + executor=PortableExecutorAvailability( + contour_id="worker-006", + state="ready", + release_id="m49-tgs-portable-executor-v2", + release_sha256="1" * 64, + image_sha256="2" * 64, + reason_code=None, + reason=None, + ), + ) + + def test_all_canonical_identities_are_recomputed_from_typed_content() -> None: definition = _registry().definitions[0] @@ -278,6 +329,33 @@ def test_conversion_to_recorded_definition_requires_and_preserves_sealed_identit assert recorded.checkpoint_policy == "non-checkpointable" +def test_blocked_definition_does_not_hide_an_unrelated_ready_definition() -> None: + registry = _registry() + blocked_lab = registry.resolve_setup("lab-v1-eomt-ddrnet-portable-v1") + blocked_m49 = registry.resolve_setup("m49-tgs-portable-v2") + ready_executor = PortableExecutorAvailability( + contour_id="worker-006", + state="ready", + release_id="lab-v1-eomt-ddrnet-executor-v1", + release_sha256="1" * 64, + image_sha256="2" * 64, + reason_code=None, + reason=None, + ) + identity = blocked_lab.identity_document() + identity["executor"] = ready_executor.identity_document() + ready_lab = replace( + blocked_lab, + executor=ready_executor, + definition_sha256=canonical_sha256(identity), + ) + mixed = PortableRunDefinitionRegistry((ready_lab, blocked_m49)) + + assert mixed.ready_recorded_definitions() == (ready_lab.to_recorded_run_definition(),) + assert mixed.to_recorded_registry().definitions == (ready_lab.to_recorded_run_definition(),) + assert mixed.resolve_setup("m49-tgs-portable-v2") is blocked_m49 + + def test_production_lab_v1_model_component_and_result_identities_are_exact() -> None: definition = _registry().definitions[0] models = {model.release_id: model for model in definition.models} diff --git a/tests/test_observatory_portable_setup_api.py b/tests/test_observatory_portable_setup_api.py index 2d2904e..149fa5e 100644 --- a/tests/test_observatory_portable_setup_api.py +++ b/tests/test_observatory_portable_setup_api.py @@ -1,13 +1,26 @@ from __future__ import annotations +from dataclasses import replace from pathlib import Path from types import SimpleNamespace from fastapi import FastAPI from fastapi.testclient import TestClient -from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry -from k1link.observatory.portable_setup_projection import PortableLabV1SetupProjector +from k1link.observatory.portable_run_definitions import ( + PortableExecutorAvailability, + PortableRunDefinitionRegistry, + canonical_sha256, +) +from k1link.observatory.portable_setup_projection import ( + PortableLabV1SetupProjector, + PortableSetupProjector, +) +from k1link.observatory.recorded_jobs import ( + ObservatoryRecordedJobIntent, + ObservatoryRecordedJobQueue, + RecordedRunDefinitionRegistry, +) from k1link.observatory.source_admission import PortableRecordedSourceCapability from k1link.sessions import SessionNotFoundError from k1link.sessions.models import SessionSummary @@ -111,4 +124,193 @@ def test_portable_setup_catalog_preserves_source_and_optional_slice_failures() - params={"source_session_id": SOURCE_SESSION_ID}, ) assert unavailable.status_code == 503 - assert unavailable.json()["detail"] == "Portable-каталог LAB V1 недоступен." + assert unavailable.json()["detail"] == "Portable-каталог сетапов недоступен." + + +def _ready_lab_registry() -> PortableRunDefinitionRegistry: + blocked = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH).resolve_setup( + "lab-v1-eomt-ddrnet-portable-v1" + ) + executor = PortableExecutorAvailability( + contour_id="worker-006", + state="ready", + release_id="lab-v1-eomt-ddrnet-executor-v1", + release_sha256="1" * 64, + image_sha256="2" * 64, + reason_code=None, + reason=None, + ) + identity = blocked.identity_document() + identity["executor"] = executor.identity_document() + return PortableRunDefinitionRegistry( + ( + replace( + blocked, + executor=executor, + definition_sha256=canonical_sha256(identity), + ), + ) + ) + + +class _PortableBinding: + def __init__( + self, + registry: PortableRunDefinitionRegistry, + queue: ObservatoryRecordedJobQueue, + ) -> None: + self.registry = registry + self.queue = queue + self.check_sha256 = "9" * 64 + self.check_count = 0 + self.submit_count = 0 + + def probe( + self, + *, + source_session_id: str, + setup_id: str, + definition_sha256: str, + ) -> PortableRecordedSourceCapability: + definition = self.registry.resolve(setup_id, definition_sha256) + return PortableRecordedSourceCapability( + source_session_id=source_session_id, + source_catalog_sha256="a" * 64, + source_adapter_sha256=definition.source_adapter.contract_sha256, + camera_segment_count=600, + ) + + def check( + self, + *, + source_session_id: str, + setup_id: str, + definition_sha256: str, + ) -> SimpleNamespace: + self.registry.resolve(setup_id, definition_sha256) + assert source_session_id == SOURCE_SESSION_ID + self.check_count += 1 + return SimpleNamespace(check_sha256=self.check_sha256) + + def submit( + self, + *, + source_session_id: str, + setup_id: str, + definition_sha256: str, + expected_check_sha256: str, + idempotency_key: str, + ) -> tuple[object, bool]: + self.registry.resolve(setup_id, definition_sha256) + assert source_session_id == SOURCE_SESSION_ID + assert expected_check_sha256 == self.check_sha256 + self.submit_count += 1 + return self.queue.submit( + ObservatoryRecordedJobIntent( + idempotency_key=idempotency_key, + source_session_id=source_session_id, + source_catalog_sha256="a" * 64, + source_bundle_sha256="b" * 64, + source_capability_manifest_sha256="c" * 64, + setup_id=setup_id, + definition_sha256=definition_sha256, + ), + enqueue=True, + ) + + +def test_portable_api_check_sha_fences_ready_submission(tmp_path: Path) -> None: + registry = _ready_lab_registry() + queue = ObservatoryRecordedJobQueue( + tmp_path, + definitions=RecordedRunDefinitionRegistry(registry.ready_recorded_definitions()), + ) + binding = _PortableBinding(registry, queue) + projector = PortableSetupProjector( + registry=registry, + capability_probe=binding, # type: ignore[arg-type] + dispatch_available=True, + ) + app = FastAPI() + app.include_router( + build_observatory_router( + _Store(), # type: ignore[arg-type] + portable_setup_projector=projector, + portable_binding_service=binding, # type: ignore[arg-type] + recorded_job_queue=queue, + ) + ) + client = TestClient(app) + definition = registry.definitions[0] + preflight = client.post( + "/api/v1/observatory/run-preflights", + json={ + "schema_version": "missioncore.observatory-run-preflight-request/v1", + "source_session_id": SOURCE_SESSION_ID, + "setup_id": definition.setup_id, + "definition_sha256": definition.definition_sha256, + }, + ) + + assert preflight.status_code == 200 + assert preflight.json()["outcome"] == "queueable" + assert preflight.json()["check_sha256"] == binding.check_sha256 + assert binding.check_count == 1 + + submitted = client.post( + "/api/v1/observatory/runs", + json={ + "schema_version": "missioncore.observatory-recorded-run-submit/v1", + "idempotency_key": "portable:lab-v1:source-005", + "source_session_id": SOURCE_SESSION_ID, + "setup_id": definition.setup_id, + "definition_sha256": definition.definition_sha256, + "check_sha256": binding.check_sha256, + }, + ) + + assert submitted.status_code == 202 + assert submitted.json()["state"] == "queued" + assert submitted.json()["setup"]["setup_id"] == definition.setup_id + assert binding.submit_count == 1 + + +def test_portable_api_rejects_blocked_executor_before_binding_submit(tmp_path: Path) -> None: + full_registry = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH) + ready_registry = _ready_lab_registry() + queue = ObservatoryRecordedJobQueue( + tmp_path, + definitions=RecordedRunDefinitionRegistry(ready_registry.ready_recorded_definitions()), + ) + binding = _PortableBinding(full_registry, queue) + projector = PortableSetupProjector( + registry=full_registry, + capability_probe=binding, # type: ignore[arg-type] + dispatch_available=True, + ) + app = FastAPI() + app.include_router( + build_observatory_router( + _Store(), # type: ignore[arg-type] + portable_setup_projector=projector, + portable_binding_service=binding, # type: ignore[arg-type] + recorded_job_queue=queue, + ) + ) + definition = full_registry.resolve_setup("m49-tgs-portable-v2") + response = TestClient(app).post( + "/api/v1/observatory/runs", + json={ + "schema_version": "missioncore.observatory-recorded-run-submit/v1", + "idempotency_key": "portable:m49:blocked", + "source_session_id": SOURCE_SESSION_ID, + "setup_id": definition.setup_id, + "definition_sha256": definition.definition_sha256, + "check_sha256": "9" * 64, + }, + ) + + assert response.status_code == 409 + assert response.json()["detail"] == "Portable executor-релиз не установлен." + assert binding.submit_count == 0 + assert queue.list_jobs() == () diff --git a/tests/test_observatory_portable_setup_projection.py b/tests/test_observatory_portable_setup_projection.py index 4267a1e..09edd01 100644 --- a/tests/test_observatory_portable_setup_projection.py +++ b/tests/test_observatory_portable_setup_projection.py @@ -13,9 +13,12 @@ from k1link.observatory.portable_run_definitions import ( from k1link.observatory.portable_setup_projection import ( PORTABLE_LAB_V1_DISPLAY_NAME, PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA, + PORTABLE_M49_DISPLAY_NAME, PortableLabV1SetupProjector, PortableSetupProjectionError, + PortableSetupProjector, PortableSourceCapabilityProbe, + portable_calculation_profile_registry, ) from k1link.observatory.source_admission import ( PortableRecordedSourceCapability, @@ -119,6 +122,61 @@ def test_projection_uses_portable_identity_and_exact_model_presentation() -> Non } +class _GenericProbe: + def __init__(self, registry: PortableRunDefinitionRegistry) -> None: + self.registry = registry + + def probe( + self, + *, + source_session_id: str, + setup_id: str, + definition_sha256: str, + ) -> PortableRecordedSourceCapability: + definition = self.registry.resolve(setup_id, definition_sha256) + return _capability( + source_session_id, + adapter_sha256=definition.source_adapter.contract_sha256, + ) + + +def test_generic_catalog_projects_lab_v1_and_model_free_m49_independently() -> None: + registry = _registry() + catalog = PortableSetupProjector( + registry=registry, + capability_probe=_GenericProbe(registry), + ).catalog(_source(NEW_SESSION_ID)) + + setups = {setup["setup_id"]: setup for setup in catalog["setups"]} + assert set(setups) == { + "lab-v1-eomt-ddrnet-portable-v1", + "m49-tgs-portable-v2", + } + m49 = setups["m49-tgs-portable-v2"] + assert m49["display_name"] == PORTABLE_M49_DISPLAY_NAME + assert m49["run_definition"]["models"] == [] + assert m49["source_compatibility"]["outcome"] == "pass" + assert m49["executor"]["state"] == "not-installed" + assert m49["preflight"]["submission_allowed"] is False + + +def test_calculation_profile_policies_cover_both_exact_portable_definitions() -> None: + registry = _registry() + profiles = portable_calculation_profile_registry(registry) + + resolved = { + definition.setup_id: profiles.resolve(definition) + for definition in registry.definitions + } + assert resolved["lab-v1-eomt-ddrnet-portable-v1"].lab_id == "LAB V1" + assert ( + resolved["lab-v1-eomt-ddrnet-portable-v1"].display_name + == PORTABLE_LAB_V1_DISPLAY_NAME + ) + assert resolved["m49-tgs-portable-v2"].lab_id == "LAB M4.9T5" + assert resolved["m49-tgs-portable-v2"].display_name == PORTABLE_M49_DISPLAY_NAME + + def test_new_compatible_source_passes_capability_but_uninstalled_executor_blocks() -> None: source = _source(NEW_SESSION_ID) setup = _projector(lambda session_id: _capability(session_id)).project(source) @@ -235,10 +293,7 @@ def test_ready_executor_still_blocks_without_a_dispatch_boundary() -> None: assert setup["preflight"] == { "outcome": "blocked", "action": "blocked", - "reason": ( - "Server-side проверка definition/check SHA и постановка portable " - "LAB V1 в очередь пока недоступны." - ), + "reason": ("Server-side проверка и постановка portable-сетапа в очередь недоступны."), "submission_allowed": False, "existing_result_ids": [], } diff --git a/tests/test_observatory_portable_worker_integration.py b/tests/test_observatory_portable_worker_integration.py new file mode 100644 index 0000000..061e8d0 --- /dev/null +++ b/tests/test_observatory_portable_worker_integration.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +from pathlib import Path +from typing import cast + +import pytest + +from k1link.artifact_gateway import CentralArtifactStore +from k1link.observatory.m49_portable_result import ( + M49_PORTABLE_RESULT_CONTRACT_SHA256, + validate_m49_portable_result, +) +from k1link.observatory.portable_artifact_transport import ( + PortableObservatoryArtifactTransport, +) +from k1link.observatory.portable_lab_v1_executor import validate_lab_v1_result_v2 +from k1link.observatory.portable_result_contract import ( + PortableResultContractValidatorRegistration, + PortableResultContractValidatorRegistry, +) +from k1link.observatory.portable_result_publisher import ( + PortableObservatoryResultPublisher, +) +from k1link.observatory.portable_run_definitions import ( + PortableRunDefinitionRegistry, +) +from k1link.observatory.portable_setup_projection import ( + PORTABLE_LAB_V1_DISPLAY_NAME, + PORTABLE_LAB_V1_SETUP_ID, + PORTABLE_M49_DISPLAY_NAME, + PORTABLE_M49_SETUP_ID, +) +from k1link.observatory.portable_worker_integration import ( + OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV, + OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV, + PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256, + PortableWorkerIntegrationError, + PortableWorkerStorageRoots, + build_portable_observatory_worker_integration, + portable_result_validator_registry, +) +from k1link.observatory.recorded_jobs import ObservatoryRecordedJobQueue +from k1link.sessions import RecordedMediaInspector, SessionStore + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" + + +def _definitions() -> PortableRunDefinitionRegistry: + return PortableRunDefinitionRegistry.from_file(REGISTRY_PATH) + + +def test_exact_validator_registry_covers_both_portable_profiles() -> None: + definitions = _definitions() + + validators = portable_result_validator_registry(definitions) + + assert validators.resolve(PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256) is validate_lab_v1_result_v2 + assert validators.resolve(M49_PORTABLE_RESULT_CONTRACT_SHA256) is validate_m49_portable_result + + +def test_validator_registry_fails_when_one_required_profile_is_absent() -> None: + definitions = _definitions() + only_lab_v1 = PortableRunDefinitionRegistry((definitions.definitions[0],)) + + with pytest.raises( + PortableWorkerIntegrationError, + match="required portable setup is unavailable", + ): + portable_result_validator_registry(only_lab_v1) + + +def test_server_integration_constructs_dormant_transport_and_publisher( + tmp_path: Path, +) -> None: + definitions = _definitions() + store = SessionStore(tmp_path / "repository") + inspector = RecordedMediaInspector(tmp_path / "media-inspections") + source_cas_root = tmp_path / "source-cas" + result_staging_root = tmp_path / "result-staging" + source_cas_root.mkdir() + result_staging_root.mkdir() + + integration = build_portable_observatory_worker_integration( + queue=cast(ObservatoryRecordedJobQueue, object()), + session_store=store, + media_inspector=inspector, + definitions=definitions, + artifact_store=CentralArtifactStore(tmp_path / "artifacts", create=True), + source_cas_root=source_cas_root, + result_staging_root=result_staging_root, + ) + + assert integration.supported_setup_ids == ( + PORTABLE_LAB_V1_SETUP_ID, + PORTABLE_M49_SETUP_ID, + ) + assert isinstance( + integration.artifact_transport, + PortableObservatoryArtifactTransport, + ) + assert isinstance( + integration.result_publisher, + PortableObservatoryResultPublisher, + ) + profiles = { + policy.setup_id: policy.display_name for policy in integration.calculation_profiles.policies + } + assert profiles == { + PORTABLE_LAB_V1_SETUP_ID: PORTABLE_LAB_V1_DISPLAY_NAME, + PORTABLE_M49_SETUP_ID: PORTABLE_M49_DISPLAY_NAME, + } + + +def test_server_integration_rejects_swapped_validator_identities( + tmp_path: Path, +) -> None: + definitions = _definitions() + swapped = PortableResultContractValidatorRegistry( + ( + PortableResultContractValidatorRegistration( + PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256, + validate_m49_portable_result, + ), + PortableResultContractValidatorRegistration( + M49_PORTABLE_RESULT_CONTRACT_SHA256, + validate_lab_v1_result_v2, + ), + ) + ) + source_cas_root = tmp_path / "source-cas" + result_staging_root = tmp_path / "result-staging" + source_cas_root.mkdir() + result_staging_root.mkdir() + + with pytest.raises( + PortableWorkerIntegrationError, + match="validator registration changed identity", + ): + build_portable_observatory_worker_integration( + queue=cast(ObservatoryRecordedJobQueue, object()), + session_store=SessionStore(tmp_path / "repository"), + media_inspector=RecordedMediaInspector(tmp_path / "media-inspections"), + definitions=definitions, + artifact_store=CentralArtifactStore(tmp_path / "artifacts", create=True), + validators=swapped, + source_cas_root=source_cas_root, + result_staging_root=result_staging_root, + ) + + +def test_storage_roots_load_only_from_existing_disjoint_central_directories( + tmp_path: Path, +) -> None: + boundary = tmp_path / "nodedc-mission-core" + artifact_store = boundary / "artifact-store" + source_cas = boundary / "observatory-worker" / "source-cas" + result_staging = boundary / "observatory-worker" / "result-staging" + for path in (artifact_store, source_cas, result_staging): + path.mkdir(parents=True, exist_ok=True) + + roots = PortableWorkerStorageRoots.from_environment( + artifact_store_root=artifact_store, + environment={ + OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV: str(source_cas), + OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV: str(result_staging), + }, + ) + + assert roots == PortableWorkerStorageRoots( + source_cas_root=source_cas, + result_staging_root=result_staging, + ) + + +def test_server_integration_has_no_data_directory_storage_fallback( + tmp_path: Path, +) -> None: + store = SessionStore(tmp_path / "repository") + + with pytest.raises( + PortableWorkerIntegrationError, + match="storage roots must be explicitly configured", + ): + build_portable_observatory_worker_integration( + queue=cast(ObservatoryRecordedJobQueue, object()), + session_store=store, + media_inspector=RecordedMediaInspector(tmp_path / "media-inspections"), + definitions=_definitions(), + artifact_store=CentralArtifactStore(tmp_path / "artifacts", create=True), + ) + + assert not (store.data_dir / "observatory-worker-source-cas").exists() + assert not (store.data_dir / "observatory-worker-result-staging").exists() + + +def test_storage_root_loading_does_not_create_an_unavailable_mount_path( + tmp_path: Path, +) -> None: + boundary = tmp_path / "nodedc-mission-core" + artifact_store = boundary / "artifact-store" + artifact_store.mkdir(parents=True) + missing_source = boundary / "observatory-worker" / "source-cas" + missing_result = boundary / "observatory-worker" / "result-staging" + + with pytest.raises( + PortableWorkerIntegrationError, + match="portable source CAS is unavailable", + ): + PortableWorkerStorageRoots.from_environment( + artifact_store_root=artifact_store, + environment={ + OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV: str(missing_source), + OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV: str(missing_result), + }, + ) + + assert not missing_source.exists() + assert not missing_result.exists() + + +def test_storage_roots_reject_escape_overlap_and_symlink(tmp_path: Path) -> None: + boundary = tmp_path / "nodedc-mission-core" + artifact_store = boundary / "artifact-store" + result_staging = boundary / "observatory-worker" / "result-staging" + outside = tmp_path / "outside" + for path in (artifact_store, result_staging, outside): + path.mkdir(parents=True, exist_ok=True) + + with pytest.raises(PortableWorkerIntegrationError, match="must be inside"): + PortableWorkerStorageRoots.from_paths( + artifact_store_root=artifact_store, + source_cas_root=outside, + result_staging_root=result_staging, + ) + + with pytest.raises(PortableWorkerIntegrationError, match="disjoint from"): + PortableWorkerStorageRoots.from_paths( + artifact_store_root=artifact_store, + source_cas_root=artifact_store, + result_staging_root=result_staging, + ) + + source_target = boundary / "observatory-worker" / "source-target" + source_target.mkdir(parents=True) + source_link = boundary / "observatory-worker" / "source-link" + source_link.symlink_to(source_target, target_is_directory=True) + with pytest.raises(PortableWorkerIntegrationError, match="canonical directory"): + PortableWorkerStorageRoots.from_paths( + artifact_store_root=artifact_store, + source_cas_root=source_link, + result_staging_root=result_staging, + ) diff --git a/tests/test_observatory_portable_worker_runtime.py b/tests/test_observatory_portable_worker_runtime.py new file mode 100644 index 0000000..e08f2e1 --- /dev/null +++ b/tests/test_observatory_portable_worker_runtime.py @@ -0,0 +1,574 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +import pytest + +from k1link.observatory.portable_run_definitions import ( + PortableExecutorAvailability, + PortableRunDefinitionRegistry, + canonical_sha256, +) +from k1link.observatory.portable_worker_runtime import ( + PortableWorkerAssetVerification, + PortableWorkerExecutorAdapter, + PortableWorkerExecutorSeal, + PortableWorkerLocalAssetBinding, + PortableWorkerResultDraft, + PortableWorkerRuntimeAdmission, + PortableWorkerRuntimePhase, + PortableWorkerRuntimePlan, + PortableWorkerRuntimeRegistry, + PortableWorkerRuntimeRegistryError, + PortableWorkerRuntimeUnavailableError, + PortableWorkerSourceStage, + inspect_runtime_candidate, +) +from k1link.observatory.worker_agent import ( + ObservatoryWorkerExecutionResult, + ObservatoryWorkerExecutorIdentity, + SealedObservatoryRecordedJob, +) + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +DEFINITION_REGISTRY = ( + REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" +) +RUNTIME_REGISTRY = ( + REPOSITORY_ROOT / "config" / "observatory-worker-runtime-candidates.json" +) + + +def _definitions() -> PortableRunDefinitionRegistry: + return PortableRunDefinitionRegistry.from_file(DEFINITION_REGISTRY) + + +def _runtime() -> PortableWorkerRuntimeRegistry: + return PortableWorkerRuntimeRegistry.from_file( + RUNTIME_REGISTRY, + definitions=_definitions(), + ) + + +def _all_keys(value: object) -> set[str]: + if isinstance(value, dict): + return set(value) | { + nested + for child in value.values() + for nested in _all_keys(child) + } + if isinstance(value, list): + return {nested for child in value for nested in _all_keys(child)} + return set() + + +def test_production_candidates_bind_exact_definitions_but_remain_blocked() -> None: + registry = _runtime() + + assert {candidate.setup_id for candidate in registry.candidates} == { + "lab-v1-eomt-ddrnet-portable-v1", + "m49-tgs-portable-v2", + } + for candidate in registry.candidates: + assert candidate.ready is False + assert candidate.executor is None + assert "executor-release-unsealed" in candidate.blockers + assert any(phase.state == "missing" for phase in candidate.phases) + with pytest.raises( + PortableWorkerRuntimeUnavailableError, + match="no executor identity", + ): + candidate.executor_identity() + + +def test_candidate_contract_is_source_independent_and_instruction_free() -> None: + document = json.loads(RUNTIME_REGISTRY.read_text(encoding="utf-8")) + serialized = json.dumps(document, sort_keys=True) + keys = _all_keys(document) + + assert "RAVNOVES" not in serialized + assert "source_session_id" not in serialized + assert "filesystem" not in serialized + assert not {"command", "commands", "argv", "env", "environment"} & keys + assert not any("priority" in key for key in keys) + + +def test_m49_reuses_portable_profile_generic_runner_and_travel_image() -> None: + candidate = _runtime().resolve( + "m49-tgs-portable-v2", + "73611f24d70319ea1edca428726d6538a3cbad012a415cc0c1a7ecb7d9b4d910", + ) + bindings = { + "m49-portable-profile": PortableWorkerLocalAssetBinding( + asset_id="m49-portable-profile", + file_path=REPOSITORY_ROOT / "config" / "perception" / "m49-tgs-portable-v2.json", + ), + "m49-portable-runner-source": PortableWorkerLocalAssetBinding( + asset_id="m49-portable-runner-source", + file_path=( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "observatory_portable" + / "run_m49_tgs_portable.cpp" + ), + ), + "m49-portable-runner-manifest": PortableWorkerLocalAssetBinding( + asset_id="m49-portable-runner-manifest", + file_path=( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "observatory_portable" + / "m49-tgs-portable-runner-source.json" + ), + ), + "m49-portable-runner-wrapper": PortableWorkerLocalAssetBinding( + asset_id="m49-portable-runner-wrapper", + file_path=( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "observatory_portable" + / "run_m49_tgs_portable.sh" + ), + ), + "m49-portable-smoke": PortableWorkerLocalAssetBinding( + asset_id="m49-portable-smoke", + file_path=( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "observatory_portable" + / "smoke_m49_tgs_portable.sh" + ), + ), + "travel-tgs-image": PortableWorkerLocalAssetBinding( + asset_id="travel-tgs-image", + image_sha256=( + "7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f" + ), + ), + } + + admission = inspect_runtime_candidate(candidate, bindings) + + assert {item.state for item in admission.assets} == {"matched"} + assert admission.ready is False + assert "portable-tgs-runner-unsealed" in admission.blockers + assert "portable-camera-lidar-timeline-unimplemented" in admission.blockers + assert "portable-result-assembler-unimplemented" in admission.blockers + + +def test_m49_portable_runner_has_no_exact_source_or_frame_count_binding() -> None: + source = ( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "observatory_portable" + / "run_m49_tgs_portable.cpp" + ).read_text(encoding="utf-8") + wrapper = ( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "observatory_portable" + / "run_m49_tgs_portable.sh" + ).read_text(encoding="utf-8") + + assert "RAVNOVES" not in source + wrapper + assert "4489" not in source + wrapper + assert "3928" not in source + wrapper + assert "schedule.rows.size()" in source + assert "verifySequence(sequence_dir, schedule.available_count)" in source + assert "std::vector values(point_count * 4)" in source + assert "--network" not in wrapper + + +def test_m49_portable_runner_source_release_is_content_addressed() -> None: + root = ( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "observatory_portable" + ) + manifest = json.loads( + (root / "m49-tgs-portable-runner-source.json").read_text(encoding="utf-8") + ) + expected_release_sha256 = manifest.pop("source_release_sha256") + actual_release_sha256 = hashlib.sha256( + json.dumps( + manifest, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ).hexdigest() + + assert actual_release_sha256 == expected_release_sha256 + assert manifest["input_contract"]["timeline_frame_count"] == "source-derived" + assert manifest["input_contract"]["available_lidar_frame_count"] == "source-derived" + for item in manifest["files"]: + path = root / item["name"] + assert path.stat().st_size == item["byte_length"] + assert hashlib.sha256(path.read_bytes()).hexdigest() == item["sha256"] + + +def test_lab_candidate_verifies_reusable_repository_assets_without_claiming_executor() -> None: + candidate = _runtime().resolve( + "lab-v1-eomt-ddrnet-portable-v1", + "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2", + ) + bindings = { + "ddrnet-goose-image": PortableWorkerLocalAssetBinding( + asset_id="ddrnet-goose-image", + image_sha256=( + "591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd" + ), + ), + "ddrnet-goose-runner": PortableWorkerLocalAssetBinding( + asset_id="ddrnet-goose-runner", + file_path=( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "lab_v1_vegetation_goose" + / "run_goose_vegetation_benchmark.py" + ), + ), + "eomt-image": PortableWorkerLocalAssetBinding( + asset_id="eomt-image", + image_sha256=( + "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794" + ), + ), + "eomt-orchestrator": PortableWorkerLocalAssetBinding( + asset_id="eomt-orchestrator", + file_path=( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "Invoke-E4FullSessionSegmentation.ps1" + ), + ), + "eomt-profile": PortableWorkerLocalAssetBinding( + asset_id="eomt-profile", + file_path=( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "e3_k1_camera1_profile.json" + ), + ), + "eomt-runner": PortableWorkerLocalAssetBinding( + asset_id="eomt-runner", + file_path=( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "run_e4_full_session_segmentation.py" + ), + ), + "vegetation-policy": PortableWorkerLocalAssetBinding( + asset_id="vegetation-policy", + file_path=( + REPOSITORY_ROOT + / "config" + / "perception" + / "lab-v1-vegetation-mission-policy-v1.json" + ), + ), + "vegetation-provider-map": PortableWorkerLocalAssetBinding( + asset_id="vegetation-provider-map", + file_path=( + REPOSITORY_ROOT + / "config" + / "perception" + / "lab-v1-vegetation-provider-label-map-v1.json" + ), + ), + } + + admission = inspect_runtime_candidate(candidate, bindings) + states = {item.asset_id: item.state for item in admission.assets} + + assert states["ddrnet-goose-image"] == "matched" + assert states["ddrnet-goose-runner"] == "matched" + assert states["eomt-image"] == "matched" + assert states["eomt-orchestrator"] == "matched" + assert states["eomt-profile"] == "matched" + assert states["eomt-runner"] == "matched" + assert states["ddrnet-portable-config"] == "missing" + assert states["ddrnet-checkpoint"] == "missing" + assert states["eomt-model-weights"] == "missing" + assert admission.ready is False + assert "ddrnet-component-port-uninstalled" in admission.blockers + assert "worker-installation-receipt-unavailable" in admission.blockers + + +def test_local_asset_tampering_is_reported_without_execution(tmp_path: Path) -> None: + candidate = _runtime().resolve( + "m49-tgs-portable-v2", + "73611f24d70319ea1edca428726d6538a3cbad012a415cc0c1a7ecb7d9b4d910", + ) + tampered = tmp_path / "m49-profile.json" + tampered.write_text("{}\n", encoding="utf-8") + + admission = inspect_runtime_candidate( + candidate, + { + "m49-portable-profile": PortableWorkerLocalAssetBinding( + asset_id="m49-portable-profile", + file_path=tampered, + ), + "travel-tgs-image": PortableWorkerLocalAssetBinding( + asset_id="travel-tgs-image", + image_sha256="7" * 64, + ), + }, + ) + + states = {item.asset_id: item.state for item in admission.assets} + assert states["m49-portable-profile"] == "mismatched" + assert states["travel-tgs-image"] == "mismatched" + assert states["m49-portable-runner-source"] == "missing" + assert states["m49-portable-runner-manifest"] == "missing" + assert states["m49-portable-runner-wrapper"] == "missing" + assert "asset-m49-portable-profile-mismatched" in admission.blockers + assert "asset-travel-tgs-image-mismatched" in admission.blockers + + +def test_runtime_registry_rejects_digest_drift_and_executable_instructions( + tmp_path: Path, +) -> None: + document = json.loads(RUNTIME_REGISTRY.read_text(encoding="utf-8")) + document["candidates"][0]["candidate_sha256"] = "f" * 64 + drifted = tmp_path / "drifted.json" + drifted.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises(PortableWorkerRuntimeRegistryError, match="digest changed"): + PortableWorkerRuntimeRegistry.from_file(drifted, definitions=_definitions()) + + document = json.loads(RUNTIME_REGISTRY.read_text(encoding="utf-8")) + document["candidates"][0]["command"] = "run-anything" + unsafe = tmp_path / "unsafe.json" + unsafe.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises(PortableWorkerRuntimeRegistryError, match="forbids 'command'"): + PortableWorkerRuntimeRegistry.from_file(unsafe, definitions=_definitions()) + + +def test_runtime_plan_is_identity_only_and_observation_only() -> None: + plan = PortableWorkerRuntimePlan( + job_id="observatory-run-" + ("a" * 32), + adapter_id="m49-tgs-worker006-portable-v2", + candidate_sha256="1" * 64, + setup_id="m49-tgs-portable-v2", + definition_sha256="2" * 64, + source_bundle_sha256="3" * 64, + source_capability_manifest_sha256="4" * 64, + result_contract_sha256="5" * 64, + phases=("source-delivery", "portable-tgs-runner"), + ).as_dict() + + assert plan["authority"] == { + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, + "production_accepted": False, + } + serialized = json.dumps(plan, sort_keys=True) + keys = _all_keys(plan) + assert not {"command", "commands", "argv", "env", "environment", "path"} & keys + assert not any("priority" in key for key in keys) + assert "D:\\" not in serialized + + +def test_ready_local_adapter_composes_only_local_ports_and_exact_job( + tmp_path: Path, +) -> None: + definitions = _definitions() + base_definition = definitions.resolve_setup("lab-v1-eomt-ddrnet-portable-v1") + executor_availability = PortableExecutorAvailability( + contour_id="worker-006", + state="ready", + release_id="lab-v1-portable-executor-v1", + release_sha256="1" * 64, + image_sha256="2" * 64, + reason_code=None, + reason=None, + ) + definition_identity = base_definition.identity_document() + definition_identity["executor"] = executor_availability.identity_document() + definition = replace( + base_definition, + executor=executor_availability, + definition_sha256=canonical_sha256(definition_identity), + ) + + blocked = _runtime().resolve( + "lab-v1-eomt-ddrnet-portable-v1", + base_definition.definition_sha256, + ) + executor_seal = PortableWorkerExecutorSeal( + release_id="lab-v1-portable-executor-v1", + release_sha256="1" * 64, + image_sha256="2" * 64, + ) + phases = tuple( + PortableWorkerRuntimePhase(phase.phase_id, "implemented") + for phase in blocked.phases + ) + candidate_identity = blocked.identity_document() + candidate_identity["definition_sha256"] = definition.definition_sha256 + candidate_identity["state"] = "ready" + candidate_identity["executor"] = executor_seal.as_dict() + candidate_identity["phases"] = [phase.as_dict() for phase in phases] + candidate_identity["blockers"] = [] + candidate = replace( + blocked, + definition_sha256=definition.definition_sha256, + state="ready", + executor=executor_seal, + phases=phases, + blockers=(), + candidate_sha256=canonical_sha256(candidate_identity), + ) + admission = PortableWorkerRuntimeAdmission( + candidate_sha256=candidate.candidate_sha256, + ready=True, + blockers=(), + assets=tuple( + PortableWorkerAssetVerification(asset.asset_id, "matched", None) + for asset in candidate.reusable_assets + ), + ) + source_root = tmp_path / "source" + result_root = tmp_path / "result" + source_root.mkdir() + result_root.mkdir() + result_id = "portable-result-001" + result_sha256 = "9" * 64 + + class SourceMaterializer: + def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage: + return PortableWorkerSourceStage( + root=source_root, + source_bundle_sha256=job.source_bundle_sha256, + source_capability_manifest_sha256=( + job.source_capability_manifest_sha256 + ), + source_adapter_sha256=job.source_adapter_sha256, + ) + + class Runner: + def run( + self, + plan: PortableWorkerRuntimePlan, + source: PortableWorkerSourceStage, + ) -> PortableWorkerResultDraft: + assert source.root == source_root + assert plan.definition_sha256 == definition.definition_sha256 + return PortableWorkerResultDraft( + root=result_root, + result_id=result_id, + result_sha256=result_sha256, + result_contract_sha256=candidate.result_contract_sha256, + ) + + class Publisher: + def publish( + self, + job: SealedObservatoryRecordedJob, + draft: PortableWorkerResultDraft, + ) -> ObservatoryWorkerExecutionResult: + assert job.source_session_id == "20260831T120000Z_viewer_live" + return ObservatoryWorkerExecutionResult( + result_id=draft.result_id, + result_sha256=draft.result_sha256, + ) + + adapter = PortableWorkerExecutorAdapter( + candidate=candidate, + definition=definition, + admission=admission, + source_materializer=SourceMaterializer(), + runner=Runner(), + publisher=Publisher(), + ) + with pytest.raises( + PortableWorkerRuntimeUnavailableError, + match="does not prove every candidate asset", + ): + PortableWorkerExecutorAdapter( + candidate=candidate, + definition=definition, + admission=replace(admission, assets=()), + source_materializer=SourceMaterializer(), + runner=Runner(), + publisher=Publisher(), + ) + job = SealedObservatoryRecordedJob( + job_id="observatory-run-" + ("a" * 32), + request_sha256="3" * 64, + identity_sha256="4" * 64, + submission_receipt_sha256="c" * 64, + source_session_id="20260831T120000Z_viewer_live", + source_catalog_sha256="5" * 64, + source_bundle_sha256="6" * 64, + source_capability_manifest_sha256="7" * 64, + source_adapter_id=definition.source_adapter.adapter_id, + source_adapter_version=definition.source_adapter.version, + source_adapter_sha256=definition.source_adapter.contract_sha256, + setup_id=definition.setup_id, + definition_id=definition.definition_id, + definition_version=definition.version, + definition_sha256=definition.definition_sha256, + executor_release_id=executor_seal.release_id, + executor_identity=ObservatoryWorkerExecutorIdentity( + release_sha256=executor_seal.release_sha256, + image_sha256=executor_seal.image_sha256, + model_manifest_sha256=definition.model_manifest_sha256, + resource_profile_sha256=definition.resource_profile.profile_sha256, + ), + model_release_ids=definition.learned_models, + resource_profile_id=definition.resource_profile.profile_id, + checkpoint_policy=definition.resource_profile.checkpoint_policy, + allowed_checkpoints=definition.resource_profile.allowed_checkpoints, + claim_generation=1, + claim_claimed_at_utc="2026-08-31T09:00:00.000Z", + claim_expires_at_utc="2026-08-31T09:05:00.000Z", + claim_heartbeat_at_utc="2026-08-31T09:00:00.000Z", + claim_renewal_count=0, + restart_from_zero=False, + ) + + assert adapter.execute(job) == ObservatoryWorkerExecutionResult( + result_id=result_id, + result_sha256=result_sha256, + ) + + with pytest.raises(PortableWorkerRuntimeUnavailableError): + PortableWorkerExecutorAdapter( + candidate=blocked, + definition=base_definition, + admission=replace(admission, candidate_sha256=blocked.candidate_sha256), + source_materializer=SourceMaterializer(), + runner=Runner(), + publisher=Publisher(), + ) diff --git a/tests/test_observatory_recorded_jobs.py b/tests/test_observatory_recorded_jobs.py index 9606f4d..5bf7e05 100644 --- a/tests/test_observatory_recorded_jobs.py +++ b/tests/test_observatory_recorded_jobs.py @@ -350,6 +350,210 @@ def test_claim_is_exactly_idempotent_including_empty_result(tmp_path: Path) -> N ) +def test_claim_lease_renews_idempotently_and_requeues_expired_unstarted_job( + tmp_path: Path, +) -> None: + clock_value = [NOW] + queue = ObservatoryRecordedJobQueue( + tmp_path, + definitions=_definitions(), + clock=lambda: clock_value[0], + claim_lease_seconds=10, + ) + job, _ = queue.submit(_intent(), enqueue=True) + claim = queue.claim_next( + claimant_id="recorded-worker", + claim_request_id="lease-poll-001", + ) + assert claim is not None + assert claim.job.claimed_at_utc == NOW + assert claim.job.claim_heartbeat_at_utc == NOW + assert claim.job.claim_expires_at_utc == "2026-08-30T21:00:10.000Z" + assert claim.job.claim_renewal_count == 0 + assert claim.job.as_dict()["claim_lease"] == { + "claimed_at_utc": NOW, + "expires_at_utc": "2026-08-30T21:00:10.000Z", + "heartbeat_at_utc": NOW, + "renewal_count": 0, + } + + clock_value[0] = "2026-08-30T21:00:04.000Z" + renewed = queue.renew_claim( + job.job_id, + claim_token=claim.claim_token, + claim_generation=claim.job.claim_generation, + heartbeat_sequence=1, + ) + repeated = queue.renew_claim( + job.job_id, + claim_token=claim.claim_token, + claim_generation=claim.job.claim_generation, + heartbeat_sequence=1, + ) + assert renewed.claim_heartbeat_at_utc == clock_value[0] + assert renewed.claim_expires_at_utc == "2026-08-30T21:00:14.000Z" + assert renewed.claim_renewal_count == 1 + assert repeated == renewed + with pytest.raises(ObservatoryRecordedQueueConflictError, match="contiguous"): + queue.renew_claim( + job.job_id, + claim_token=claim.claim_token, + claim_generation=claim.job.claim_generation, + heartbeat_sequence=3, + ) + + clock_value[0] = "2026-08-30T21:00:14.000Z" + recovered = queue.recover_stale_claims() + assert [item.job_id for item in recovered] == [job.job_id] + assert recovered[0].state == "queued" + assert recovered[0].active_claim_token is None + assert recovered[0].claim_expires_at_utc is None + assert recovered[0].claim_renewal_count == 0 + with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"): + queue.start(job.job_id, claim_token=claim.claim_token) + with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"): + queue.succeed( + job.job_id, + claim_token=claim.claim_token, + result_id="expired-worker-result", + result_sha256=RESULT_SHA, + ) + + replacement = queue.claim_next( + claimant_id="recorded-worker", + claim_request_id="lease-poll-002", + ) + assert replacement is not None + assert replacement.job.job_id == job.job_id + assert replacement.job.claim_generation == claim.job.claim_generation + 1 + assert replacement.claim_token != claim.claim_token + with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="receipt"): + queue.claim_next( + claimant_id="recorded-worker", + claim_request_id="lease-poll-001", + ) + + +def test_expired_running_claim_is_quarantined_and_stale_terminal_is_fenced( + tmp_path: Path, +) -> None: + clock_value = [NOW] + queue = ObservatoryRecordedJobQueue( + tmp_path, + definitions=_definitions(), + clock=lambda: clock_value[0], + claim_lease_seconds=10, + ) + running, claim = _running_job(queue) + + clock_value[0] = "2026-08-30T21:00:10.000Z" + recovered = queue.recover_stale_claims() + assert [item.job_id for item in recovered] == [running.job_id] + quarantined = recovered[0] + assert quarantined.state == "reconciliation-required" + assert quarantined.terminal_code == "claim-lease-expired" + assert quarantined.active_claim_token is None + with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"): + queue.succeed( + running.job_id, + claim_token=claim.claim_token, + result_id="late-worker-result", + result_sha256=RESULT_SHA, + ) + assert queue.get(running.job_id).result_id is None + assert ( + queue.claim_next( + claimant_id="recorded-worker", + claim_request_id="blocked-after-expired-running", + ) + is None + ) + + +def test_legacy_sqlite_claim_schema_migrates_without_reusing_old_token( + tmp_path: Path, +) -> None: + queue = _queue(tmp_path) + job, _ = queue.submit(_intent(), enqueue=True) + claim = queue.claim_next( + claimant_id="recorded-worker", + claim_request_id="legacy-claim-001", + ) + assert claim is not None + with sqlite3.connect(queue.database_path) as connection: + for column in ( + "claimed_at_utc", + "claim_expires_at_utc", + "claim_heartbeat_at_utc", + "claim_renewal_count", + ): + connection.execute( + f"ALTER TABLE observatory_recorded_jobs DROP COLUMN {column}" + ) + connection.commit() + + migrated_queue = _queue(tmp_path) + migrated = migrated_queue.get(job.job_id) + assert migrated.state == "queued" + assert migrated.claim_generation == 1 + assert migrated.active_claim_token is None + assert migrated.claimed_at_utc is None + assert migrated.claim_expires_at_utc is None + assert migrated.claim_heartbeat_at_utc is None + assert migrated.claim_renewal_count == 0 + with sqlite3.connect(migrated_queue.database_path) as connection: + columns = { + row[1] + for row in connection.execute( + "PRAGMA table_info(observatory_recorded_jobs)" + ).fetchall() + } + assert { + "claimed_at_utc", + "claim_expires_at_utc", + "claim_heartbeat_at_utc", + "claim_renewal_count", + }.issubset(columns) + replacement = migrated_queue.claim_next( + claimant_id="recorded-worker", + claim_request_id="legacy-claim-002", + ) + assert replacement is not None + assert replacement.job.claim_generation == 2 + assert replacement.claim_token != claim.claim_token + + +def test_legacy_sqlite_running_owner_migrates_to_reconciliation( + tmp_path: Path, +) -> None: + queue = _queue(tmp_path) + running, claim = _running_job(queue) + with sqlite3.connect(queue.database_path) as connection: + for column in ( + "claimed_at_utc", + "claim_expires_at_utc", + "claim_heartbeat_at_utc", + "claim_renewal_count", + ): + connection.execute( + f"ALTER TABLE observatory_recorded_jobs DROP COLUMN {column}" + ) + connection.commit() + + migrated_queue = _queue(tmp_path) + migrated = migrated_queue.get(running.job_id) + assert migrated.state == "reconciliation-required" + assert migrated.terminal_code == "claim-lease-migration" + assert migrated.active_claim_token is None + with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"): + migrated_queue.succeed( + running.job_id, + claim_token=claim.claim_token, + result_id="late-legacy-result", + result_sha256=RESULT_SHA, + ) + + def test_single_worker_resource_has_only_one_recorded_owner(tmp_path: Path) -> None: queue = _queue(tmp_path) first, _ = queue.submit(_intent()) diff --git a/tests/test_observatory_setups.py b/tests/test_observatory_setups.py index cf4439d..d538c0e 100644 --- a/tests/test_observatory_setups.py +++ b/tests/test_observatory_setups.py @@ -9,10 +9,15 @@ import pytest from fastapi import FastAPI from fastapi.testclient import TestClient -from k1link.observatory import LaboratorySetupRegistry, LaboratorySetupRegistryError +from k1link.observatory import ( + OBSERVATORY_CALCULATION_PROFILE_SCHEMA, + LaboratorySetupRegistry, + LaboratorySetupRegistryError, +) from k1link.sessions import LabReplayCapability, LabSessionBinding, SessionNotFoundError from k1link.sessions.models import SessionSummary from k1link.web.observatory_api import build_observatory_router +from k1link.web.session_api import build_session_router REPOSITORY_ROOT = Path(__file__).resolve().parents[1] REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-laboratory-setups.json" @@ -163,6 +168,34 @@ def test_repository_setup_registry_keeps_real_definition_and_pre_definition_resu assert existing["preflight"]["submission_allowed"] is False +def test_registry_attributes_only_the_exact_admitted_preserved_result() -> None: + registry = _registry() + projection = _rav004_projection() + + assert registry.observatory_calculation_profile(projection) == { + "schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA, + "setup_id": "lab-v1-ravnoves004tree-final", + "display_name": "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39", + "origin": "existing-result", + "definition_id": None, + "definition_version": None, + "definition_sha256": None, + } + assert registry.observatory_calculation_profile( + replace( + projection, + session_id="lab-v1-vegetation-shadow-" + "0" * 64, + ) + ) is None + assert projection.lab is not None + assert registry.observatory_calculation_profile( + replace( + projection, + lab=replace(projection.lab, provenance={}), + ) + ) is None + + @pytest.mark.parametrize( ("source", "reason_code"), [ @@ -238,6 +271,54 @@ class _Store: raise SessionNotFoundError(session_id) from exc return SimpleNamespace(summary=summary) + def list_recent( + self, + *, + limit: int, + cursor: str | None, + scope: str, + include_capability_projections: bool = False, + ): + del limit, cursor + items = ( + (_rav004_projection(),) + if scope == "laboratory" and include_capability_projections + else () + ) + return SimpleNamespace(items=items, next_cursor=None) + + +def test_session_catalog_v3_projects_the_exact_preserved_profile() -> None: + registry = _registry() + app = FastAPI() + app.include_router( + build_session_router( + _Store(), # type: ignore[arg-type] + lab_calculation_profile_resolver=registry.observatory_calculation_profile, + ) + ) + client = TestClient(app) + + v2_lab = client.get( + "/api/v1/observation-sessions", + params={"scope": "laboratory", "lab_contract": "v2"}, + ).json()["items"][0]["lab"] + v3_lab = client.get( + "/api/v1/observation-sessions", + params={"scope": "laboratory", "lab_contract": "v3"}, + ).json()["items"][0]["lab"] + + assert "calculation_profile" not in v2_lab + assert v3_lab["calculation_profile"] == { + "schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA, + "setup_id": "lab-v1-ravnoves004tree-final", + "display_name": "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39", + "origin": "existing-result", + "definition_id": None, + "definition_version": None, + "definition_sha256": None, + } + def test_observatory_setup_catalog_and_preflight_are_read_only_and_fail_closed() -> None: app = FastAPI() diff --git a/tests/test_observatory_source_admission.py b/tests/test_observatory_source_admission.py index 57f8728..01a9a32 100644 --- a/tests/test_observatory_source_admission.py +++ b/tests/test_observatory_source_admission.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import json from dataclasses import replace from pathlib import Path @@ -334,6 +335,121 @@ def test_portable_source_admission_is_independent_from_session_label( assert capability["camera_profile"]["height"] == 600 +def test_admission_seals_exact_catalogued_spatial_replay_metadata( + tmp_path: Path, +) -> None: + session_id = "20260831T000000Z_viewer_live" + detail = _detail(session_id, "ignored-label") + metadata_path = tmp_path / session_id / "captures/mqtt_live/mqtt.metadata.jsonl" + metadata_path.parent.mkdir(parents=True) + metadata_payload = b'{"offset":0,"topic":"/points"}\n' + metadata_path.write_bytes(metadata_payload) + detail = replace( + detail, + artifacts=( + *detail.artifacts, + SessionArtifact( + artifact_id="raw-transport-index", + kind="raw-transport-index", + media_type="application/x-ndjson", + byte_length=len(metadata_payload), + sha256=None, + integrity_status="verified", + ), + ), + ) + store = _Store(tmp_path, detail) + store.replay = replace( + store.replay, + artifacts=( + *store.replay.artifacts, + ReplayArtifact( + artifact_id="raw-transport-index", + path=metadata_path, + media_type="application/x-ndjson", + file_byte_length=len(metadata_payload), + replay_byte_length=len(metadata_payload), + expected_sha256=None, + ), + ), + ) + service = RecordedK1SourceAdmissionService( + data_dir=tmp_path, + session_store=store, # type: ignore[arg-type] + media_inspector=_Inspector(_manifest(session_id, tmp_path)), # type: ignore[arg-type] + requirements=_requirements(), + ) + + admission = service.check(session_id) + + bundle = json.loads(admission.source_bundle) + metadata = next( + member + for member in bundle["spatial_replay"]["members"] + if member["artifact_id"] == "raw-transport-index" + ) + assert metadata == { + "artifact_id": "raw-transport-index", + "media_type": "application/x-ndjson", + "byte_length": len(metadata_payload), + "replay_byte_length": len(metadata_payload), + "sha256": hashlib.sha256(metadata_payload).hexdigest(), + } + assert "path" not in json.dumps(metadata, sort_keys=True) + + +def test_admission_rejects_spatial_metadata_outside_session_root( + tmp_path: Path, +) -> None: + session_id = "20260831T000000Z_viewer_live" + detail = _detail(session_id, "ignored-label") + (tmp_path / session_id).mkdir() + metadata_path = tmp_path / "outside" / "mqtt.metadata.jsonl" + metadata_path.parent.mkdir(parents=True) + metadata_path.write_bytes(b"{}\n") + detail = replace( + detail, + artifacts=( + *detail.artifacts, + SessionArtifact( + artifact_id="raw-transport-index", + kind="raw-transport-index", + media_type="application/x-ndjson", + byte_length=3, + sha256=None, + integrity_status="verified", + ), + ), + ) + store = _Store(tmp_path, detail) + store.replay = replace( + store.replay, + artifacts=( + *store.replay.artifacts, + ReplayArtifact( + artifact_id="raw-transport-index", + path=metadata_path, + media_type="application/x-ndjson", + file_byte_length=3, + replay_byte_length=3, + expected_sha256=None, + ), + ), + ) + service = RecordedK1SourceAdmissionService( + data_dir=tmp_path, + session_store=store, # type: ignore[arg-type] + media_inspector=_Inspector(_manifest(session_id, tmp_path)), # type: ignore[arg-type] + requirements=_requirements(), + ) + + with pytest.raises( + PortableSourceAdmissionIntegrityError, + match="escapes its admitted session root", + ): + service.check(session_id) + + def test_catalog_capability_check_restores_media_without_preparing_it( tmp_path: Path, ) -> None: diff --git a/tests/test_observatory_worker_agent.py b/tests/test_observatory_worker_agent.py index edc0430..301aa88 100644 --- a/tests/test_observatory_worker_agent.py +++ b/tests/test_observatory_worker_agent.py @@ -19,6 +19,7 @@ from k1link.observatory.worker_agent import ( WORKER_006_CONTOUR_ID, ObservatoryWorkerAgent, ObservatoryWorkerAgentBusyError, + ObservatoryWorkerCycleReport, ObservatoryWorkerExecutionResult, ObservatoryWorkerExecutorIdentity, ObservatoryWorkerExecutorRegistration, @@ -105,6 +106,8 @@ class FakeTransport: starts: list[str] = field(default_factory=list) successes: list[tuple[str, str, str]] = field(default_factory=list) failures: list[tuple[str, str, str]] = field(default_factory=list) + renewals: list[int] = field(default_factory=list) + renewed: Event | None = None def claim_next( self, @@ -132,6 +135,27 @@ class FakeTransport: self.starts.append(job_id) return self.queue.start(job_id, claim_token=claim_token).as_dict() + def renew_claim( + self, + *, + claimant_id: str, + job_id: str, + claim_token: str, + claim_generation: int, + heartbeat_sequence: int, + ) -> Mapping[str, object]: + assert claimant_id == WORKER_006_CONTOUR_ID + renewed = self.queue.renew_claim( + job_id, + claim_token=claim_token, + claim_generation=claim_generation, + heartbeat_sequence=heartbeat_sequence, + ).as_dict() + self.renewals.append(heartbeat_sequence) + if self.renewed is not None: + self.renewed.set() + return renewed + def succeed( self, *, @@ -228,12 +252,137 @@ def test_worker_agent_executes_one_sealed_allowlisted_job(tmp_path: Path) -> Non sealed_job = executor.jobs[0] assert sealed_job.executor_identity == _identity() assert sealed_job.source_bundle_sha256 == SOURCE_BUNDLE_SHA + assert ( + sealed_job.submission_receipt_sha256 + == queue.get(job_id).submission_receipt_sha256 + ) + assert sealed_job.claim_expires_at_utc is not None + assert sealed_job.claim_heartbeat_at_utc is not None + assert sealed_job.claim_renewal_count == 0 assert not hasattr(sealed_job, "command") assert not hasattr(sealed_job, "path") assert not hasattr(sealed_job, "environment") assert queue.get(job_id).state == "succeeded" +@dataclass(slots=True) +class BlockingExecutor: + entered: Event + release: Event + + def execute( + self, + _job: SealedObservatoryRecordedJob, + ) -> ObservatoryWorkerExecutionResult: + self.entered.set() + assert self.release.wait(timeout=2) + return ObservatoryWorkerExecutionResult( + result_id="lab-v1-result", + result_sha256=RESULT_SHA, + ) + + +def _heartbeat_agent( + transport: FakeTransport, + executor: BlockingExecutor, +) -> ObservatoryWorkerAgent: + return ObservatoryWorkerAgent( + transport=transport, + executors=ObservatoryWorkerExecutorRegistry( + ( + ObservatoryWorkerExecutorRegistration( + identity=_identity(), + adapter=executor, + ), + ) + ), + claim_request_id_factory=lambda: "worker-006:heartbeat-cycle", + heartbeat_interval_seconds=0.01, + heartbeat_stop_timeout_seconds=1.0, + ) + + +def test_worker_agent_renews_claim_through_executor_and_upload_window( + tmp_path: Path, +) -> None: + queue = _queue(tmp_path) + job_id = _enqueue(queue) + entered = Event() + release = Event() + renewed = Event() + transport = FakeTransport(queue, renewed=renewed) + agent = _heartbeat_agent( + transport, + BlockingExecutor(entered=entered, release=release), + ) + reports: list[ObservatoryWorkerCycleReport] = [] + + thread = Thread(target=lambda: reports.append(agent.run_once())) + thread.start() + assert entered.wait(timeout=2) + assert renewed.wait(timeout=2) + release.set() + thread.join(timeout=2) + + assert not thread.is_alive() + assert len(reports) == 1 + assert reports[0].state == "succeeded" + assert transport.renewals + assert transport.renewals == list(range(1, len(transport.renewals) + 1)) + assert queue.get(job_id).state == "succeeded" + + +@dataclass(slots=True) +class FailingRenewTransport(FakeTransport): + renewal_attempted: Event = field(default_factory=Event) + + def renew_claim( + self, + *, + claimant_id: str, + job_id: str, + claim_token: str, + claim_generation: int, + heartbeat_sequence: int, + ) -> Mapping[str, object]: + assert claimant_id == WORKER_006_CONTOUR_ID + assert job_id + assert claim_token + assert claim_generation == 1 + assert heartbeat_sequence == 1 + self.renewal_attempted.set() + raise RuntimeError("synthetic heartbeat transport loss") + + +def test_worker_agent_never_seals_result_after_heartbeat_loss(tmp_path: Path) -> None: + queue = _queue(tmp_path) + job_id = _enqueue(queue) + entered = Event() + release = Event() + transport = FailingRenewTransport(queue) + agent = _heartbeat_agent( + transport, + BlockingExecutor(entered=entered, release=release), + ) + reports: list[ObservatoryWorkerCycleReport] = [] + + thread = Thread(target=lambda: reports.append(agent.run_once())) + thread.start() + assert entered.wait(timeout=2) + assert transport.renewal_attempted.wait(timeout=2) + release.set() + thread.join(timeout=2) + + assert not thread.is_alive() + assert len(reports) == 1 + report = reports[0] + assert report.state == "lease-lost" + assert report.failure_code == "claim-heartbeat-lost" + assert transport.successes == [] + assert transport.failures == [] + assert queue.get(job_id).state == "running" + + def test_worker_agent_leaves_empty_queue_untouched(tmp_path: Path) -> None: queue = _queue(tmp_path) transport = FakeTransport(queue) @@ -299,9 +448,31 @@ def _corrupt_job_identity(payload: dict[str, object]) -> dict[str, object]: return changed +def _remove_claim_lease(payload: dict[str, object]) -> dict[str, object]: + changed = deepcopy(payload) + job = changed["job"] + assert isinstance(job, dict) + job["claim_lease"] = None + return changed + + +def _corrupt_submission_receipt(payload: dict[str, object]) -> dict[str, object]: + changed = deepcopy(payload) + job = changed["job"] + assert isinstance(job, dict) + job["submission_receipt_sha256"] = "c" * 64 + return changed + + @pytest.mark.parametrize( "mutator", - [_spoof_claimant, _inject_unknown_execution_payload, _corrupt_job_identity], + [ + _spoof_claimant, + _inject_unknown_execution_payload, + _corrupt_job_identity, + _corrupt_submission_receipt, + _remove_claim_lease, + ], ) def test_spoofed_unknown_or_corrupted_claim_is_rejected_without_execution( tmp_path: Path, @@ -370,6 +541,17 @@ class BlockingEmptyTransport: ) -> Mapping[str, object]: raise AssertionError("an empty transport cannot start a job") + def renew_claim( + self, + *, + claimant_id: str, + job_id: str, + claim_token: str, + claim_generation: int, + heartbeat_sequence: int, + ) -> Mapping[str, object]: + raise AssertionError("an empty transport cannot renew a job") + def succeed( self, *, diff --git a/tests/test_observatory_worker_api.py b/tests/test_observatory_worker_api.py index 4d0886a..7c34aa1 100644 --- a/tests/test_observatory_worker_api.py +++ b/tests/test_observatory_worker_api.py @@ -2,12 +2,15 @@ from __future__ import annotations import hashlib from pathlib import Path -from typing import Any +from typing import Any, cast import pytest from fastapi import FastAPI from fastapi.testclient import TestClient +from k1link.observatory.portable_artifact_transport import ( + PortableArtifactTransportUnavailableError, +) from k1link.observatory.recorded_jobs import ( ObservatoryRecordedJobIntent, ObservatoryRecordedJobQueue, @@ -15,6 +18,8 @@ from k1link.observatory.recorded_jobs import ( RecordedRunDefinitionRegistry, ) from k1link.web.observatory_worker_api import ( + OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, + OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER, OBSERVATORY_WORKER_CONTOUR_HEADER, ObservatoryWorkerAuthentication, build_observatory_worker_router, @@ -29,6 +34,7 @@ WORKER_HEADERS = { } CLAIM_SCHEMA = "missioncore.observatory-worker-claim-request/v1" START_SCHEMA = "missioncore.observatory-worker-start-request/v1" +RENEW_SCHEMA = "missioncore.observatory-worker-renew-request/v1" CHECKPOINT_SCHEMA = "missioncore.observatory-worker-checkpoint-request/v1" SUCCEED_SCHEMA = "missioncore.observatory-worker-succeed-request/v1" FAIL_SCHEMA = "missioncore.observatory-worker-fail-request/v1" @@ -74,6 +80,33 @@ def _services(tmp_path: Path) -> tuple[TestClient, ObservatoryRecordedJobQueue]: return TestClient(app), queue +class _ArtifactTransportWithoutCompletedPackage: + def require_completed_for_success(self, **_values: object) -> Path: + raise PortableArtifactTransportUnavailableError("package is incomplete") + + +def _services_with_artifact_transport( + tmp_path: Path, +) -> tuple[TestClient, ObservatoryRecordedJobQueue]: + definition = _definition() + queue = ObservatoryRecordedJobQueue( + tmp_path, + definitions=RecordedRunDefinitionRegistry((definition,)), + ) + app = FastAPI() + app.include_router( + build_observatory_worker_router( + queue, + authentication=ObservatoryWorkerAuthentication( + bearer_token_sha256=WORKER_TOKEN_SHA256, + contour_id="worker-006", + ), + artifact_transport=_ArtifactTransportWithoutCompletedPackage(), # type: ignore[arg-type] + ) + ) + return TestClient(app), queue + + def _enqueue( queue: ObservatoryRecordedJobQueue, *, @@ -110,7 +143,7 @@ def _claim( }, ) assert response.status_code == 200 - return response.json() + return cast(dict[str, Any], response.json()) def test_worker_authentication_requires_digest_and_configured_contour( @@ -280,6 +313,48 @@ def test_worker_can_start_checkpoint_and_read_job_but_stale_token_fails_closed( assert "active_claim_token" not in fetched.json() +def test_worker_can_renew_exact_claim_lease_idempotently(tmp_path: Path) -> None: + client, queue = _services(tmp_path) + job_id = _enqueue(queue) + claim = _claim(client) + initial_lease = claim["job"]["claim_lease"] + assert initial_lease["renewal_count"] == 0 + request = { + "schema_version": RENEW_SCHEMA, + "claim_token": claim["claim_token"], + "claim_generation": claim["job"]["claim_generation"], + "heartbeat_sequence": 1, + } + + renewed = client.post( + f"/api/v1/worker/observatory/recorded-jobs/{job_id}/lease/renew", + headers=WORKER_HEADERS, + json=request, + ) + repeated = client.post( + f"/api/v1/worker/observatory/recorded-jobs/{job_id}/lease/renew", + headers=WORKER_HEADERS, + json=request, + ) + stale_generation = client.post( + f"/api/v1/worker/observatory/recorded-jobs/{job_id}/lease/renew", + headers=WORKER_HEADERS, + json={**request, "claim_generation": request["claim_generation"] + 1}, + ) + injected = client.post( + f"/api/v1/worker/observatory/recorded-jobs/{job_id}/lease/renew", + headers=WORKER_HEADERS, + json={**request, "command": ["python", "untrusted.py"]}, + ) + + assert renewed.status_code == 200 + assert renewed.json()["claim_lease"]["renewal_count"] == 1 + assert repeated.status_code == 200 + assert repeated.json() == renewed.json() + assert stale_generation.status_code == 409 + assert injected.status_code == 422 + + def test_worker_can_publish_success_idempotently(tmp_path: Path) -> None: client, queue = _services(tmp_path) job_id = _enqueue(queue) @@ -328,6 +403,59 @@ def test_worker_can_publish_success_idempotently(tmp_path: Path) -> None: assert conflicting_failure.status_code == 409 +def test_artifact_transport_blocks_success_without_completed_package( + tmp_path: Path, +) -> None: + client, queue = _services_with_artifact_transport(tmp_path) + job_id = _enqueue(queue) + claim = _claim(client) + claim_token = claim["claim_token"] + generation = claim["job"]["claim_generation"] + client.post( + f"/api/v1/worker/observatory/recorded-jobs/{job_id}/start", + headers=WORKER_HEADERS, + json={"schema_version": START_SCHEMA, "claim_token": claim_token}, + ) + + missing_claim_headers = client.get( + f"/api/v1/worker/observatory/recorded-jobs/{job_id}/source-materialization", + headers=WORKER_HEADERS, + ) + malformed_generation = client.get( + f"/api/v1/worker/observatory/recorded-jobs/{job_id}/source-materialization", + headers={ + **WORKER_HEADERS, + OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER: claim_token, + OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER: "0", + }, + ) + malformed_claim_token = client.get( + f"/api/v1/worker/observatory/recorded-jobs/{job_id}/source-materialization", + headers={ + **WORKER_HEADERS, + OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER: "../../not-a-claim-token", + OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER: str(generation), + }, + ) + blocked = client.post( + f"/api/v1/worker/observatory/recorded-jobs/{job_id}/succeed", + headers=WORKER_HEADERS, + json={ + "schema_version": SUCCEED_SCHEMA, + "claim_token": claim_token, + "result_id": "portable-result-without-upload", + "result_sha256": "b" * 64, + }, + ) + + assert generation == 1 + assert missing_claim_headers.status_code == 422 + assert malformed_generation.status_code == 422 + assert malformed_claim_token.status_code == 422 + assert blocked.status_code == 409 + assert queue.get(job_id).state == "running" + + def test_worker_can_fail_claimed_job_idempotently(tmp_path: Path) -> None: client, queue = _services(tmp_path) job_id = _enqueue(queue) diff --git a/tests/test_observatory_worker_app_wiring.py b/tests/test_observatory_worker_app_wiring.py index 34d1f6c..5d28ace 100644 --- a/tests/test_observatory_worker_app_wiring.py +++ b/tests/test_observatory_worker_app_wiring.py @@ -2,6 +2,14 @@ from __future__ import annotations from pathlib import Path +from k1link.observatory.m49_portable_result import ( + M49_PORTABLE_RESULT_CONTRACT_SHA256, + validate_m49_portable_result, +) +from k1link.observatory.portable_lab_v1_executor import validate_lab_v1_result_v2 +from k1link.observatory.portable_worker_integration import ( + PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256, +) from k1link.web import app as app_module WORKER_ROUTE_PREFIX = "/api/v1/worker/observatory" @@ -9,12 +17,34 @@ WORKER_ROUTE_PREFIX = "/api/v1/worker/observatory" def test_worker_router_is_hard_disabled_until_lease_and_publisher_exist() -> None: assert app_module.OBSERVATORY_RECORDED_JOB_QUEUE is not None + assert app_module.OBSERVATORY_PORTABLE_RESULT_VALIDATORS is not None + assert ( + app_module.OBSERVATORY_PORTABLE_RESULT_VALIDATORS.resolve( + PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256 + ) + is validate_lab_v1_result_v2 + ) + assert ( + app_module.OBSERVATORY_PORTABLE_RESULT_VALIDATORS.resolve( + M49_PORTABLE_RESULT_CONTRACT_SHA256 + ) + is validate_m49_portable_result + ) assert app_module.OBSERVATORY_WORKER_CLAIM_LEASE_READY is False assert app_module.OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY is False assert app_module.OBSERVATORY_WORKER_PRODUCTION_API_ENABLED is False + assert app_module.OBSERVATORY_WORKER_DISPATCH_READY is False assert app_module.OBSERVATORY_WORKER_AUTHENTICATION is None + assert app_module.OBSERVATORY_WORKER_AUTHENTICATION_ERROR is not None assert app_module.OBSERVATORY_WORKER_API_ERROR is not None assert "hard-disabled" in app_module.OBSERVATORY_WORKER_API_ERROR + if app_module.session_artifact_gateway is None: + assert app_module.OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None + assert app_module.OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR is not None + assert ( + "central artifact store" + in app_module.OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR + ) assert not any( getattr(route, "path", "").startswith(WORKER_ROUTE_PREFIX) for route in app_module.app.routes diff --git a/tests/test_observatory_worker_http_transport.py b/tests/test_observatory_worker_http_transport.py new file mode 100644 index 0000000..37bd360 --- /dev/null +++ b/tests/test_observatory_worker_http_transport.py @@ -0,0 +1,515 @@ +from __future__ import annotations + +import hashlib +import json +from collections.abc import Callable +from pathlib import Path + +import httpx +import pytest + +from k1link.observatory.portable_artifact_transport import ( + PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA, + PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA, + PORTABLE_SOURCE_MATERIALIZATION_SCHEMA, +) +from k1link.observatory.portable_result_contract import ( + OBSERVATION_ONLY_AUTHORITY, + PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA, + RESULT_DOCUMENT_ROLE, + PortableResultArtifact, + PortableResultPackageManifest, + canonical_json, +) +from k1link.observatory.portable_run_definitions import canonical_sha256 +from k1link.observatory.portable_worker_runtime import PortableWorkerResultDraft +from k1link.observatory.worker_agent import ( + WORKER_006_CONTOUR_ID, + ObservatoryWorkerExecutorIdentity, + SealedObservatoryRecordedJob, +) +from k1link.observatory.worker_http_transport import ( + ObservatoryWorkerHttpError, + ObservatoryWorkerHttpGateway, +) + +JOB_ID = f"observatory-run-{'1' * 32}" +CLAIM_TOKEN = "2" * 64 +BEARER_TOKEN = "worker-006-test-bearer-token-000001" +NOW = "2026-08-31T11:00:00.000Z" + + +def _job(*, bundle_sha256: str, capability_sha256: str) -> SealedObservatoryRecordedJob: + return SealedObservatoryRecordedJob( + job_id=JOB_ID, + request_sha256="3" * 64, + identity_sha256="4" * 64, + submission_receipt_sha256="c" * 64, + source_session_id="20260831T105500Z_viewer_live", + source_catalog_sha256="5" * 64, + source_bundle_sha256=bundle_sha256, + source_capability_manifest_sha256=capability_sha256, + source_adapter_id="sealed-session-source", + source_adapter_version=1, + source_adapter_sha256="6" * 64, + setup_id="portable-lab-v1", + definition_id="portable-lab-v1-definition", + definition_version=1, + definition_sha256="7" * 64, + executor_release_id="portable-lab-v1-worker", + executor_identity=ObservatoryWorkerExecutorIdentity( + release_sha256="8" * 64, + image_sha256="9" * 64, + model_manifest_sha256="a" * 64, + resource_profile_sha256="b" * 64, + ), + model_release_ids=("eomt-cityscapes-large", "ddrnet-39"), + resource_profile_id="worker006-single-gpu", + checkpoint_policy="cooperative", + allowed_checkpoints=("semantic-pass",), + claim_generation=1, + claim_claimed_at_utc=NOW, + claim_expires_at_utc="2026-08-31T11:05:00.000Z", + claim_heartbeat_at_utc=NOW, + claim_renewal_count=0, + restart_from_zero=False, + ) + + +def _claim_response() -> httpx.Response: + return httpx.Response( + 200, + json={ + "claim_token": CLAIM_TOKEN, + "job": {"job_id": JOB_ID, "claim_generation": 1}, + }, + ) + + +def _cache_claim(gateway: ObservatoryWorkerHttpGateway) -> None: + payload = gateway.claim_next( + claimant_id=WORKER_006_CONTOUR_ID, + claim_request_id="worker-006:http-transport-test", + ) + assert payload is not None + + +def _source_member( + job: SealedObservatoryRecordedJob, + *, + kind: str, + payload: bytes, + artifact_id: str | None = None, + primary: bool = False, + camera_epoch: int | None = None, + camera_sequence: int | None = None, + media_type: str, +) -> tuple[dict[str, object], bytes]: + sha256 = hashlib.sha256(payload).hexdigest() + identity = { + "job_identity_sha256": job.identity_sha256, + "source_bundle_sha256": job.source_bundle_sha256, + "kind": kind, + "artifact_id": artifact_id, + "primary": primary, + "camera_epoch": camera_epoch, + "camera_sequence": camera_sequence, + "media_type": media_type, + "byte_length": len(payload), + "sha256": sha256, + } + return ( + { + "member_id": hashlib.sha256(canonical_json(identity)).hexdigest(), + "kind": kind, + "media_type": media_type, + "byte_length": len(payload), + "sha256": sha256, + "artifact_id": artifact_id, + "primary": primary, + "camera_epoch": camera_epoch, + "camera_sequence": camera_sequence, + }, + payload, + ) + + +def _source_contract( + job: SealedObservatoryRecordedJob, + *, + inject_path: bool = False, +) -> tuple[dict[str, object], dict[str, bytes]]: + rows = [ + _source_member( + job, + kind="source-bundle", + payload=b"source-bundle", + media_type="application/json", + ), + _source_member( + job, + kind="source-capability", + payload=b"source-capability", + media_type="application/json", + ), + _source_member( + job, + kind="spatial-replay", + payload=b"sealed-raw-replay", + artifact_id="raw-primary", + primary=True, + media_type="application/x-nodedc-k1mqtt", + ), + _source_member( + job, + kind="spatial-replay-metadata", + payload=b'{"offset":0,"topic":"/points"}\n', + artifact_id="raw-transport-index", + media_type="application/x-ndjson", + ), + _source_member( + job, + kind="camera-init", + payload=b"sealed-camera-init", + artifact_id="recorded-camera-right", + camera_epoch=1, + media_type='video/mp4; codecs="avc1.641028"', + ), + _source_member( + job, + kind="camera-segment", + payload=b"sealed-camera-segment", + artifact_id="recorded-camera-right", + camera_epoch=1, + camera_sequence=1, + media_type="video/iso.segment", + ), + ] + members = [row for row, _payload in rows] + members.sort(key=lambda row: str(row["member_id"])) + if inject_path: + members[0]["path"] = "../../operator-secret" + payloads = {str(row["member_id"]): payload for row, payload in rows} + return ( + { + "schema_version": PORTABLE_SOURCE_MATERIALIZATION_SCHEMA, + "job_id": job.job_id, + "job_identity_sha256": job.identity_sha256, + "claim_generation": job.claim_generation, + "source": { + "session_id": job.source_session_id, + "bundle_sha256": job.source_bundle_sha256, + "capability_manifest_sha256": ( + job.source_capability_manifest_sha256 + ), + }, + "members": members, + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + }, + payloads, + ) + + +def test_http_gateway_materializes_only_exact_claim_bound_members( + tmp_path: Path, +) -> None: + bundle_payload = b"source-bundle" + capability_payload = b"source-capability" + job = _job( + bundle_sha256=hashlib.sha256(bundle_payload).hexdigest(), + capability_sha256=hashlib.sha256(capability_payload).hexdigest(), + ) + manifest, payloads = _source_contract(job) + artifact_requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/claims"): + assert request.headers["authorization"] == f"Bearer {BEARER_TOKEN}" + assert request.headers["x-mission-core-contour-id"] == "worker-006" + return _claim_response() + assert request.headers["x-mission-core-claim-token"] == CLAIM_TOKEN + assert request.headers["x-mission-core-claim-generation"] == "1" + artifact_requests.append(request) + if request.url.path.endswith("/source-materialization"): + return httpx.Response(200, json=manifest) + member_id = request.url.path.rsplit("/", 1)[-1] + payload = payloads[member_id] + return httpx.Response( + 200, + content=payload, + headers={ + "X-Mission-Core-Content-Sha256": hashlib.sha256(payload).hexdigest() + }, + ) + + with ObservatoryWorkerHttpGateway( + base_url="http://127.0.0.1:18080", + bearer_token=BEARER_TOKEN, + work_root=tmp_path / "worker", + transport=httpx.MockTransport(handler), + ) as gateway: + _cache_claim(gateway) + stage = gateway.materialize(job) + + assert (stage.root / "source-bundle.json").read_bytes() == bundle_payload + assert (stage.root / "source-capability.json").read_bytes() == capability_payload + assert (stage.root / "mqtt.raw.k1mqtt").read_bytes() == b"sealed-raw-replay" + assert (stage.root / "mqtt.metadata.jsonl").read_bytes() == ( + b'{"offset":0,"topic":"/points"}\n' + ) + assert (stage.root / "camera/epoch-1/init.mp4").read_bytes() == b"sealed-camera-init" + assert ( + stage.root / "camera/epoch-1/segments/1.m4s" + ).read_bytes() == b"sealed-camera-segment" + persisted = json.loads( + (stage.root / "materialization-manifest.json").read_text(encoding="utf-8") + ) + assert persisted == manifest + assert "path" not in json.dumps(persisted, sort_keys=True) + assert len(artifact_requests) == 1 + len(payloads) + + +def test_http_gateway_rejects_server_selected_source_path(tmp_path: Path) -> None: + job = _job( + bundle_sha256=hashlib.sha256(b"source-bundle").hexdigest(), + capability_sha256=hashlib.sha256(b"source-capability").hexdigest(), + ) + manifest, _payloads = _source_contract(job, inject_path=True) + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/claims"): + return _claim_response() + return httpx.Response(200, json=manifest) + + with ObservatoryWorkerHttpGateway( + base_url="http://localhost:18080", + bearer_token=BEARER_TOKEN, + work_root=tmp_path / "worker", + transport=httpx.MockTransport(handler), + ) as gateway: + _cache_claim(gateway) + with pytest.raises( + ObservatoryWorkerHttpError, + match="member fields changed", + ): + gateway.materialize(job) + + assert not list((tmp_path / "worker").rglob("operator-secret")) + + +def _result_package( + tmp_path: Path, + job: SealedObservatoryRecordedJob, +) -> tuple[PortableWorkerResultDraft, PortableResultPackageManifest, bytes]: + result_id = "portable-http-result-001" + result_payload = b'{"accepted":true}' + artifact = PortableResultArtifact( + role=RESULT_DOCUMENT_ROLE, + relative_path="artifacts/result.json", + media_type="application/json", + byte_length=len(result_payload), + sha256=hashlib.sha256(result_payload).hexdigest(), + ) + created_at = "2026-08-31T11:01:00.000Z" + job_document: dict[str, object] = { + "job_id": job.job_id, + "request_sha256": job.request_sha256, + "identity_sha256": job.identity_sha256, + "submission_receipt_sha256": job.submission_receipt_sha256, + "claim_generation": job.claim_generation, + } + source_document: dict[str, object] = {} + definition_document: dict[str, object] = {} + result_document: dict[str, object] = {"result_id": result_id} + identity = { + "schema_version": PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA, + "created_at_utc": created_at, + "job": job_document, + "source": source_document, + "run_definition": definition_document, + "result": result_document, + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + "artifacts": [artifact.as_dict()], + } + package = PortableResultPackageManifest( + identity_sha256=canonical_sha256(identity), + created_at_utc=created_at, + job=job_document, + source=source_document, + run_definition=definition_document, + result=result_document, + authority=dict(OBSERVATION_ONLY_AUTHORITY), + artifacts=(artifact,), + ) + root = tmp_path / "draft" + (root / "artifacts").mkdir(parents=True) + (root / "manifest.json").write_bytes(package.canonical_bytes) + (root / "artifacts/result.json").write_bytes(result_payload) + return ( + PortableWorkerResultDraft( + root=root, + result_id=result_id, + result_sha256=package.manifest_sha256, + result_contract_sha256="c" * 64, + ), + package, + result_payload, + ) + + +def _result_handler( + *, + job: SealedObservatoryRecordedJob, + package: PortableResultPackageManifest, + result_payload: bytes, + receipt_mutator: Callable[[dict[str, object]], None] | None = None, +) -> tuple[httpx.MockTransport, list[str]]: + artifact = package.artifacts[0] + member_id = hashlib.sha256( + canonical_json( + { + "package_identity_sha256": package.identity_sha256, + "artifact": artifact.as_dict(), + } + ) + ).hexdigest() + requests: list[str] = [] + + def plan(uploaded: bool) -> dict[str, object]: + return { + "schema_version": PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA, + "job_id": job.job_id, + "claim_generation": job.claim_generation, + "result_id": str(package.result["result_id"]), + "result_sha256": package.manifest_sha256, + "package_identity_sha256": package.identity_sha256, + "members": [ + { + "member_id": member_id, + "role": artifact.role, + "media_type": artifact.media_type, + "byte_length": artifact.byte_length, + "sha256": artifact.sha256, + "uploaded": uploaded, + } + ], + "complete": uploaded, + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/claims"): + return _claim_response() + assert request.headers["x-mission-core-claim-token"] == CLAIM_TOKEN + assert request.headers["x-mission-core-claim-generation"] == "1" + requests.append(request.url.path) + if request.url.path.endswith("/manifest"): + assert request.read() == package.canonical_bytes + return httpx.Response(200, json=plan(False)) + if "/members/" in request.url.path: + assert request.url.path.endswith(member_id) + assert request.read() == result_payload + return httpx.Response(200, json=plan(True)) + receipt_identity: dict[str, object] = { + "schema_version": PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA, + "job_id": job.job_id, + "job_identity_sha256": job.identity_sha256, + "claim_generation": job.claim_generation, + "claim_token_sha256": hashlib.sha256( + CLAIM_TOKEN.encode("ascii") + ).hexdigest(), + "result_id": str(package.result["result_id"]), + "result_sha256": package.manifest_sha256, + "package_identity_sha256": package.identity_sha256, + "member_count": len(package.artifacts), + "total_bytes": sum(item.byte_length for item in package.artifacts), + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + receipt = { + **receipt_identity, + "receipt_sha256": hashlib.sha256( + canonical_json(receipt_identity) + ).hexdigest(), + } + if receipt_mutator is not None: + receipt_mutator(receipt) + return httpx.Response(200, json=receipt) + + return httpx.MockTransport(handler), requests + + +def test_http_gateway_uploads_atomic_package_and_verifies_receipt( + tmp_path: Path, +) -> None: + job = _job(bundle_sha256="d" * 64, capability_sha256="e" * 64) + draft, package, result_payload = _result_package(tmp_path, job) + transport, requests = _result_handler( + job=job, + package=package, + result_payload=result_payload, + ) + + with ObservatoryWorkerHttpGateway( + base_url="https://mission-core.invalid", + bearer_token=BEARER_TOKEN, + work_root=tmp_path / "worker", + transport=transport, + ) as gateway: + _cache_claim(gateway) + result = gateway.publish(job, draft) + + assert result.result_id == draft.result_id + assert result.result_sha256 == draft.result_sha256 + assert any(path.endswith("/manifest") for path in requests) + assert any("/members/" in path for path in requests) + assert any(path.endswith("/complete") for path in requests) + + +def test_http_gateway_rejects_changed_completion_receipt(tmp_path: Path) -> None: + job = _job(bundle_sha256="d" * 64, capability_sha256="e" * 64) + draft, package, result_payload = _result_package(tmp_path, job) + + def mutate(receipt: dict[str, object]) -> None: + receipt["job_identity_sha256"] = "f" * 64 + + transport, _requests = _result_handler( + job=job, + package=package, + result_payload=result_payload, + receipt_mutator=mutate, + ) + with ObservatoryWorkerHttpGateway( + base_url="http://[::1]:18080", + bearer_token=BEARER_TOKEN, + work_root=tmp_path / "worker", + transport=transport, + ) as gateway: + _cache_claim(gateway) + with pytest.raises( + ObservatoryWorkerHttpError, + match="completion receipt differs", + ): + gateway.publish(job, draft) + + +@pytest.mark.parametrize( + "base_url", + [ + "http://mission-core.example", + "http://127.0.0.1:18080/api", + "https://user:secret@mission-core.example", + ], +) +def test_http_gateway_rejects_unsafe_base_urls( + tmp_path: Path, + base_url: str, +) -> None: + with pytest.raises(ValueError, match="base URL|loopback"): + ObservatoryWorkerHttpGateway( + base_url=base_url, + bearer_token=BEARER_TOKEN, + work_root=tmp_path / "worker", + transport=httpx.MockTransport( + lambda _request: httpx.Response(500) + ), + ) diff --git a/tests/test_observatory_worker_service.py b/tests/test_observatory_worker_service.py new file mode 100644 index 0000000..b3d5634 --- /dev/null +++ b/tests/test_observatory_worker_service.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from threading import Event +from typing import cast + +import pytest + +from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry +from k1link.observatory.recorded_jobs import RecordedRunDefinition +from k1link.observatory.worker_agent import ( + ObservatoryWorkerCycleReport, + ObservatoryWorkerExecutionResult, + ObservatoryWorkerExecutorIdentity, + ObservatoryWorkerExecutorRegistration, + ObservatoryWorkerExecutorRegistry, + SealedObservatoryRecordedJob, +) +from k1link.observatory.worker_http_transport import ObservatoryWorkerHttpError +from k1link.observatory.worker_service import ( + InstalledObservatoryWorkerService, + ObservatoryWorkerServiceConfiguration, + ObservatoryWorkerServiceError, + load_observatory_worker_bearer_token, + require_ready_executor_coverage, +) + + +def _definition(*, release_sha256: str = "3" * 64) -> RecordedRunDefinition: + return RecordedRunDefinition( + setup_id="portable-lab-v1", + definition_id="portable-lab-v1-definition", + definition_version=1, + definition_sha256="1" * 64, + source_adapter_id="sealed-session-source", + source_adapter_version=1, + source_adapter_sha256="2" * 64, + executor_release_id="portable-lab-v1-worker", + executor_release_sha256=release_sha256, + executor_image_sha256="4" * 64, + model_release_ids=("eomt-cityscapes-large", "ddrnet-39"), + model_manifest_sha256="5" * 64, + resource_profile_id="worker006-single-gpu", + resource_profile_sha256="6" * 64, + checkpoint_policy="cooperative", + allowed_checkpoints=("semantic-pass",), + ) + + +@dataclass(frozen=True) +class _ReadyDefinitions: + definitions: tuple[RecordedRunDefinition, ...] + + def ready_recorded_definitions(self) -> tuple[RecordedRunDefinition, ...]: + return self.definitions + + +class _Executor: + def execute( + self, + job: SealedObservatoryRecordedJob, + ) -> ObservatoryWorkerExecutionResult: + raise AssertionError(job) + + +def _identity(definition: RecordedRunDefinition) -> ObservatoryWorkerExecutorIdentity: + return ObservatoryWorkerExecutorIdentity( + release_sha256=definition.executor_release_sha256, + image_sha256=definition.executor_image_sha256, + model_manifest_sha256=definition.model_manifest_sha256, + resource_profile_sha256=definition.resource_profile_sha256, + ) + + +def _configuration(tmp_path: Path, **overrides: object) -> ObservatoryWorkerServiceConfiguration: + values: dict[str, object] = { + "base_url": "http://127.0.0.1:18080", + "bearer_token_file": tmp_path / "worker.token", + "work_root": tmp_path / "work", + "idle_poll_seconds": 0.05, + "transport_backoff_seconds": 0.05, + "max_consecutive_transport_failures": 2, + } + values.update(overrides) + return ObservatoryWorkerServiceConfiguration(**values) # type: ignore[arg-type] + + +def test_service_configuration_requires_loopback_for_plain_http(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="loopback"): + _configuration(tmp_path, base_url="http://mission-core.internal:8000") + + configured = ObservatoryWorkerServiceConfiguration.from_environment( + { + "MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE": str(tmp_path / "worker.token"), + "MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT": str(tmp_path / "work"), + } + ) + + assert configured.base_url == "http://127.0.0.1:18080" + + with pytest.raises(ValueError, match="failure bound"): + _configuration(tmp_path, max_consecutive_transport_failures=2.5) + + with pytest.raises(ValueError, match="poll interval"): + _configuration(tmp_path, idle_poll_seconds=True) + + +def test_worker_token_loader_requires_private_exact_ascii_file(tmp_path: Path) -> None: + token = tmp_path / "worker.token" + token.write_text("worker-006-test-bearer-token-000001", encoding="ascii") + token.chmod(0o600) + + assert load_observatory_worker_bearer_token(token) == ( + "worker-006-test-bearer-token-000001" + ) + + token.chmod(0o644) + with pytest.raises(ObservatoryWorkerServiceError, match="permissions"): + load_observatory_worker_bearer_token(token) + + link = tmp_path / "worker-link.token" + link.symlink_to(token) + with pytest.raises(ObservatoryWorkerServiceError, match="unavailable"): + load_observatory_worker_bearer_token(link) + + +def test_install_time_coverage_requires_each_ready_executor_identity() -> None: + definition = _definition() + definitions = cast( + PortableRunDefinitionRegistry, + _ReadyDefinitions((definition,)), + ) + matching = ObservatoryWorkerExecutorRegistry( + (ObservatoryWorkerExecutorRegistration(_identity(definition), _Executor()),) + ) + + assert require_ready_executor_coverage( + definitions=definitions, + executors=matching, + ) == (_identity(definition),) + + with pytest.raises(ObservatoryWorkerServiceError, match="exact local executor"): + require_ready_executor_coverage( + definitions=definitions, + executors=ObservatoryWorkerExecutorRegistry(()), + ) + + with pytest.raises(ObservatoryWorkerServiceError, match="no portable"): + require_ready_executor_coverage( + definitions=cast( + PortableRunDefinitionRegistry, + _ReadyDefinitions(()), + ), + executors=matching, + ) + + +@dataclass +class _FakeGateway: + closed: bool = False + + def close(self) -> None: + self.closed = True + + +@dataclass +class _FakeAgent: + reports: list[ObservatoryWorkerCycleReport] + transport_failure: bool = False + + def run_once(self) -> ObservatoryWorkerCycleReport: + if self.transport_failure: + raise ObservatoryWorkerHttpError("offline") + return self.reports.pop(0) + + +def test_polling_service_stops_cleanly_after_an_empty_cycle(tmp_path: Path) -> None: + stop = Event() + gateway = _FakeGateway() + report = ObservatoryWorkerCycleReport( + state="empty", + claim_request_id="worker-006:test", + ) + service = InstalledObservatoryWorkerService( + configuration=_configuration(tmp_path), + gateway=cast(object, gateway), # type: ignore[arg-type] + agent=cast(object, _FakeAgent([report])), # type: ignore[arg-type] + ) + + service.run(stop=stop, on_cycle=lambda _: stop.set()) + + assert gateway.closed is True + + +def test_polling_service_exits_after_bounded_transport_failures(tmp_path: Path) -> None: + gateway = _FakeGateway() + service = InstalledObservatoryWorkerService( + configuration=_configuration(tmp_path), + gateway=cast(object, gateway), # type: ignore[arg-type] + agent=cast(object, _FakeAgent([], transport_failure=True)), # type: ignore[arg-type] + ) + + with pytest.raises(ObservatoryWorkerServiceError, match="failure bound"): + service.run(stop=Event()) + + assert gateway.closed is True diff --git a/tests/test_observatory_worker_tunnel_launchd.py b/tests/test_observatory_worker_tunnel_launchd.py new file mode 100644 index 0000000..3d779db --- /dev/null +++ b/tests/test_observatory_worker_tunnel_launchd.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import plistlib +from pathlib import Path + +import pytest + +from k1link.observatory.worker_tunnel_launchd import ( + OBSERVATORY_WORKER_TUNNEL_LABEL, + ObservatoryWorkerTunnelPlanError, + plan_observatory_worker_tunnel_launch_agent, +) + + +def _executable(path: Path) -> Path: + path.write_text("#!/bin/sh\n", encoding="utf-8") + path.chmod(0o700) + return path + + +def test_tunnel_plan_is_reverse_loopback_only_and_contains_no_credential( + tmp_path: Path, +) -> None: + data = tmp_path / "mission-core" + data.mkdir(mode=0o700) + ssh = _executable(tmp_path / "ssh") + agent = tmp_path / "worker-tunnel.plist" + + plan = plan_observatory_worker_tunnel_launch_agent( + data_directory=data, + agent_path=agent, + ssh_path=ssh, + ) + document = plistlib.loads(plan.desired_payload) + + assert document["Label"] == OBSERVATORY_WORKER_TUNNEL_LABEL + assert document["ProgramArguments"][-3:] == [ + "-R", + "127.0.0.1:18080:127.0.0.1:8000", + "mission-gpu", + ] + assert "127.0.0.1:18080:127.0.0.1:8000" in document["ProgramArguments"] + assert document["KeepAlive"] is True + assert document["RunAtLoad"] is True + assert document["AbandonProcessGroup"] is False + assert "token" not in plan.desired_payload.decode("utf-8").lower() + assert plan.to_dict()["changes"]["durable_mutation_performed"] is False + + +def test_tunnel_plan_rejects_nonprivate_data_directory(tmp_path: Path) -> None: + data = tmp_path / "mission-core" + data.mkdir(mode=0o755) + data.chmod(0o755) + + with pytest.raises(ObservatoryWorkerTunnelPlanError, match="private"): + plan_observatory_worker_tunnel_launch_agent( + data_directory=data, + agent_path=tmp_path / "agent.plist", + ssh_path=_executable(tmp_path / "ssh"), + ) diff --git a/tests/test_session_api.py b/tests/test_session_api.py index e101c24..4823f42 100644 --- a/tests/test_session_api.py +++ b/tests/test_session_api.py @@ -350,7 +350,7 @@ def test_session_router_exposes_immutable_lab_provenance(tmp_path: Path) -> None assert_no_local_paths((item, detail), repository) -def test_session_router_rolls_capability_projections_out_only_in_v2( +def test_session_router_versions_capability_and_calculation_profile_projections( tmp_path: Path, ) -> None: repository = tmp_path / "repo" @@ -390,7 +390,21 @@ def test_session_router_rolls_capability_projections_out_only_in_v2( "method": lab_method(), }, ) - router = build_session_router(store) + calculation_profile = { + "schema_version": "missioncore.observatory-calculation-profile/v1", + "setup_id": "lab-v1-ravnoves004tree-final", + "display_name": "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39", + "origin": "existing-result", + "definition_id": None, + "definition_version": None, + "definition_sha256": None, + } + router = build_session_router( + store, + lab_calculation_profile_resolver=lambda summary: ( + calculation_profile if summary.session_id == canonical.session_id else None + ), + ) list_route = endpoint(router, "/api/v1/observation-sessions", "GET") default_items = list_route(limit=20, cursor=None, scope="all")["items"] @@ -415,11 +429,30 @@ def test_session_router_rolls_capability_projections_out_only_in_v2( assert set(by_id) == {legacy.session_id, canonical.session_id} assert by_id[legacy.session_id]["lab"]["replay_capability"] is None assert by_id[canonical.session_id]["lab"]["replay_capability"] == capability.as_dict() + assert "calculation_profile" not in by_id[legacy.session_id]["lab"] + assert "calculation_profile" not in by_id[canonical.session_id]["lab"] + + v3_labs = list_route( + limit=20, + cursor=None, + scope="laboratory", + lab_contract="v3", + )["items"] + v3_by_id = {item["id"]: item for item in v3_labs} + assert v3_by_id[legacy.session_id]["lab"]["calculation_profile"] is None + assert ( + v3_by_id[canonical.session_id]["lab"]["calculation_profile"] + == calculation_profile + ) + assert ( + v3_by_id[canonical.session_id]["lab"]["replay_capability"] + == capability.as_dict() + ) application = FastAPI() application.include_router(router) assert TestClient(application).get( - "/api/v1/observation-sessions?lab_contract=v3" + "/api/v1/observation-sessions?lab_contract=v4" ).status_code == 422