feat(observatory): verify exact published result reuse
This commit is contained in:
@@ -1,4 +1,5 @@
|
|||||||
import type {
|
import type {
|
||||||
|
ObservatoryLaboratoryPreservedResult,
|
||||||
ObservatoryLaboratoryRunDefinition,
|
ObservatoryLaboratoryRunDefinition,
|
||||||
ObservatoryLaboratorySetup,
|
ObservatoryLaboratorySetup,
|
||||||
ObservatoryLaboratorySetupCatalog,
|
ObservatoryLaboratorySetupCatalog,
|
||||||
@@ -23,13 +24,14 @@ export function decodePortableCatalog(value: unknown): ObservatoryLaboratorySetu
|
|||||||
);
|
);
|
||||||
exact(row.schema_version, PORTABLE_CATALOG_SCHEMA, "schema_version portable-каталога");
|
exact(row.schema_version, PORTABLE_CATALOG_SCHEMA, "schema_version portable-каталога");
|
||||||
observationAuthority(row.authority);
|
observationAuthority(row.authority);
|
||||||
|
const sourceSessionId = text(row.source_session_id, "portable source_session_id");
|
||||||
return {
|
return {
|
||||||
sourceSessionId: text(row.source_session_id, "portable source_session_id"),
|
sourceSessionId,
|
||||||
setups: array(row.setups, "portable setups").map(decodePortableSetup),
|
setups: array(row.setups, "portable setups").map((item) => decodePortableSetup(item, sourceSessionId)),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function decodePortableSetup(value: unknown): ObservatoryLaboratorySetup {
|
function decodePortableSetup(value: unknown, sourceSessionId: string): ObservatoryLaboratorySetup {
|
||||||
const row = record(value, "portable-сетап");
|
const row = record(value, "portable-сетап");
|
||||||
exactKeys(row, [
|
exactKeys(row, [
|
||||||
"authority", "description", "display_name", "executor", "existing_results",
|
"authority", "description", "display_name", "executor", "existing_results",
|
||||||
@@ -39,6 +41,8 @@ function decodePortableSetup(value: unknown): ObservatoryLaboratorySetup {
|
|||||||
exact(row.origin, "portable-definition", "portable origin");
|
exact(row.origin, "portable-definition", "portable origin");
|
||||||
observationAuthority(row.authority);
|
observationAuthority(row.authority);
|
||||||
decodePortableSourceRequirements(row.source_requirements);
|
decodePortableSourceRequirements(row.source_requirements);
|
||||||
|
const setupId = text(row.setup_id, "portable setup_id");
|
||||||
|
const runDefinition = decodePortableRunDefinition(row.run_definition);
|
||||||
|
|
||||||
const compatibility = record(row.source_compatibility, "portable source_compatibility");
|
const compatibility = record(row.source_compatibility, "portable source_compatibility");
|
||||||
exactKeys(
|
exactKeys(
|
||||||
@@ -96,12 +100,12 @@ function decodePortableSetup(value: unknown): ObservatoryLaboratorySetup {
|
|||||||
], "portable preflight");
|
], "portable preflight");
|
||||||
const preflightOutcome = oneOf(
|
const preflightOutcome = oneOf(
|
||||||
preflight.outcome,
|
preflight.outcome,
|
||||||
["ready", "blocked"] as const,
|
["existing", "ready", "blocked"] as const,
|
||||||
"portable preflight outcome",
|
"portable preflight outcome",
|
||||||
);
|
);
|
||||||
const preflightAction = oneOf(
|
const preflightAction = oneOf(
|
||||||
preflight.action,
|
preflight.action,
|
||||||
["check", "blocked"] as const,
|
["open-existing", "check", "blocked"] as const,
|
||||||
"portable preflight action",
|
"portable preflight action",
|
||||||
);
|
);
|
||||||
const submissionAllowed = boolean(
|
const submissionAllowed = boolean(
|
||||||
@@ -112,28 +116,37 @@ function decodePortableSetup(value: unknown): ObservatoryLaboratorySetup {
|
|||||||
submissionAllowed !== (preflightOutcome === "ready")
|
submissionAllowed !== (preflightOutcome === "ready")
|
||||||
|| (preflightOutcome === "ready" && preflightAction !== "check")
|
|| (preflightOutcome === "ready" && preflightAction !== "check")
|
||||||
|| (preflightOutcome === "blocked" && preflightAction !== "blocked")
|
|| (preflightOutcome === "blocked" && preflightAction !== "blocked")
|
||||||
|
|| (preflightOutcome === "existing" && preflightAction !== "open-existing")
|
||||||
|
|| (submissionAllowed && (!compatible || !executorReady))
|
||||||
) {
|
) {
|
||||||
throw new ObservatoryPortableSetupDecodeError(
|
throw new ObservatoryPortableSetupDecodeError(
|
||||||
"Portable preflight: состояние запуска противоречиво.",
|
"Portable preflight: состояние запуска противоречиво.",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const existingResults = array(row.existing_results, "portable existing_results");
|
const existingResults = array(row.existing_results, "portable existing_results").map(
|
||||||
|
(item) => decodePortableResult(item, sourceSessionId, setupId, runDefinition),
|
||||||
|
);
|
||||||
const existingResultIds = array(
|
const existingResultIds = array(
|
||||||
preflight.existing_result_ids,
|
preflight.existing_result_ids,
|
||||||
"portable existing_result_ids",
|
"portable existing_result_ids",
|
||||||
);
|
).map((item) => text(item, "portable existing result id"));
|
||||||
if (existingResults.length > 0 || existingResultIds.length > 0) {
|
if (
|
||||||
|
(existingResults.length > 0) !== (preflightOutcome === "existing")
|
||||||
|
|| existingResultIds.length !== existingResults.length
|
||||||
|
|| new Set(existingResultIds).size !== existingResultIds.length
|
||||||
|
|| existingResults.some((result, index) => result.resultId !== existingResultIds[index])
|
||||||
|
) {
|
||||||
throw new ObservatoryPortableSetupDecodeError(
|
throw new ObservatoryPortableSetupDecodeError(
|
||||||
"Portable result: проверяемая привязка результата к RunDefinition ещё не поддерживается.",
|
"Portable result: готовность не соответствует проверенным результатам.",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
setupId: text(row.setup_id, "portable setup_id"),
|
setupId,
|
||||||
displayName: text(row.display_name, "portable display_name"),
|
displayName: text(row.display_name, "portable display_name"),
|
||||||
description: text(row.description, "portable description"),
|
description: text(row.description, "portable description"),
|
||||||
origin: "portable-definition",
|
origin: "portable-definition",
|
||||||
runDefinition: decodePortableRunDefinition(row.run_definition),
|
runDefinition,
|
||||||
compatibility: {
|
compatibility: {
|
||||||
compatible,
|
compatible,
|
||||||
reasons: compatible
|
reasons: compatible
|
||||||
@@ -146,17 +159,58 @@ function decodePortableSetup(value: unknown): ObservatoryLaboratorySetup {
|
|||||||
reasonCode: executorReasonCode,
|
reasonCode: executorReasonCode,
|
||||||
reason: executorReason,
|
reason: executorReason,
|
||||||
},
|
},
|
||||||
preservedResults: [],
|
preservedResults: existingResults,
|
||||||
preflight: {
|
preflight: {
|
||||||
outcome: preflightOutcome,
|
outcome: preflightOutcome,
|
||||||
action: preflightAction,
|
action: preflightAction,
|
||||||
reason: text(preflight.reason, "portable preflight reason"),
|
reason: text(preflight.reason, "portable preflight reason"),
|
||||||
submissionAllowed,
|
submissionAllowed,
|
||||||
existingResultIds: [],
|
existingResultIds,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function decodePortableResult(
|
||||||
|
value: unknown, sourceSessionId: string, setupId: string,
|
||||||
|
definition: ObservatoryLaboratoryRunDefinition,
|
||||||
|
): ObservatoryLaboratoryPreservedResult {
|
||||||
|
const row = record(value, "portable result");
|
||||||
|
exactKeys(row, [
|
||||||
|
"result_id", "result_kind", "relation", "access", "created_at_utc",
|
||||||
|
"observatory_projection_available", "identity",
|
||||||
|
], "portable result");
|
||||||
|
exact(row.result_kind, definition.resultKind, "portable result kind");
|
||||||
|
exact(row.relation, "exact-recorded-computation", "portable result relation");
|
||||||
|
exact(row.access, "observatory", "portable result access");
|
||||||
|
exact(row.observatory_projection_available, true, "portable result availability");
|
||||||
|
const identity = record(row.identity, "portable result identity");
|
||||||
|
exactKeys(identity, [
|
||||||
|
"job_id", "source_session_id", "source_catalog_sha256", "source_bundle_sha256",
|
||||||
|
"source_capability_manifest_sha256", "setup_id", "definition_sha256",
|
||||||
|
"package_sha256", "artifact_manifest_id",
|
||||||
|
], "portable result identity");
|
||||||
|
exact(identity.source_session_id, sourceSessionId, "portable result source");
|
||||||
|
exact(identity.setup_id, setupId, "portable result setup");
|
||||||
|
exact(identity.definition_sha256, definition.definitionSha256, "portable result definition");
|
||||||
|
text(identity.job_id, "portable result job");
|
||||||
|
for (const key of [
|
||||||
|
"source_catalog_sha256", "source_bundle_sha256", "source_capability_manifest_sha256",
|
||||||
|
"definition_sha256", "package_sha256", "artifact_manifest_id",
|
||||||
|
]) {
|
||||||
|
if (!SHA256.test(text(identity[key], `portable result ${key}`))) {
|
||||||
|
throw new ObservatoryPortableSetupDecodeError(`Portable result: некорректный ${key}.`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
resultId: text(row.result_id, "portable result id"),
|
||||||
|
resultKind: text(row.result_kind, "portable result kind"),
|
||||||
|
relation: "exact-recorded-computation",
|
||||||
|
access: "observatory",
|
||||||
|
createdAtUtc: text(row.created_at_utc, "portable result created_at_utc"),
|
||||||
|
observatoryProjectionAvailable: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function decodePortableSourceRequirements(value: unknown): void {
|
function decodePortableSourceRequirements(value: unknown): void {
|
||||||
const row = record(value, "portable source_requirements");
|
const row = record(value, "portable source_requirements");
|
||||||
exactKeys(row, [
|
exactKeys(row, [
|
||||||
|
|||||||
@@ -323,7 +323,7 @@ test("portable LAB V1 rejects an unbound existing result projection", async () =
|
|||||||
authority,
|
authority,
|
||||||
}), { status: 200 }),
|
}), { status: 200 }),
|
||||||
}),
|
}),
|
||||||
/значение изменилось|значение не поддерживается|проверяемая привязка результата/,
|
/значение изменилось|значение не поддерживается|обнаружены неизвестные поля/,
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -348,6 +348,76 @@ test("portable LAB V1 rejects heavyweight compatibility evidence", async () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
function cachedPortableSetup() {
|
||||||
|
const setup = portableSetup();
|
||||||
|
setup.existing_results = [{
|
||||||
|
result_id: "portable-result-001",
|
||||||
|
result_kind: setup.run_definition.result_kind,
|
||||||
|
relation: "exact-recorded-computation",
|
||||||
|
access: "observatory",
|
||||||
|
created_at_utc: "2026-09-03T00:00:00Z",
|
||||||
|
observatory_projection_available: true,
|
||||||
|
identity: {
|
||||||
|
job_id: "observatory-run-001",
|
||||||
|
source_session_id: "source-a",
|
||||||
|
source_catalog_sha256: "a".repeat(64),
|
||||||
|
source_bundle_sha256: "b".repeat(64),
|
||||||
|
source_capability_manifest_sha256: "c".repeat(64),
|
||||||
|
setup_id: setup.setup_id,
|
||||||
|
definition_sha256: setup.run_definition.definition_sha256,
|
||||||
|
package_sha256: "d".repeat(64),
|
||||||
|
artifact_manifest_id: "e".repeat(64),
|
||||||
|
},
|
||||||
|
}];
|
||||||
|
setup.preflight = {
|
||||||
|
outcome: "existing", action: "open-existing",
|
||||||
|
reason: "Точный результат проверен.", submission_allowed: false,
|
||||||
|
existing_result_ids: [setup.existing_results[0].result_id],
|
||||||
|
};
|
||||||
|
return setup;
|
||||||
|
}
|
||||||
|
|
||||||
|
function fetchCachedPortable(setup) {
|
||||||
|
return fetchObservatoryPortableLaboratorySetups("source-a", {
|
||||||
|
fetcher: async () => new Response(JSON.stringify({
|
||||||
|
schema_version: "missioncore.observatory-portable-setup-catalog/v2",
|
||||||
|
source_session_id: "source-a", setups: [setup], authority,
|
||||||
|
}), { status: 200 }),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
test("portable exact cached result is readable without an installed executor", async () => {
|
||||||
|
const catalog = await fetchCachedPortable(cachedPortableSetup());
|
||||||
|
const setup = catalog.setups[0];
|
||||||
|
assert.equal(setup.executor.state, "not-installed");
|
||||||
|
assert.equal(setup.preflight.outcome, "existing");
|
||||||
|
assert.equal(setup.preflight.submissionAllowed, false);
|
||||||
|
assert.deepEqual(setup.preflight.existingResultIds, ["portable-result-001"]);
|
||||||
|
assert.equal(setup.preservedResults[0].access, "observatory");
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const [label, change] of [
|
||||||
|
["another source", (s) => { s.existing_results[0].identity.source_session_id = "source-b"; }],
|
||||||
|
["another setup", (s) => { s.existing_results[0].identity.setup_id = "another-profile"; }],
|
||||||
|
["another version", (s) => { s.existing_results[0].identity.definition_sha256 = "f".repeat(64); }],
|
||||||
|
["bad package digest", (s) => { s.existing_results[0].identity.package_sha256 = "not-a-digest"; }],
|
||||||
|
["unavailable artifact", (s) => { s.existing_results[0].observatory_projection_available = false; }],
|
||||||
|
["missing binding", (s) => { delete s.existing_results[0].identity; }],
|
||||||
|
["unrelated result ID", (s) => { s.preflight.existing_result_ids = ["different-result"]; }],
|
||||||
|
["duplicate result", (s) => {
|
||||||
|
s.existing_results.push(s.existing_results[0]);
|
||||||
|
s.preflight.existing_result_ids.push(s.preflight.existing_result_ids[0]);
|
||||||
|
}],
|
||||||
|
["contradictory action", (s) => { s.preflight.action = "check"; }],
|
||||||
|
["cached but queueable", (s) => { s.preflight.submission_allowed = true; }],
|
||||||
|
]) {
|
||||||
|
test(`portable cached result rejects ${label}`, async () => {
|
||||||
|
const setup = cachedPortableSetup();
|
||||||
|
change(setup);
|
||||||
|
await assert.rejects(fetchCachedPortable(setup));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
test("Observatory preflight sends the exact selected definition and never submits a run", async () => {
|
test("Observatory preflight sends the exact selected definition and never submits a run", async () => {
|
||||||
const selected = (await fetchObservatoryLaboratorySetups("source-a", {
|
const selected = (await fetchObservatoryLaboratorySetups("source-a", {
|
||||||
fetcher: async () => new Response(JSON.stringify({
|
fetcher: async () => new Response(JSON.stringify({
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ from __future__ import annotations
|
|||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Final
|
from typing import Final
|
||||||
@@ -109,6 +110,7 @@ class PortableRecordedQueueBindingService:
|
|||||||
media_inspector: RecordedMediaInspector,
|
media_inspector: RecordedMediaInspector,
|
||||||
definitions: PortableRunDefinitionRegistry,
|
definitions: PortableRunDefinitionRegistry,
|
||||||
queue: ObservatoryRecordedJobQueue | None = None,
|
queue: ObservatoryRecordedJobQueue | None = None,
|
||||||
|
published_result_available: Callable[[ObservatoryRecordedJob], bool] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.data_dir = data_dir.expanduser().resolve()
|
self.data_dir = data_dir.expanduser().resolve()
|
||||||
if self.data_dir != session_store.data_dir:
|
if self.data_dir != session_store.data_dir:
|
||||||
@@ -123,6 +125,7 @@ class PortableRecordedQueueBindingService:
|
|||||||
self._media_inspector = media_inspector
|
self._media_inspector = media_inspector
|
||||||
self._definitions = definitions
|
self._definitions = definitions
|
||||||
self._queue = queue
|
self._queue = queue
|
||||||
|
self._published_result_available = published_result_available
|
||||||
|
|
||||||
def probe(
|
def probe(
|
||||||
self,
|
self,
|
||||||
@@ -215,6 +218,7 @@ class PortableRecordedQueueBindingService:
|
|||||||
preparation.intent(idempotency_key=idempotency_key),
|
preparation.intent(idempotency_key=idempotency_key),
|
||||||
enqueue=enqueue,
|
enqueue=enqueue,
|
||||||
reject_duplicate_computation=True,
|
reject_duplicate_computation=True,
|
||||||
|
published_result_available=self._published_result_available,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _resolve_definition(
|
def _resolve_definition(
|
||||||
|
|||||||
@@ -0,0 +1,305 @@
|
|||||||
|
"""Verified recorded-result reuse without inference or source replay preparation."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import stat
|
||||||
|
from collections import OrderedDict
|
||||||
|
from dataclasses import fields
|
||||||
|
from pathlib import Path
|
||||||
|
from threading import RLock
|
||||||
|
|
||||||
|
from k1link.artifact_gateway import ArtifactGatewayError, CentralArtifactStore
|
||||||
|
from k1link.observatory.portable_result_contract import (
|
||||||
|
PORTABLE_RESULT_PACKAGE_SCHEMA,
|
||||||
|
RESULT_DOCUMENT_ROLE,
|
||||||
|
RESULT_PACKAGE_MANIFEST_ROLE,
|
||||||
|
PortableCalculationProfileRegistry,
|
||||||
|
PortableResultPackageManifest,
|
||||||
|
PortableResultPublisherError,
|
||||||
|
job_identity_document,
|
||||||
|
result_identity_document,
|
||||||
|
run_definition_document,
|
||||||
|
source_identity_document,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_result_publisher import (
|
||||||
|
resolve_published_portable_calculation_profile,
|
||||||
|
)
|
||||||
|
from k1link.observatory.portable_run_definitions import (
|
||||||
|
PortableRunDefinition,
|
||||||
|
PortableRunDefinitionRegistry,
|
||||||
|
PortableRunDefinitionRegistryError,
|
||||||
|
)
|
||||||
|
from k1link.observatory.recorded_jobs import (
|
||||||
|
ObservatoryRecordedJob,
|
||||||
|
ObservatoryRecordedJobQueue,
|
||||||
|
ObservatoryRecordedQueueConflictError,
|
||||||
|
)
|
||||||
|
from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore
|
||||||
|
|
||||||
|
|
||||||
|
class PortableResultCacheCheckRequired(ObservatoryRecordedQueueConflictError):
|
||||||
|
"""Refresh verifies a newly published/changed object outside the queue write lock."""
|
||||||
|
|
||||||
|
|
||||||
|
class PortableResultCache:
|
||||||
|
"""Join the durable publication receipt, Session binding and immutable CAS.
|
||||||
|
|
||||||
|
Content checks are memoized by filesystem fingerprint, not unconditional availability.
|
||||||
|
Every lookup rechecks metadata. First use/changed metadata streams SHA-256
|
||||||
|
with bounded memory; no videos, RRDs or model tensors are kept in RAM.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
sessions: SessionStore,
|
||||||
|
artifacts: CentralArtifactStore,
|
||||||
|
queue: ObservatoryRecordedJobQueue,
|
||||||
|
definitions: PortableRunDefinitionRegistry,
|
||||||
|
calculation_profiles: PortableCalculationProfileRegistry,
|
||||||
|
) -> None:
|
||||||
|
if sessions.data_dir != queue.data_dir:
|
||||||
|
raise ValueError("portable result cache and queue roots disagree")
|
||||||
|
self._sessions = sessions
|
||||||
|
self._artifacts = artifacts
|
||||||
|
self._queue = queue
|
||||||
|
self._definitions = definitions
|
||||||
|
self._profiles = calculation_profiles
|
||||||
|
self._verified: OrderedDict[tuple[Path, str], tuple[tuple[int, ...], bool]] = OrderedDict()
|
||||||
|
self._lock = RLock()
|
||||||
|
|
||||||
|
def find(
|
||||||
|
self,
|
||||||
|
source_session_id: str,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
) -> tuple[dict[str, object], ...]:
|
||||||
|
"""Return the newest usable exact result; older evidence remains untouched."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
source, catalog_sha256 = self._sessions.get_session_with_catalog_snapshot(
|
||||||
|
source_session_id
|
||||||
|
)
|
||||||
|
except (SessionNotFoundError, SessionIntegrityError):
|
||||||
|
return ()
|
||||||
|
if source.summary.lab is not None:
|
||||||
|
return ()
|
||||||
|
candidates = self._queue.published_results(
|
||||||
|
source_session_id=source_session_id,
|
||||||
|
source_catalog_sha256=catalog_sha256,
|
||||||
|
setup_id=definition.setup_id,
|
||||||
|
definition_sha256=definition.definition_sha256,
|
||||||
|
)
|
||||||
|
for job in candidates:
|
||||||
|
result = self._projection(job)
|
||||||
|
if result is not None:
|
||||||
|
return (result,)
|
||||||
|
return ()
|
||||||
|
|
||||||
|
def available(self, job: ObservatoryRecordedJob) -> bool:
|
||||||
|
"""Queue INSERT fence: verify this row without re-entering the queue DB."""
|
||||||
|
|
||||||
|
if not self._lock.acquire(blocking=False):
|
||||||
|
raise PortableResultCacheCheckRequired("published result verification is in progress")
|
||||||
|
try:
|
||||||
|
return self._projection(job, warm_only=True) is not None
|
||||||
|
finally:
|
||||||
|
self._lock.release()
|
||||||
|
|
||||||
|
def _projection(
|
||||||
|
self,
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
*,
|
||||||
|
warm_only: bool = False,
|
||||||
|
) -> dict[str, object] | None:
|
||||||
|
if (
|
||||||
|
job.state != "succeeded"
|
||||||
|
or job.publication_state != "published"
|
||||||
|
or job.result_id is None
|
||||||
|
or job.result_sha256 is None
|
||||||
|
or job.terminal_code != "result-sealed"
|
||||||
|
or job.terminal_claim_token_sha256 is None
|
||||||
|
or job.claim_generation < 1
|
||||||
|
or job.active_claim_token is not None
|
||||||
|
or job.active_claimant_id is not None
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
definition = self._definitions.resolve(job.setup_id, job.definition_sha256)
|
||||||
|
recorded = definition.to_recorded_run_definition()
|
||||||
|
if any(
|
||||||
|
getattr(recorded, field.name) != getattr(job, field.name)
|
||||||
|
for field in fields(recorded)
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
summary = self._sessions.get_session(job.result_id).summary
|
||||||
|
profile = self._profiles.resolve(definition)
|
||||||
|
if (
|
||||||
|
resolve_published_portable_calculation_profile(
|
||||||
|
summary,
|
||||||
|
definitions=self._definitions,
|
||||||
|
calculation_profiles=self._profiles,
|
||||||
|
)
|
||||||
|
!= profile.as_dict()
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
binding = summary.lab
|
||||||
|
assert binding is not None
|
||||||
|
provenance = binding.provenance
|
||||||
|
published_package = provenance.get("result_package")
|
||||||
|
if (
|
||||||
|
provenance.get("job") != job_identity_document(job)
|
||||||
|
or provenance.get("source") != source_identity_document(job)
|
||||||
|
or not isinstance(published_package, dict)
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
manifest_id = published_package.get("artifact_manifest_id")
|
||||||
|
if not isinstance(manifest_id, str):
|
||||||
|
return None
|
||||||
|
manifest = self._artifacts.read_manifest(manifest_id)
|
||||||
|
package_member = manifest.member(RESULT_PACKAGE_MANIFEST_ROLE)
|
||||||
|
if (
|
||||||
|
package_member.sha256 != job.result_sha256
|
||||||
|
or package_member.media_type != "application/json"
|
||||||
|
or not 0 < package_member.byte_length <= 1024 * 1024
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
package_path = self._artifacts.object_path(package_member.sha256)
|
||||||
|
# This small contract is parsed on every lookup, never trusted by stat alone.
|
||||||
|
_fingerprint(package_path, package_member.byte_length)
|
||||||
|
with package_path.open("rb") as stream:
|
||||||
|
payload = stream.read(1024 * 1024 + 1)
|
||||||
|
package = PortableResultPackageManifest.from_bytes(payload)
|
||||||
|
if (
|
||||||
|
len(payload) != package_member.byte_length
|
||||||
|
or package.manifest_sha256 != job.result_sha256
|
||||||
|
or package.job != job_identity_document(job)
|
||||||
|
or package.source != source_identity_document(job)
|
||||||
|
or package.run_definition != run_definition_document(definition)
|
||||||
|
or package.result != result_identity_document(definition, job.result_id)
|
||||||
|
or manifest.artifact_type != "observatory-portable-result"
|
||||||
|
or manifest.subject_id != job.result_id
|
||||||
|
or dict(manifest.metadata)
|
||||||
|
!= {
|
||||||
|
"package-sha256": job.result_sha256,
|
||||||
|
"package-identity-sha256": package.identity_sha256,
|
||||||
|
"job-id": job.job_id,
|
||||||
|
"job-identity-sha256": job.identity_sha256,
|
||||||
|
"definition-sha256": job.definition_sha256,
|
||||||
|
"source-bundle-sha256": job.source_bundle_sha256,
|
||||||
|
"result-contract-sha256": definition.result_contract.contract_sha256,
|
||||||
|
"calculation-profile-sha256": profile.identity_sha256,
|
||||||
|
}
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
result_document = next(
|
||||||
|
item for item in package.artifacts if item.role == RESULT_DOCUMENT_ROLE
|
||||||
|
)
|
||||||
|
if published_package != {
|
||||||
|
"schema_version": PORTABLE_RESULT_PACKAGE_SCHEMA,
|
||||||
|
"manifest_sha256": job.result_sha256,
|
||||||
|
"identity_sha256": package.identity_sha256,
|
||||||
|
"artifact_manifest_id": manifest_id,
|
||||||
|
"result_document_sha256": result_document.sha256,
|
||||||
|
"artifacts": [item.as_dict() for item in package.artifacts],
|
||||||
|
}:
|
||||||
|
return None
|
||||||
|
expected_members = {
|
||||||
|
item.role: (item.sha256, item.byte_length, item.media_type)
|
||||||
|
for item in package.artifacts
|
||||||
|
}
|
||||||
|
expected_members[RESULT_PACKAGE_MANIFEST_ROLE] = (
|
||||||
|
job.result_sha256,
|
||||||
|
len(payload),
|
||||||
|
"application/json",
|
||||||
|
)
|
||||||
|
if expected_members != {
|
||||||
|
item.role: (item.sha256, item.byte_length, item.media_type)
|
||||||
|
for item in manifest.members
|
||||||
|
}:
|
||||||
|
return None
|
||||||
|
for member in manifest.members:
|
||||||
|
self._verify_file(
|
||||||
|
self._artifacts.object_path(member.sha256),
|
||||||
|
member.sha256,
|
||||||
|
member.byte_length,
|
||||||
|
warm_only=warm_only,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"result_id": job.result_id,
|
||||||
|
"result_kind": binding.result_kind,
|
||||||
|
"relation": "exact-recorded-computation",
|
||||||
|
"access": "observatory",
|
||||||
|
"created_at_utc": binding.run_created_at_utc,
|
||||||
|
"observatory_projection_available": True,
|
||||||
|
"identity": {
|
||||||
|
"job_id": job.job_id,
|
||||||
|
"source_session_id": job.source_session_id,
|
||||||
|
"source_catalog_sha256": job.source_catalog_sha256,
|
||||||
|
"source_bundle_sha256": job.source_bundle_sha256,
|
||||||
|
"source_capability_manifest_sha256": job.source_capability_manifest_sha256,
|
||||||
|
"setup_id": job.setup_id,
|
||||||
|
"definition_sha256": job.definition_sha256,
|
||||||
|
"package_sha256": job.result_sha256,
|
||||||
|
"artifact_manifest_id": manifest_id,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
except (
|
||||||
|
ArtifactGatewayError,
|
||||||
|
PortableResultPublisherError,
|
||||||
|
PortableRunDefinitionRegistryError,
|
||||||
|
SessionNotFoundError,
|
||||||
|
SessionIntegrityError,
|
||||||
|
OSError,
|
||||||
|
ValueError,
|
||||||
|
KeyError,
|
||||||
|
TypeError,
|
||||||
|
):
|
||||||
|
# Missing/corrupt cache is not proof of computation coverage.
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _verify_file(
|
||||||
|
self,
|
||||||
|
path: Path,
|
||||||
|
digest: str,
|
||||||
|
byte_length: int,
|
||||||
|
*,
|
||||||
|
warm_only: bool,
|
||||||
|
) -> None:
|
||||||
|
with self._lock:
|
||||||
|
before = _fingerprint(path, byte_length)
|
||||||
|
key = (path, digest)
|
||||||
|
cached = self._verified.get(key)
|
||||||
|
if cached is not None and cached[0] == before:
|
||||||
|
self._verified.move_to_end(key)
|
||||||
|
if not cached[1]:
|
||||||
|
raise ValueError("cached result object digest is invalid")
|
||||||
|
return
|
||||||
|
self._verified.pop(key, None)
|
||||||
|
if warm_only:
|
||||||
|
raise PortableResultCacheCheckRequired("refresh published result verification")
|
||||||
|
checksum = hashlib.sha256()
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
while chunk := stream.read(1024 * 1024):
|
||||||
|
checksum.update(chunk)
|
||||||
|
if _fingerprint(path, byte_length) != before:
|
||||||
|
raise ValueError("cached result object changed")
|
||||||
|
valid = checksum.hexdigest() == digest
|
||||||
|
self._verified[key] = (before, valid)
|
||||||
|
while len(self._verified) > 4096:
|
||||||
|
self._verified.popitem(last=False)
|
||||||
|
if not valid:
|
||||||
|
raise ValueError("cached result object digest is invalid")
|
||||||
|
|
||||||
|
|
||||||
|
def _fingerprint(path: Path, byte_length: int) -> tuple[int, ...]:
|
||||||
|
metadata = path.lstat()
|
||||||
|
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size != byte_length:
|
||||||
|
raise ValueError("cached result object is unavailable or not regular")
|
||||||
|
return (
|
||||||
|
metadata.st_dev,
|
||||||
|
metadata.st_ino,
|
||||||
|
metadata.st_size,
|
||||||
|
metadata.st_mtime_ns,
|
||||||
|
metadata.st_ctime_ns,
|
||||||
|
)
|
||||||
@@ -22,6 +22,7 @@ from k1link.observatory.portable_run_definitions import (
|
|||||||
PortableRunDefinitionRegistry,
|
PortableRunDefinitionRegistry,
|
||||||
PortableRunDefinitionRegistryError,
|
PortableRunDefinitionRegistryError,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.recorded_jobs import ObservatoryRecordedQueueError
|
||||||
from k1link.observatory.source_admission import (
|
from k1link.observatory.source_admission import (
|
||||||
PortableRecordedSourceCapability,
|
PortableRecordedSourceCapability,
|
||||||
PortableSourceAdmissionError,
|
PortableSourceAdmissionError,
|
||||||
@@ -90,6 +91,14 @@ class PortableDefinitionCapabilityProbe(Protocol):
|
|||||||
type PortableSourceCapabilityProbe = Callable[[str], PortableRecordedSourceCapability]
|
type PortableSourceCapabilityProbe = Callable[[str], PortableRecordedSourceCapability]
|
||||||
|
|
||||||
|
|
||||||
|
class PortablePublishedResultLookup(Protocol):
|
||||||
|
def find(
|
||||||
|
self,
|
||||||
|
source_session_id: str,
|
||||||
|
definition: PortableRunDefinition,
|
||||||
|
) -> tuple[dict[str, object], ...]: ...
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class _SourceCompatibility:
|
class _SourceCompatibility:
|
||||||
compatible: bool
|
compatible: bool
|
||||||
@@ -116,6 +125,7 @@ class PortableSetupProjector:
|
|||||||
capability_probe: PortableDefinitionCapabilityProbe,
|
capability_probe: PortableDefinitionCapabilityProbe,
|
||||||
dispatch_available: bool = False,
|
dispatch_available: bool = False,
|
||||||
equipment_capture_registry: EquipmentCaptureRegistry | None = None,
|
equipment_capture_registry: EquipmentCaptureRegistry | None = None,
|
||||||
|
result_cache: PortablePublishedResultLookup | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
if not hasattr(capability_probe, "probe"):
|
if not hasattr(capability_probe, "probe"):
|
||||||
raise PortableSetupProjectionError("portable source capability probe is unavailable")
|
raise PortableSetupProjectionError("portable source capability probe is unavailable")
|
||||||
@@ -123,6 +133,7 @@ class PortableSetupProjector:
|
|||||||
self._capability_probe = capability_probe
|
self._capability_probe = capability_probe
|
||||||
self._dispatch_available = dispatch_available
|
self._dispatch_available = dispatch_available
|
||||||
self._equipment_capture_registry = equipment_capture_registry
|
self._equipment_capture_registry = equipment_capture_registry
|
||||||
|
self._result_cache = result_cache
|
||||||
for definition in registry.definitions:
|
for definition in registry.definitions:
|
||||||
_validate_model_presentation(definition)
|
_validate_model_presentation(definition)
|
||||||
if definition.authority.as_dict() != _OBSERVATION_ONLY_AUTHORITY:
|
if definition.authority.as_dict() != _OBSERVATION_ONLY_AUTHORITY:
|
||||||
@@ -166,8 +177,19 @@ class PortableSetupProjector:
|
|||||||
presentation = _presentation(definition)
|
presentation = _presentation(definition)
|
||||||
compatibility = self._probe_source(definition, source, presentation)
|
compatibility = self._probe_source(definition, source, presentation)
|
||||||
executor = definition.executor
|
executor = definition.executor
|
||||||
|
try:
|
||||||
|
existing = (
|
||||||
|
()
|
||||||
|
if self._result_cache is None
|
||||||
|
else self._result_cache.find(source.session_id, definition)
|
||||||
|
)
|
||||||
|
except (ObservatoryRecordedQueueError, OSError) as exc:
|
||||||
|
raise PortableSetupProjectionError("portable result cache is unavailable") from exc
|
||||||
submission_allowed = (
|
submission_allowed = (
|
||||||
compatibility.compatible and executor.ready and self._dispatch_available
|
not existing
|
||||||
|
and compatibility.compatible
|
||||||
|
and executor.ready
|
||||||
|
and self._dispatch_available
|
||||||
)
|
)
|
||||||
return {
|
return {
|
||||||
"setup_id": definition.setup_id,
|
"setup_id": definition.setup_id,
|
||||||
@@ -191,17 +213,23 @@ class PortableSetupProjector:
|
|||||||
"reason_code": executor.reason_code,
|
"reason_code": executor.reason_code,
|
||||||
"reason": executor.reason,
|
"reason": executor.reason,
|
||||||
},
|
},
|
||||||
"existing_results": [],
|
"existing_results": list(existing),
|
||||||
"preflight": {
|
"preflight": {
|
||||||
"outcome": "ready" if submission_allowed else "blocked",
|
"outcome": "existing" if existing else "ready" if submission_allowed else "blocked",
|
||||||
"action": "check" if submission_allowed else "blocked",
|
"action": "open-existing"
|
||||||
"reason": self._preflight_reason(
|
if existing
|
||||||
|
else "check"
|
||||||
|
if submission_allowed
|
||||||
|
else "blocked",
|
||||||
|
"reason": "Точный расчёт сохранён и проверен; доступен без запуска моделей."
|
||||||
|
if existing
|
||||||
|
else self._preflight_reason(
|
||||||
definition,
|
definition,
|
||||||
compatibility,
|
compatibility,
|
||||||
presentation,
|
presentation,
|
||||||
),
|
),
|
||||||
"submission_allowed": submission_allowed,
|
"submission_allowed": submission_allowed,
|
||||||
"existing_result_ids": [],
|
"existing_result_ids": [result["result_id"] for result in existing],
|
||||||
},
|
},
|
||||||
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
||||||
}
|
}
|
||||||
@@ -214,11 +242,9 @@ class PortableSetupProjector:
|
|||||||
) -> _SourceCompatibility:
|
) -> _SourceCompatibility:
|
||||||
source_session_id = source.session_id
|
source_session_id = source.session_id
|
||||||
if self._equipment_capture_registry is not None:
|
if self._equipment_capture_registry is not None:
|
||||||
expected_profile = (
|
expected_profile = self._equipment_capture_registry.compatible_profile_for_requirements(
|
||||||
self._equipment_capture_registry.compatible_profile_for_requirements(
|
|
||||||
definition.source_requirements.as_dict()
|
definition.source_requirements.as_dict()
|
||||||
)
|
)
|
||||||
)
|
|
||||||
if expected_profile is None:
|
if expected_profile is None:
|
||||||
raise PortableSetupProjectionError(
|
raise PortableSetupProjectionError(
|
||||||
"portable source requirements have no capture profile allowlist"
|
"portable source requirements have no capture profile allowlist"
|
||||||
@@ -232,8 +258,7 @@ class PortableSetupProjector:
|
|||||||
reason="У записи нет проверенной привязки к оборудованию и профилю записи.",
|
reason="У записи нет проверенной привязки к оборудованию и профилю записи.",
|
||||||
)
|
)
|
||||||
if (
|
if (
|
||||||
attestation.equipment_model_id
|
attestation.equipment_model_id != expected_profile.equipment.equipment_model_id
|
||||||
!= expected_profile.equipment.equipment_model_id
|
|
||||||
or attestation.equipment_model_sha256
|
or attestation.equipment_model_sha256
|
||||||
!= expected_profile.equipment.equipment_model_sha256
|
!= expected_profile.equipment.equipment_model_sha256
|
||||||
):
|
):
|
||||||
@@ -245,8 +270,7 @@ class PortableSetupProjector:
|
|||||||
)
|
)
|
||||||
if (
|
if (
|
||||||
attestation.capture_profile_id != expected_profile.capture_profile_id
|
attestation.capture_profile_id != expected_profile.capture_profile_id
|
||||||
or attestation.capture_profile_sha256
|
or attestation.capture_profile_sha256 != expected_profile.capture_profile_sha256
|
||||||
!= expected_profile.capture_profile_sha256
|
|
||||||
):
|
):
|
||||||
return _SourceCompatibility(
|
return _SourceCompatibility(
|
||||||
compatible=False,
|
compatible=False,
|
||||||
|
|||||||
@@ -1202,6 +1202,7 @@ class ObservatoryRecordedJobQueue:
|
|||||||
*,
|
*,
|
||||||
enqueue: bool = False,
|
enqueue: bool = False,
|
||||||
reject_duplicate_computation: bool = False,
|
reject_duplicate_computation: bool = False,
|
||||||
|
published_result_available: Callable[[ObservatoryRecordedJob], bool] | None = None,
|
||||||
) -> tuple[ObservatoryRecordedJob, bool]:
|
) -> tuple[ObservatoryRecordedJob, bool]:
|
||||||
"""Accept one immutable identity and optionally queue it atomically."""
|
"""Accept one immutable identity and optionally queue it atomically."""
|
||||||
|
|
||||||
@@ -1242,6 +1243,20 @@ class ObservatoryRecordedJobQueue:
|
|||||||
).fetchone()
|
).fetchone()
|
||||||
if duplicate is not None:
|
if duplicate is not None:
|
||||||
raise ObservatoryRecordedQueueDuplicateError(duplicate["job_id"])
|
raise ObservatoryRecordedQueueDuplicateError(duplicate["job_id"])
|
||||||
|
if published_result_available is not None:
|
||||||
|
# Recheck under the INSERT lock: publication may have completed
|
||||||
|
# after catalog/preflight. The verifier must not re-enter this DB.
|
||||||
|
published = connection.execute(
|
||||||
|
"SELECT * FROM observatory_recorded_jobs "
|
||||||
|
"WHERE identity_sha256 = ? AND state = 'succeeded' "
|
||||||
|
"AND publication_state = 'published' "
|
||||||
|
"ORDER BY created_at_utc DESC, job_id DESC",
|
||||||
|
(identity_sha256,),
|
||||||
|
).fetchall()
|
||||||
|
for row in published:
|
||||||
|
candidate = _job_from_row(row)
|
||||||
|
if published_result_available(candidate):
|
||||||
|
raise ObservatoryRecordedQueueDuplicateError(candidate.job_id)
|
||||||
self._require_capacity(
|
self._require_capacity(
|
||||||
connection,
|
connection,
|
||||||
table="observatory_recorded_jobs",
|
table="observatory_recorded_jobs",
|
||||||
@@ -2091,6 +2106,27 @@ class ObservatoryRecordedJobQueue:
|
|||||||
).fetchall()
|
).fetchall()
|
||||||
return tuple(_job_from_row(row) for row in rows)
|
return tuple(_job_from_row(row) for row in rows)
|
||||||
|
|
||||||
|
def published_results(
|
||||||
|
self, *, source_session_id: str, source_catalog_sha256: str,
|
||||||
|
setup_id: str, definition_sha256: str,
|
||||||
|
) -> tuple[ObservatoryRecordedJob, ...]:
|
||||||
|
"""Exact cache candidates, never inferred from labels or a truncated job page."""
|
||||||
|
|
||||||
|
_validate_pattern(source_session_id, _SESSION_ID, "source session id")
|
||||||
|
_validate_pattern(setup_id, _IDENTIFIER, "setup id")
|
||||||
|
_validate_pattern(source_catalog_sha256, _SHA256, "source catalog sha256")
|
||||||
|
_validate_pattern(definition_sha256, _SHA256, "definition sha256")
|
||||||
|
with self._read_connection() as connection:
|
||||||
|
rows = connection.execute(
|
||||||
|
"SELECT * FROM observatory_recorded_jobs "
|
||||||
|
"WHERE source_session_id = ? AND source_catalog_sha256 = ? "
|
||||||
|
"AND setup_id = ? AND definition_sha256 = ? "
|
||||||
|
"AND state = 'succeeded' AND publication_state = 'published' "
|
||||||
|
"ORDER BY created_at_utc DESC, job_id DESC",
|
||||||
|
(source_session_id, source_catalog_sha256, setup_id, definition_sha256),
|
||||||
|
).fetchall()
|
||||||
|
return tuple(_job_from_row(row) for row in rows)
|
||||||
|
|
||||||
def request_live(self, intent: ObservatoryLiveLeaseIntent) -> tuple[ObservatoryLiveLease, bool]:
|
def request_live(self, intent: ObservatoryLiveLeaseIntent) -> tuple[ObservatoryLiveLease, bool]:
|
||||||
"""Close recorded admission without allowing a monolith to delay live K1."""
|
"""Close recorded admission without allowing a monolith to delay live K1."""
|
||||||
|
|
||||||
@@ -2746,6 +2782,18 @@ class ObservatoryRecordedJobQueue:
|
|||||||
connection.executescript(_SCHEMA_SQL)
|
connection.executescript(_SCHEMA_SQL)
|
||||||
self._migrate_claim_lease_schema(connection)
|
self._migrate_claim_lease_schema(connection)
|
||||||
self._migrate_publication_schema(connection)
|
self._migrate_publication_schema(connection)
|
||||||
|
# A legacy database may not have publication columns until
|
||||||
|
# the migration above. Never create the partial index earlier.
|
||||||
|
connection.execute(
|
||||||
|
"CREATE INDEX IF NOT EXISTS observatory_recorded_jobs_published_source "
|
||||||
|
"ON observatory_recorded_jobs "
|
||||||
|
"(source_session_id, source_catalog_sha256, setup_id, definition_sha256) "
|
||||||
|
"WHERE state = 'succeeded' AND publication_state = 'published'"
|
||||||
|
)
|
||||||
|
connection.execute(
|
||||||
|
"CREATE INDEX IF NOT EXISTS observatory_recorded_jobs_computation "
|
||||||
|
"ON observatory_recorded_jobs (identity_sha256)"
|
||||||
|
)
|
||||||
self._validate_schema(connection)
|
self._validate_schema(connection)
|
||||||
self._validate_existing_capacity(connection)
|
self._validate_existing_capacity(connection)
|
||||||
connection.commit()
|
connection.commit()
|
||||||
|
|||||||
@@ -54,6 +54,7 @@ from k1link.observatory.portable_queue_binding import (
|
|||||||
PortableQueueBindingError,
|
PortableQueueBindingError,
|
||||||
PortableRecordedQueueBindingService,
|
PortableRecordedQueueBindingService,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.portable_result_cache import PortableResultCache
|
||||||
from k1link.observatory.portable_result_contract import (
|
from k1link.observatory.portable_result_contract import (
|
||||||
PortableCalculationProfileRegistry,
|
PortableCalculationProfileRegistry,
|
||||||
PortableResultContractValidatorRegistry,
|
PortableResultContractValidatorRegistry,
|
||||||
@@ -538,24 +539,41 @@ else:
|
|||||||
OBSERVATORY_PORTABLE_BINDING_SERVICE: PortableRecordedQueueBindingService | None
|
OBSERVATORY_PORTABLE_BINDING_SERVICE: PortableRecordedQueueBindingService | None
|
||||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableSetupProjector | None
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableSetupProjector | None
|
||||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
|
||||||
|
OBSERVATORY_PORTABLE_RESULT_CACHE: PortableResultCache | None = None
|
||||||
try:
|
try:
|
||||||
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None:
|
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None:
|
||||||
raise PortableSetupProjectionError(
|
raise PortableSetupProjectionError(
|
||||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR
|
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR
|
||||||
or "portable definition registry is unavailable"
|
or "portable definition registry is unavailable"
|
||||||
)
|
)
|
||||||
|
if (
|
||||||
|
session_artifact_gateway is not None
|
||||||
|
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||||
|
and OBSERVATORY_PORTABLE_CALCULATION_PROFILES is not None
|
||||||
|
):
|
||||||
|
OBSERVATORY_PORTABLE_RESULT_CACHE = PortableResultCache(
|
||||||
|
sessions=session_store, artifacts=session_artifact_gateway.store,
|
||||||
|
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||||
|
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||||
|
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
|
||||||
|
)
|
||||||
OBSERVATORY_PORTABLE_BINDING_SERVICE = PortableRecordedQueueBindingService(
|
OBSERVATORY_PORTABLE_BINDING_SERVICE = PortableRecordedQueueBindingService(
|
||||||
data_dir=session_store.data_dir,
|
data_dir=session_store.data_dir,
|
||||||
session_store=session_store,
|
session_store=session_store,
|
||||||
media_inspector=session_recorded_media_inspector,
|
media_inspector=session_recorded_media_inspector,
|
||||||
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||||
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||||
|
published_result_available=(
|
||||||
|
None if OBSERVATORY_PORTABLE_RESULT_CACHE is None
|
||||||
|
else OBSERVATORY_PORTABLE_RESULT_CACHE.available
|
||||||
|
),
|
||||||
)
|
)
|
||||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableSetupProjector(
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableSetupProjector(
|
||||||
registry=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
registry=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||||
capability_probe=OBSERVATORY_PORTABLE_BINDING_SERVICE,
|
capability_probe=OBSERVATORY_PORTABLE_BINDING_SERVICE,
|
||||||
dispatch_available=OBSERVATORY_WORKER_DISPATCH_READY,
|
dispatch_available=OBSERVATORY_WORKER_DISPATCH_READY,
|
||||||
equipment_capture_registry=session_store.equipment_capture_registry,
|
equipment_capture_registry=session_store.equipment_capture_registry,
|
||||||
|
result_cache=OBSERVATORY_PORTABLE_RESULT_CACHE,
|
||||||
)
|
)
|
||||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None
|
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None
|
||||||
except (
|
except (
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ from k1link.observatory.portable_queue_binding import (
|
|||||||
PortableQueueBindingStaleCheckError,
|
PortableQueueBindingStaleCheckError,
|
||||||
PortableRecordedQueueBindingService,
|
PortableRecordedQueueBindingService,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.portable_result_cache import PortableResultCacheCheckRequired
|
||||||
from k1link.observatory.portable_result_contract import PortableResultPublisherError
|
from k1link.observatory.portable_result_contract import PortableResultPublisherError
|
||||||
from k1link.observatory.portable_result_publisher import (
|
from k1link.observatory.portable_result_publisher import (
|
||||||
PortableObservatoryResultPublisher,
|
PortableObservatoryResultPublisher,
|
||||||
@@ -298,6 +299,29 @@ def build_observatory_router(
|
|||||||
)
|
)
|
||||||
compatible = compatibility.get("compatible") is True
|
compatible = compatibility.get("compatible") is True
|
||||||
executor_ready = executor.get("state") == "ready" and executor.get("ready") is True
|
executor_ready = executor.get("state") == "ready" and executor.get("ready") is True
|
||||||
|
projected_preflight = projected.get("preflight")
|
||||||
|
if (
|
||||||
|
isinstance(projected_preflight, dict)
|
||||||
|
and projected_preflight.get("outcome") == "existing"
|
||||||
|
):
|
||||||
|
return {
|
||||||
|
"schema_version": OBSERVATORY_RUN_PREFLIGHT_SCHEMA,
|
||||||
|
"source_session_id": request.source_session_id,
|
||||||
|
"setup_id": request.setup_id,
|
||||||
|
"definition_sha256": expected_digest,
|
||||||
|
"check_sha256": None,
|
||||||
|
"outcome": "existing",
|
||||||
|
"submission_allowed": False,
|
||||||
|
"checks": [{
|
||||||
|
"check_id": "published-result",
|
||||||
|
"outcome": "pass",
|
||||||
|
"reason_code": "exact-published-result-verified",
|
||||||
|
"message": "Точный расчёт сохранён; повторный inference не требуется.",
|
||||||
|
}],
|
||||||
|
"existing_result_ids": projected_preflight["existing_result_ids"],
|
||||||
|
"executor": executor,
|
||||||
|
"authority": projected.get("authority", dict(_OBSERVATION_ONLY_AUTHORITY)),
|
||||||
|
}
|
||||||
checked = None
|
checked = None
|
||||||
check_reason: str | None = None
|
check_reason: str | None = None
|
||||||
if (
|
if (
|
||||||
@@ -895,6 +919,11 @@ def build_observatory_router(
|
|||||||
status_code=409,
|
status_code=409,
|
||||||
detail="Идентичность RunDefinition изменилась; повторите preflight.",
|
detail="Идентичность RunDefinition изменилась; повторите preflight.",
|
||||||
)
|
)
|
||||||
|
if portable_projection.get("existing_results"):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Точный расчёт уже сохранён. Обновите каталог и откройте результат.",
|
||||||
|
)
|
||||||
if (
|
if (
|
||||||
projected_executor.get("state") != "ready"
|
projected_executor.get("state") != "ready"
|
||||||
or projected_executor.get("ready") is not True
|
or projected_executor.get("ready") is not True
|
||||||
@@ -930,11 +959,16 @@ def build_observatory_router(
|
|||||||
status_code=409,
|
status_code=409,
|
||||||
detail="Portable-привязка источника не прошла проверку целостности.",
|
detail="Portable-привязка источника не прошла проверку целостности.",
|
||||||
) from exc
|
) from exc
|
||||||
|
except PortableResultCacheCheckRequired as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Опубликованный результат требует проверки. Обновите каталог.",
|
||||||
|
) from exc
|
||||||
except ObservatoryRecordedQueueDuplicateError as exc:
|
except ObservatoryRecordedQueueDuplicateError as exc:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=409,
|
status_code=409,
|
||||||
detail=(
|
detail=(
|
||||||
"Такой расчёт уже выполняется или ожидает публикации. "
|
"Такой расчёт уже выполняется, ожидает публикации или сохранён. "
|
||||||
"Обновите список расчётов; повторный запуск не создан."
|
"Обновите список расчётов; повторный запуск не создан."
|
||||||
),
|
),
|
||||||
) from exc
|
) from exc
|
||||||
|
|||||||
@@ -3,14 +3,22 @@ from __future__ import annotations
|
|||||||
import copy
|
import copy
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from dataclasses import replace
|
from dataclasses import replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import cast
|
from typing import cast
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
from k1link.artifact_gateway import CentralArtifactStore
|
from k1link.artifact_gateway import CentralArtifactStore
|
||||||
|
from k1link.observatory.portable_result_cache import (
|
||||||
|
PortableResultCache,
|
||||||
|
PortableResultCacheCheckRequired,
|
||||||
|
)
|
||||||
from k1link.observatory.portable_result_contract import (
|
from k1link.observatory.portable_result_contract import (
|
||||||
OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||||
PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
|
PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
|
||||||
@@ -35,22 +43,27 @@ from k1link.observatory.portable_run_definitions import (
|
|||||||
PortableRunDefinitionRegistry,
|
PortableRunDefinitionRegistry,
|
||||||
canonical_sha256,
|
canonical_sha256,
|
||||||
)
|
)
|
||||||
|
from k1link.observatory.portable_setup_projection import PortableSetupProjector
|
||||||
from k1link.observatory.recorded_jobs import (
|
from k1link.observatory.recorded_jobs import (
|
||||||
ObservatoryRecordedJob,
|
ObservatoryRecordedJob,
|
||||||
ObservatoryRecordedJobIntent,
|
ObservatoryRecordedJobIntent,
|
||||||
ObservatoryRecordedJobQueue,
|
ObservatoryRecordedJobQueue,
|
||||||
|
ObservatoryRecordedQueueDuplicateError,
|
||||||
|
ObservatoryRecordedQueueIntegrityError,
|
||||||
RecordedRunDefinitionRegistry,
|
RecordedRunDefinitionRegistry,
|
||||||
)
|
)
|
||||||
from k1link.observatory.source_admission import (
|
from k1link.observatory.source_admission import (
|
||||||
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||||
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||||
PORTABLE_SOURCE_DOCUMENT_DIRECTORY,
|
PORTABLE_SOURCE_DOCUMENT_DIRECTORY,
|
||||||
|
PortableSourceAdmissionIntegrityError,
|
||||||
)
|
)
|
||||||
from k1link.sessions import (
|
from k1link.sessions import (
|
||||||
ObservationArchiveSource,
|
ObservationArchiveSource,
|
||||||
ObservationSessionCandidate,
|
ObservationSessionCandidate,
|
||||||
SessionStore,
|
SessionStore,
|
||||||
)
|
)
|
||||||
|
from k1link.web.observatory_api import build_observatory_router
|
||||||
|
|
||||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||||
@@ -232,6 +245,7 @@ def _package(
|
|||||||
definition: PortableRunDefinition,
|
definition: PortableRunDefinition,
|
||||||
result_id: str = RESULT_ID,
|
result_id: str = RESULT_ID,
|
||||||
accepted: bool = True,
|
accepted: bool = True,
|
||||||
|
extra_payload: bytes | None = None,
|
||||||
) -> tuple[Path, PortableResultPackageManifest]:
|
) -> tuple[Path, PortableResultPackageManifest]:
|
||||||
result_document = {
|
result_document = {
|
||||||
"schema_version": definition.result_contract.result_schema,
|
"schema_version": definition.result_contract.result_schema,
|
||||||
@@ -248,17 +262,32 @@ def _package(
|
|||||||
byte_length=len(result_bytes),
|
byte_length=len(result_bytes),
|
||||||
sha256=hashlib.sha256(result_bytes).hexdigest(),
|
sha256=hashlib.sha256(result_bytes).hexdigest(),
|
||||||
)
|
)
|
||||||
|
extra = (
|
||||||
|
()
|
||||||
|
if extra_payload is None
|
||||||
|
else (
|
||||||
|
PortableResultArtifact(
|
||||||
|
role="spatial-evidence",
|
||||||
|
relative_path="artifacts/spatial.bin",
|
||||||
|
media_type="application/octet-stream",
|
||||||
|
byte_length=len(extra_payload),
|
||||||
|
sha256=hashlib.sha256(extra_payload).hexdigest(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
package = PortableResultPackageManifest.create(
|
package = PortableResultPackageManifest.create(
|
||||||
job=job,
|
job=job,
|
||||||
definition=definition,
|
definition=definition,
|
||||||
result_id=result_id,
|
result_id=result_id,
|
||||||
created_at_utc=NOW,
|
created_at_utc=NOW,
|
||||||
artifacts=(artifact,),
|
artifacts=(artifact, *extra),
|
||||||
)
|
)
|
||||||
root = tmp_path / "packages" / package.manifest_sha256
|
root = tmp_path / "packages" / package.manifest_sha256
|
||||||
(root / "artifacts").mkdir(parents=True)
|
(root / "artifacts").mkdir(parents=True)
|
||||||
(root / "manifest.json").write_bytes(package.canonical_bytes)
|
(root / "manifest.json").write_bytes(package.canonical_bytes)
|
||||||
(root / "artifacts" / "result.json").write_bytes(result_bytes)
|
(root / "artifacts" / "result.json").write_bytes(result_bytes)
|
||||||
|
if extra_payload is not None:
|
||||||
|
(root / "artifacts" / "spatial.bin").write_bytes(extra_payload)
|
||||||
return root, package
|
return root, package
|
||||||
|
|
||||||
|
|
||||||
@@ -320,6 +349,8 @@ def _fixture(
|
|||||||
*,
|
*,
|
||||||
result_id: str = RESULT_ID,
|
result_id: str = RESULT_ID,
|
||||||
accepted: bool = True,
|
accepted: bool = True,
|
||||||
|
extra_payload: bytes | None = None,
|
||||||
|
for_publication: bool = False,
|
||||||
) -> tuple[
|
) -> tuple[
|
||||||
PortableRunDefinitionRegistry,
|
PortableRunDefinitionRegistry,
|
||||||
PortableRunDefinition,
|
PortableRunDefinition,
|
||||||
@@ -343,8 +374,10 @@ def _fixture(
|
|||||||
definition=definition,
|
definition=definition,
|
||||||
result_id=result_id,
|
result_id=result_id,
|
||||||
accepted=accepted,
|
accepted=accepted,
|
||||||
|
extra_payload=extra_payload,
|
||||||
)
|
)
|
||||||
succeeded = queue.succeed(
|
complete = queue.complete_for_publication if for_publication else queue.succeed
|
||||||
|
succeeded = complete(
|
||||||
running.job_id,
|
running.job_id,
|
||||||
claim_token=claim_token,
|
claim_token=claim_token,
|
||||||
result_id=result_id,
|
result_id=result_id,
|
||||||
@@ -410,15 +443,16 @@ def test_verified_package_publishes_immutable_binding_and_profile_provenance(
|
|||||||
assert view["viewer_capability"] == first.binding.replay_capability.as_dict()
|
assert view["viewer_capability"] == first.binding.replay_capability.as_dict()
|
||||||
|
|
||||||
summary = store.get_session(RESULT_ID).summary
|
summary = store.get_session(RESULT_ID).summary
|
||||||
assert summary.display_name == (
|
assert summary.display_name == ("Portable result source · полный маршрут и воспроизведение")
|
||||||
"Portable result source · полный маршрут и воспроизведение"
|
|
||||||
)
|
|
||||||
profiles = PortableCalculationProfileRegistry((_profile(definition),))
|
profiles = PortableCalculationProfileRegistry((_profile(definition),))
|
||||||
assert resolve_published_portable_calculation_profile(
|
assert (
|
||||||
|
resolve_published_portable_calculation_profile(
|
||||||
summary,
|
summary,
|
||||||
definitions=registry,
|
definitions=registry,
|
||||||
calculation_profiles=profiles,
|
calculation_profiles=profiles,
|
||||||
) == _profile(definition).as_dict()
|
)
|
||||||
|
== _profile(definition).as_dict()
|
||||||
|
)
|
||||||
|
|
||||||
assert summary.lab is not None
|
assert summary.lab is not None
|
||||||
drifted_provenance = copy.deepcopy(summary.lab.provenance)
|
drifted_provenance = copy.deepcopy(summary.lab.provenance)
|
||||||
@@ -586,9 +620,7 @@ def test_package_manifest_rejects_noncanonical_or_authority_elevating_documents(
|
|||||||
PortableResultPackageIntegrityError,
|
PortableResultPackageIntegrityError,
|
||||||
match="not canonical JSON",
|
match="not canonical JSON",
|
||||||
):
|
):
|
||||||
PortableResultPackageManifest.from_bytes(
|
PortableResultPackageManifest.from_bytes(json.dumps(manifest.as_dict(), indent=2).encode())
|
||||||
json.dumps(manifest.as_dict(), indent=2).encode()
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_legacy_canonical_result_namespace_cannot_be_republished(
|
def test_legacy_canonical_result_namespace_cannot_be_republished(
|
||||||
@@ -614,3 +646,258 @@ def test_legacy_canonical_result_namespace_cannot_be_republished(
|
|||||||
publisher.publish(job=job, package_root=package_root)
|
publisher.publish(job=job, package_root=package_root)
|
||||||
|
|
||||||
assert store.get_lab_instance(legacy_result_id) is None
|
assert store.get_lab_instance(legacy_result_id) is None
|
||||||
|
|
||||||
|
|
||||||
|
def _cache_fixture(tmp_path: Path, *, published: bool = True):
|
||||||
|
registry, definition, sessions, job, package_root = _fixture(
|
||||||
|
tmp_path,
|
||||||
|
extra_payload=b"synthetic spatial evidence",
|
||||||
|
for_publication=True,
|
||||||
|
)
|
||||||
|
queue = ObservatoryRecordedJobQueue(
|
||||||
|
sessions.data_dir,
|
||||||
|
definitions=RecordedRunDefinitionRegistry(registry.ready_recorded_definitions()),
|
||||||
|
clock=lambda: NOW,
|
||||||
|
)
|
||||||
|
artifacts = CentralArtifactStore(tmp_path / "central-artifacts", create=True)
|
||||||
|
publisher = _publisher(
|
||||||
|
tmp_path,
|
||||||
|
store=sessions,
|
||||||
|
registry=registry,
|
||||||
|
profile=_profile(definition),
|
||||||
|
validator=_validator,
|
||||||
|
)
|
||||||
|
if published:
|
||||||
|
publisher.publish(job=job, package_root=package_root)
|
||||||
|
job = queue.mark_published(job.job_id)
|
||||||
|
cache = PortableResultCache(
|
||||||
|
sessions=sessions,
|
||||||
|
artifacts=artifacts,
|
||||||
|
queue=queue,
|
||||||
|
definitions=registry,
|
||||||
|
calculation_profiles=PortableCalculationProfileRegistry((_profile(definition),)),
|
||||||
|
)
|
||||||
|
return cache, queue, sessions, artifacts, registry, definition, job, publisher, package_root
|
||||||
|
|
||||||
|
|
||||||
|
def _retry_intent(job: ObservatoryRecordedJob, key: str = "another-operator-click"):
|
||||||
|
return ObservatoryRecordedJobIntent(
|
||||||
|
idempotency_key=key,
|
||||||
|
source_session_id=job.source_session_id,
|
||||||
|
source_catalog_sha256=job.source_catalog_sha256,
|
||||||
|
source_bundle_sha256=job.source_bundle_sha256,
|
||||||
|
source_capability_manifest_sha256=job.source_capability_manifest_sha256,
|
||||||
|
setup_id=job.setup_id,
|
||||||
|
definition_sha256=job.definition_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_exact_publication_roundtrip_and_atomic_duplicate_guard(tmp_path: Path) -> None:
|
||||||
|
cache, queue, sessions, artifacts, _, definition, job, _, _ = _cache_fixture(tmp_path)
|
||||||
|
results = cache.find(SOURCE_SESSION_ID, definition)
|
||||||
|
assert [result["result_id"] for result in results] == [RESULT_ID]
|
||||||
|
assert results[0]["identity"]["source_bundle_sha256"] == job.source_bundle_sha256
|
||||||
|
assert results[0]["identity"]["definition_sha256"] == definition.definition_sha256
|
||||||
|
view = PortableResultViewService(sessions=sessions, artifacts=artifacts).read(RESULT_ID)
|
||||||
|
assert view["result_id"] == RESULT_ID
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueDuplicateError) as duplicate:
|
||||||
|
queue.submit(_retry_intent(job), published_result_available=cache.available)
|
||||||
|
assert duplicate.value.job_id == job.job_id
|
||||||
|
# Same request remains idempotent even when the cache guard would reject a new click.
|
||||||
|
retried, created = queue.submit(
|
||||||
|
_retry_intent(job, job.idempotency_key),
|
||||||
|
published_result_available=cache.available,
|
||||||
|
)
|
||||||
|
assert retried.job_id == job.job_id and not created
|
||||||
|
assert len(queue.list_jobs()) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_does_not_hide_another_source_snapshot_or_profile_version(tmp_path: Path) -> None:
|
||||||
|
cache, queue, sessions, _, _, definition, job, _, _ = _cache_fixture(tmp_path)
|
||||||
|
assert not cache.find("another-source", definition)
|
||||||
|
identity = definition.identity_document()
|
||||||
|
identity["version"] = definition.version + 1
|
||||||
|
newer = replace(
|
||||||
|
definition,
|
||||||
|
version=definition.version + 1,
|
||||||
|
definition_sha256=canonical_sha256(identity),
|
||||||
|
)
|
||||||
|
assert not cache.find(SOURCE_SESSION_ID, newer)
|
||||||
|
assert not queue.published_results(
|
||||||
|
source_session_id=SOURCE_SESSION_ID,
|
||||||
|
source_catalog_sha256=job.source_catalog_sha256,
|
||||||
|
setup_id=job.setup_id,
|
||||||
|
definition_sha256="f" * 64,
|
||||||
|
)
|
||||||
|
with sqlite3.connect(sessions.database_path) as connection:
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE observation_sessions SET total_bytes = total_bytes + 1 WHERE session_id = ?",
|
||||||
|
(SOURCE_SESSION_ID,),
|
||||||
|
)
|
||||||
|
assert not cache.find(SOURCE_SESSION_ID, definition)
|
||||||
|
assert sessions.get_lab_instance(RESULT_ID) is not None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("field", ["executor_image_sha256", "source_bundle_sha256"])
|
||||||
|
def test_queue_rejects_mutated_job_identity_before_cache_lookup(tmp_path: Path, field: str) -> None:
|
||||||
|
cache, _, _, _, _, definition, job, _, _ = _cache_fixture(tmp_path)
|
||||||
|
assert cache.find(SOURCE_SESSION_ID, definition)
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueIntegrityError):
|
||||||
|
replace(job, **{field: "f" * 64})
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("missing", [True, False])
|
||||||
|
def test_cache_requires_intact_central_manifest(tmp_path: Path, missing: bool) -> None:
|
||||||
|
cache, _, _, artifacts, _, definition, _, _, _ = _cache_fixture(tmp_path)
|
||||||
|
result = cache.find(SOURCE_SESSION_ID, definition)[0]
|
||||||
|
path = artifacts.manifest_path(result["identity"]["artifact_manifest_id"])
|
||||||
|
if missing:
|
||||||
|
path.unlink()
|
||||||
|
else:
|
||||||
|
path.write_text("{}")
|
||||||
|
assert not cache.find(SOURCE_SESSION_ID, definition)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("failure", ["missing", "same-size-corrupt", "symlink"])
|
||||||
|
@pytest.mark.parametrize("role", ["result-document", "spatial-evidence", "result-package-manifest"])
|
||||||
|
def test_cache_invalidates_any_changed_artifact_without_erasing_history(
|
||||||
|
tmp_path: Path,
|
||||||
|
failure: str,
|
||||||
|
role: str,
|
||||||
|
) -> None:
|
||||||
|
cache, queue, sessions, artifacts, _, definition, job, _, _ = _cache_fixture(tmp_path)
|
||||||
|
result = cache.find(SOURCE_SESSION_ID, definition)[0]
|
||||||
|
manifest = artifacts.read_manifest(result["identity"]["artifact_manifest_id"])
|
||||||
|
path = artifacts.object_path(manifest.member(role).sha256)
|
||||||
|
original = path.read_bytes()
|
||||||
|
metadata = path.stat()
|
||||||
|
if failure == "missing":
|
||||||
|
path.unlink()
|
||||||
|
elif failure == "symlink":
|
||||||
|
target = tmp_path / "not-an-immutable-object"
|
||||||
|
target.write_bytes(original)
|
||||||
|
path.unlink()
|
||||||
|
path.symlink_to(target)
|
||||||
|
else:
|
||||||
|
path.write_bytes(b"!" * len(original))
|
||||||
|
os.utime(path, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
|
||||||
|
assert not cache.find(SOURCE_SESSION_ID, definition)
|
||||||
|
assert sessions.get_lab_instance(RESULT_ID) is not None
|
||||||
|
_, created = queue.submit(_retry_intent(job), published_result_available=cache.available)
|
||||||
|
assert created
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("field", ["job", "source", "run_definition", "result_package"])
|
||||||
|
def test_cache_rejects_changed_publication_binding(tmp_path: Path, field: str) -> None:
|
||||||
|
cache, _, sessions, _, _, definition, _, _, _ = _cache_fixture(tmp_path)
|
||||||
|
assert cache.find(SOURCE_SESSION_ID, definition)
|
||||||
|
binding = sessions.get_lab_instance(RESULT_ID)
|
||||||
|
assert binding is not None
|
||||||
|
provenance = copy.deepcopy(binding.provenance)
|
||||||
|
provenance[field]["unexpected"] = "changed"
|
||||||
|
with sqlite3.connect(sessions.database_path) as connection:
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE observation_lab_instances SET provenance_json = ? WHERE session_id = ?",
|
||||||
|
(json.dumps(provenance), RESULT_ID),
|
||||||
|
)
|
||||||
|
assert not cache.find(SOURCE_SESSION_ID, definition)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_publication_race_requires_refresh_without_hashing_under_queue_lock(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
cache, queue, _, _, _, definition, job, publisher, root = _cache_fixture(
|
||||||
|
tmp_path,
|
||||||
|
published=False,
|
||||||
|
)
|
||||||
|
assert not cache.find(SOURCE_SESSION_ID, definition)
|
||||||
|
publisher.publish(job=job, package_root=root)
|
||||||
|
# Sealed and even session-published are insufficient without the durable receipt.
|
||||||
|
assert not cache.find(SOURCE_SESSION_ID, definition)
|
||||||
|
queue.mark_published(job.job_id)
|
||||||
|
with pytest.raises(PortableResultCacheCheckRequired):
|
||||||
|
queue.submit(_retry_intent(job), published_result_available=cache.available)
|
||||||
|
assert len(queue.list_jobs()) == 1
|
||||||
|
assert cache.find(SOURCE_SESSION_ID, definition)
|
||||||
|
with pytest.raises(ObservatoryRecordedQueueDuplicateError):
|
||||||
|
queue.submit(_retry_intent(job), published_result_available=cache.available)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_warm_refresh_does_not_reread_large_artifacts(tmp_path: Path, monkeypatch) -> None:
|
||||||
|
cache, _, _, artifacts, _, definition, _, _, _ = _cache_fixture(tmp_path)
|
||||||
|
result = cache.find(SOURCE_SESSION_ID, definition)[0]
|
||||||
|
manifest = artifacts.read_manifest(result["identity"]["artifact_manifest_id"])
|
||||||
|
spatial = artifacts.object_path(manifest.member("spatial-evidence").sha256)
|
||||||
|
original_open = Path.open
|
||||||
|
|
||||||
|
def guarded_open(path, *args, **kwargs):
|
||||||
|
assert path != spatial, "warm refresh must only stat an already verified large artifact"
|
||||||
|
return original_open(path, *args, **kwargs)
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "open", guarded_open)
|
||||||
|
assert cache.find(SOURCE_SESSION_ID, definition)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_api_existing_skips_source_preparation_submit_and_worker(tmp_path: Path) -> None:
|
||||||
|
cache, queue, sessions, artifacts, registry, definition, job, _, _ = _cache_fixture(tmp_path)
|
||||||
|
|
||||||
|
class NoCompute:
|
||||||
|
def probe(self, **_kwargs):
|
||||||
|
raise PortableSourceAdmissionIntegrityError("raw source temporarily offline")
|
||||||
|
|
||||||
|
def check(self, **_kwargs):
|
||||||
|
pytest.fail("cached preflight must not prepare the source")
|
||||||
|
|
||||||
|
def submit(self, **_kwargs):
|
||||||
|
pytest.fail("cached result must not create model work")
|
||||||
|
|
||||||
|
binding = NoCompute()
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(
|
||||||
|
build_observatory_router(
|
||||||
|
sessions,
|
||||||
|
portable_setup_projector=PortableSetupProjector(
|
||||||
|
registry=registry,
|
||||||
|
capability_probe=binding,
|
||||||
|
dispatch_available=False,
|
||||||
|
result_cache=cache,
|
||||||
|
),
|
||||||
|
portable_binding_service=binding,
|
||||||
|
recorded_job_queue=queue,
|
||||||
|
portable_result_view=PortableResultViewService(sessions=sessions, artifacts=artifacts),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
client = TestClient(app)
|
||||||
|
catalog = client.get(
|
||||||
|
"/api/v1/observatory/portable-laboratory-setups",
|
||||||
|
params={
|
||||||
|
"source_session_id": SOURCE_SESSION_ID,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert catalog.status_code == 200
|
||||||
|
setup = catalog.json()["setups"][0]
|
||||||
|
assert setup["source_compatibility"]["compatible"] is False
|
||||||
|
assert setup["preflight"]["outcome"] == "existing"
|
||||||
|
assert setup["preflight"]["submission_allowed"] is False
|
||||||
|
request = {
|
||||||
|
"schema_version": "missioncore.observatory-run-preflight-request/v1",
|
||||||
|
"source_session_id": SOURCE_SESSION_ID,
|
||||||
|
"setup_id": definition.setup_id,
|
||||||
|
"definition_sha256": definition.definition_sha256,
|
||||||
|
}
|
||||||
|
preflight = client.post("/api/v1/observatory/run-preflights", json=request)
|
||||||
|
assert preflight.status_code == 200
|
||||||
|
assert preflight.json()["outcome"] == "existing"
|
||||||
|
assert preflight.json()["check_sha256"] is None
|
||||||
|
assert preflight.json()["existing_result_ids"] == [RESULT_ID]
|
||||||
|
request.update(
|
||||||
|
{
|
||||||
|
"schema_version": "missioncore.observatory-recorded-run-submit/v1",
|
||||||
|
"idempotency_key": "new-click",
|
||||||
|
"check_sha256": "f" * 64,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
assert client.post("/api/v1/observatory/runs", json=request).status_code == 409
|
||||||
|
request["idempotency_key"] = job.idempotency_key
|
||||||
|
assert client.post("/api/v1/observatory/runs", json=request).status_code == 202
|
||||||
|
assert len(queue.list_jobs()) == 1
|
||||||
|
|||||||
@@ -1057,6 +1057,30 @@ def test_operator_reconciliation_is_durable_idempotent_and_unblocks_queue(
|
|||||||
assert replacement.job.job_id == next_job.job_id
|
assert replacement.job.job_id == next_job.job_id
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_indexes_follow_legacy_publication_column_migration(tmp_path: Path) -> None:
|
||||||
|
queue = _queue(tmp_path)
|
||||||
|
job, _ = queue.submit(_intent(), enqueue=True)
|
||||||
|
with sqlite3.connect(queue.database_path) as connection:
|
||||||
|
connection.execute("DROP INDEX observatory_recorded_jobs_published_source")
|
||||||
|
connection.execute("DROP INDEX observatory_recorded_jobs_computation")
|
||||||
|
for column in (
|
||||||
|
"publication_state", "publication_attempts", "publication_error", "published_at_utc",
|
||||||
|
):
|
||||||
|
connection.execute(f"ALTER TABLE observatory_recorded_jobs DROP COLUMN {column}")
|
||||||
|
migrated = _queue(tmp_path)
|
||||||
|
assert migrated.get(job.job_id) == job
|
||||||
|
assert not migrated.published_results(
|
||||||
|
source_session_id=job.source_session_id, source_catalog_sha256=job.source_catalog_sha256,
|
||||||
|
setup_id=job.setup_id, definition_sha256=job.definition_sha256,
|
||||||
|
)
|
||||||
|
with sqlite3.connect(queue.database_path) as connection:
|
||||||
|
names = {
|
||||||
|
row[1] for row in connection.execute("PRAGMA index_list(observatory_recorded_jobs)")
|
||||||
|
}
|
||||||
|
assert "observatory_recorded_jobs_published_source" in names
|
||||||
|
assert "observatory_recorded_jobs_computation" in names
|
||||||
|
|
||||||
|
|
||||||
def test_legacy_sqlite_claim_schema_migrates_without_reusing_old_token(
|
def test_legacy_sqlite_claim_schema_migrates_without_reusing_old_token(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
Reference in New Issue
Block a user