SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(observatory): verify exact published result reuse

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-09-03 09:14:54 +03:00
parent 80fc0058cb
commit 1e4ddc2cff
10 changed files with 910 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_observatory_portable_result_publisher.py
+300 -13
View File
@@ -3,14 +3,22 @@ from __future__ import annotations
import copy
import hashlib
import json
import os
import sqlite3
from collections.abc import Callable
from dataclasses import replace
from pathlib import Path
from typing import cast
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from k1link.artifact_gateway import CentralArtifactStore
from k1link.observatory.portable_result_cache import (
PortableResultCache,
PortableResultCacheCheckRequired,
)
from k1link.observatory.portable_result_contract import (
OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
@@ -35,22 +43,27 @@ from k1link.observatory.portable_run_definitions import (
PortableRunDefinitionRegistry,
canonical_sha256,
)
from k1link.observatory.portable_setup_projection import PortableSetupProjector
from k1link.observatory.recorded_jobs import (
ObservatoryRecordedJob,
ObservatoryRecordedJobIntent,
ObservatoryRecordedJobQueue,
ObservatoryRecordedQueueDuplicateError,
ObservatoryRecordedQueueIntegrityError,
RecordedRunDefinitionRegistry,
)
from k1link.observatory.source_admission import (
PORTABLE_SOURCE_BUNDLE_SCHEMA,
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
PORTABLE_SOURCE_DOCUMENT_DIRECTORY,
PortableSourceAdmissionIntegrityError,
)
from k1link.sessions import (
ObservationArchiveSource,
ObservationSessionCandidate,
SessionStore,
)
from k1link.web.observatory_api import build_observatory_router
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
@@ -232,6 +245,7 @@ def _package(
definition: PortableRunDefinition,
result_id: str = RESULT_ID,
accepted: bool = True,
extra_payload: bytes | None = None,
) -> tuple[Path, PortableResultPackageManifest]:
result_document = {
"schema_version": definition.result_contract.result_schema,
@@ -248,17 +262,32 @@ def _package(
byte_length=len(result_bytes),
sha256=hashlib.sha256(result_bytes).hexdigest(),
)
extra = (
()
if extra_payload is None
else (
PortableResultArtifact(
role="spatial-evidence",
relative_path="artifacts/spatial.bin",
media_type="application/octet-stream",
byte_length=len(extra_payload),
sha256=hashlib.sha256(extra_payload).hexdigest(),
),
)
)
package = PortableResultPackageManifest.create(
job=job,
definition=definition,
result_id=result_id,
created_at_utc=NOW,
artifacts=(artifact,),
artifacts=(artifact, *extra),
)
root = tmp_path / "packages" / package.manifest_sha256
(root / "artifacts").mkdir(parents=True)
(root / "manifest.json").write_bytes(package.canonical_bytes)
(root / "artifacts" / "result.json").write_bytes(result_bytes)
if extra_payload is not None:
(root / "artifacts" / "spatial.bin").write_bytes(extra_payload)
return root, package
@@ -320,6 +349,8 @@ def _fixture(
*,
result_id: str = RESULT_ID,
accepted: bool = True,
extra_payload: bytes | None = None,
for_publication: bool = False,
) -> tuple[
PortableRunDefinitionRegistry,
PortableRunDefinition,
@@ -343,8 +374,10 @@ def _fixture(
definition=definition,
result_id=result_id,
accepted=accepted,
extra_payload=extra_payload,
)
succeeded = queue.succeed(
complete = queue.complete_for_publication if for_publication else queue.succeed
succeeded = complete(
running.job_id,
claim_token=claim_token,
result_id=result_id,
@@ -410,15 +443,16 @@ def test_verified_package_publishes_immutable_binding_and_profile_provenance(
assert view["viewer_capability"] == first.binding.replay_capability.as_dict()
summary = store.get_session(RESULT_ID).summary
assert summary.display_name == (
"Portable result source · полный маршрут и воспроизведение"
)
assert summary.display_name == ("Portable result source · полный маршрут и воспроизведение")
profiles = PortableCalculationProfileRegistry((_profile(definition),))
assert resolve_published_portable_calculation_profile(
summary,
definitions=registry,
calculation_profiles=profiles,
) == _profile(definition).as_dict()
assert (
resolve_published_portable_calculation_profile(
summary,
definitions=registry,
calculation_profiles=profiles,
)
== _profile(definition).as_dict()
)
assert summary.lab is not None
drifted_provenance = copy.deepcopy(summary.lab.provenance)
@@ -586,9 +620,7 @@ def test_package_manifest_rejects_noncanonical_or_authority_elevating_documents(
PortableResultPackageIntegrityError,
match="not canonical JSON",
):
PortableResultPackageManifest.from_bytes(
json.dumps(manifest.as_dict(), indent=2).encode()
)
PortableResultPackageManifest.from_bytes(json.dumps(manifest.as_dict(), indent=2).encode())
def test_legacy_canonical_result_namespace_cannot_be_republished(
@@ -614,3 +646,258 @@ def test_legacy_canonical_result_namespace_cannot_be_republished(
publisher.publish(job=job, package_root=package_root)
assert store.get_lab_instance(legacy_result_id) is None
def _cache_fixture(tmp_path: Path, *, published: bool = True):
registry, definition, sessions, job, package_root = _fixture(
tmp_path,
extra_payload=b"synthetic spatial evidence",
for_publication=True,
)
queue = ObservatoryRecordedJobQueue(
sessions.data_dir,
definitions=RecordedRunDefinitionRegistry(registry.ready_recorded_definitions()),
clock=lambda: NOW,
)
artifacts = CentralArtifactStore(tmp_path / "central-artifacts", create=True)
publisher = _publisher(
tmp_path,
store=sessions,
registry=registry,
profile=_profile(definition),
validator=_validator,
)
if published:
publisher.publish(job=job, package_root=package_root)
job = queue.mark_published(job.job_id)
cache = PortableResultCache(
sessions=sessions,
artifacts=artifacts,
queue=queue,
definitions=registry,
calculation_profiles=PortableCalculationProfileRegistry((_profile(definition),)),
)
return cache, queue, sessions, artifacts, registry, definition, job, publisher, package_root
def _retry_intent(job: ObservatoryRecordedJob, key: str = "another-operator-click"):
return ObservatoryRecordedJobIntent(
idempotency_key=key,
source_session_id=job.source_session_id,
source_catalog_sha256=job.source_catalog_sha256,
source_bundle_sha256=job.source_bundle_sha256,
source_capability_manifest_sha256=job.source_capability_manifest_sha256,
setup_id=job.setup_id,
definition_sha256=job.definition_sha256,
)
def test_cache_exact_publication_roundtrip_and_atomic_duplicate_guard(tmp_path: Path) -> None:
cache, queue, sessions, artifacts, _, definition, job, _, _ = _cache_fixture(tmp_path)
results = cache.find(SOURCE_SESSION_ID, definition)
assert [result["result_id"] for result in results] == [RESULT_ID]
assert results[0]["identity"]["source_bundle_sha256"] == job.source_bundle_sha256
assert results[0]["identity"]["definition_sha256"] == definition.definition_sha256
view = PortableResultViewService(sessions=sessions, artifacts=artifacts).read(RESULT_ID)
assert view["result_id"] == RESULT_ID
with pytest.raises(ObservatoryRecordedQueueDuplicateError) as duplicate:
queue.submit(_retry_intent(job), published_result_available=cache.available)
assert duplicate.value.job_id == job.job_id
# Same request remains idempotent even when the cache guard would reject a new click.
retried, created = queue.submit(
_retry_intent(job, job.idempotency_key),
published_result_available=cache.available,
)
assert retried.job_id == job.job_id and not created
assert len(queue.list_jobs()) == 1
def test_cache_does_not_hide_another_source_snapshot_or_profile_version(tmp_path: Path) -> None:
cache, queue, sessions, _, _, definition, job, _, _ = _cache_fixture(tmp_path)
assert not cache.find("another-source", definition)
identity = definition.identity_document()
identity["version"] = definition.version + 1
newer = replace(
definition,
version=definition.version + 1,
definition_sha256=canonical_sha256(identity),
)
assert not cache.find(SOURCE_SESSION_ID, newer)
assert not queue.published_results(
source_session_id=SOURCE_SESSION_ID,
source_catalog_sha256=job.source_catalog_sha256,
setup_id=job.setup_id,
definition_sha256="f" * 64,
)
with sqlite3.connect(sessions.database_path) as connection:
connection.execute(
"UPDATE observation_sessions SET total_bytes = total_bytes + 1 WHERE session_id = ?",
(SOURCE_SESSION_ID,),
)
assert not cache.find(SOURCE_SESSION_ID, definition)
assert sessions.get_lab_instance(RESULT_ID) is not None
@pytest.mark.parametrize("field", ["executor_image_sha256", "source_bundle_sha256"])
def test_queue_rejects_mutated_job_identity_before_cache_lookup(tmp_path: Path, field: str) -> None:
cache, _, _, _, _, definition, job, _, _ = _cache_fixture(tmp_path)
assert cache.find(SOURCE_SESSION_ID, definition)
with pytest.raises(ObservatoryRecordedQueueIntegrityError):
replace(job, **{field: "f" * 64})
@pytest.mark.parametrize("missing", [True, False])
def test_cache_requires_intact_central_manifest(tmp_path: Path, missing: bool) -> None:
cache, _, _, artifacts, _, definition, _, _, _ = _cache_fixture(tmp_path)
result = cache.find(SOURCE_SESSION_ID, definition)[0]
path = artifacts.manifest_path(result["identity"]["artifact_manifest_id"])
if missing:
path.unlink()
else:
path.write_text("{}")
assert not cache.find(SOURCE_SESSION_ID, definition)
@pytest.mark.parametrize("failure", ["missing", "same-size-corrupt", "symlink"])
@pytest.mark.parametrize("role", ["result-document", "spatial-evidence", "result-package-manifest"])
def test_cache_invalidates_any_changed_artifact_without_erasing_history(
tmp_path: Path,
failure: str,
role: str,
) -> None:
cache, queue, sessions, artifacts, _, definition, job, _, _ = _cache_fixture(tmp_path)
result = cache.find(SOURCE_SESSION_ID, definition)[0]
manifest = artifacts.read_manifest(result["identity"]["artifact_manifest_id"])
path = artifacts.object_path(manifest.member(role).sha256)
original = path.read_bytes()
metadata = path.stat()
if failure == "missing":
path.unlink()
elif failure == "symlink":
target = tmp_path / "not-an-immutable-object"
target.write_bytes(original)
path.unlink()
path.symlink_to(target)
else:
path.write_bytes(b"!" * len(original))
os.utime(path, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
assert not cache.find(SOURCE_SESSION_ID, definition)
assert sessions.get_lab_instance(RESULT_ID) is not None
_, created = queue.submit(_retry_intent(job), published_result_available=cache.available)
assert created
@pytest.mark.parametrize("field", ["job", "source", "run_definition", "result_package"])
def test_cache_rejects_changed_publication_binding(tmp_path: Path, field: str) -> None:
cache, _, sessions, _, _, definition, _, _, _ = _cache_fixture(tmp_path)
assert cache.find(SOURCE_SESSION_ID, definition)
binding = sessions.get_lab_instance(RESULT_ID)
assert binding is not None
provenance = copy.deepcopy(binding.provenance)
provenance[field]["unexpected"] = "changed"
with sqlite3.connect(sessions.database_path) as connection:
connection.execute(
"UPDATE observation_lab_instances SET provenance_json = ? WHERE session_id = ?",
(json.dumps(provenance), RESULT_ID),
)
assert not cache.find(SOURCE_SESSION_ID, definition)
def test_cache_publication_race_requires_refresh_without_hashing_under_queue_lock(
tmp_path: Path,
) -> None:
cache, queue, _, _, _, definition, job, publisher, root = _cache_fixture(
tmp_path,
published=False,
)
assert not cache.find(SOURCE_SESSION_ID, definition)
publisher.publish(job=job, package_root=root)
# Sealed and even session-published are insufficient without the durable receipt.
assert not cache.find(SOURCE_SESSION_ID, definition)
queue.mark_published(job.job_id)
with pytest.raises(PortableResultCacheCheckRequired):
queue.submit(_retry_intent(job), published_result_available=cache.available)
assert len(queue.list_jobs()) == 1
assert cache.find(SOURCE_SESSION_ID, definition)
with pytest.raises(ObservatoryRecordedQueueDuplicateError):
queue.submit(_retry_intent(job), published_result_available=cache.available)
def test_cache_warm_refresh_does_not_reread_large_artifacts(tmp_path: Path, monkeypatch) -> None:
cache, _, _, artifacts, _, definition, _, _, _ = _cache_fixture(tmp_path)
result = cache.find(SOURCE_SESSION_ID, definition)[0]
manifest = artifacts.read_manifest(result["identity"]["artifact_manifest_id"])
spatial = artifacts.object_path(manifest.member("spatial-evidence").sha256)
original_open = Path.open
def guarded_open(path, *args, **kwargs):
assert path != spatial, "warm refresh must only stat an already verified large artifact"
return original_open(path, *args, **kwargs)
monkeypatch.setattr(Path, "open", guarded_open)
assert cache.find(SOURCE_SESSION_ID, definition)
def test_cache_api_existing_skips_source_preparation_submit_and_worker(tmp_path: Path) -> None:
cache, queue, sessions, artifacts, registry, definition, job, _, _ = _cache_fixture(tmp_path)
class NoCompute:
def probe(self, **_kwargs):
raise PortableSourceAdmissionIntegrityError("raw source temporarily offline")
def check(self, **_kwargs):
pytest.fail("cached preflight must not prepare the source")
def submit(self, **_kwargs):
pytest.fail("cached result must not create model work")
binding = NoCompute()
app = FastAPI()
app.include_router(
build_observatory_router(
sessions,
portable_setup_projector=PortableSetupProjector(
registry=registry,
capability_probe=binding,
dispatch_available=False,
result_cache=cache,
),
portable_binding_service=binding,
recorded_job_queue=queue,
portable_result_view=PortableResultViewService(sessions=sessions, artifacts=artifacts),
)
)
client = TestClient(app)
catalog = client.get(
"/api/v1/observatory/portable-laboratory-setups",
params={
"source_session_id": SOURCE_SESSION_ID,
},
)
assert catalog.status_code == 200
setup = catalog.json()["setups"][0]
assert setup["source_compatibility"]["compatible"] is False
assert setup["preflight"]["outcome"] == "existing"
assert setup["preflight"]["submission_allowed"] is False
request = {
"schema_version": "missioncore.observatory-run-preflight-request/v1",
"source_session_id": SOURCE_SESSION_ID,
"setup_id": definition.setup_id,
"definition_sha256": definition.definition_sha256,
}
preflight = client.post("/api/v1/observatory/run-preflights", json=request)
assert preflight.status_code == 200
assert preflight.json()["outcome"] == "existing"
assert preflight.json()["check_sha256"] is None
assert preflight.json()["existing_result_ids"] == [RESULT_ID]
request.update(
{
"schema_version": "missioncore.observatory-recorded-run-submit/v1",
"idempotency_key": "new-click",
"check_sha256": "f" * 64,
}
)
assert client.post("/api/v1/observatory/runs", json=request).status_code == 409
request["idempotency_key"] = job.idempotency_key
assert client.post("/api/v1/observatory/runs", json=request).status_code == 202
assert len(queue.list_jobs()) == 1
tests/test_observatory_recorded_jobs.py
+24
View File
@@ -1057,6 +1057,30 @@ def test_operator_reconciliation_is_durable_idempotent_and_unblocks_queue(
assert replacement.job.job_id == next_job.job_id
def test_cache_indexes_follow_legacy_publication_column_migration(tmp_path: Path) -> None:
queue = _queue(tmp_path)
job, _ = queue.submit(_intent(), enqueue=True)
with sqlite3.connect(queue.database_path) as connection:
connection.execute("DROP INDEX observatory_recorded_jobs_published_source")
connection.execute("DROP INDEX observatory_recorded_jobs_computation")
for column in (
"publication_state", "publication_attempts", "publication_error", "published_at_utc",
):
connection.execute(f"ALTER TABLE observatory_recorded_jobs DROP COLUMN {column}")
migrated = _queue(tmp_path)
assert migrated.get(job.job_id) == job
assert not migrated.published_results(
source_session_id=job.source_session_id, source_catalog_sha256=job.source_catalog_sha256,
setup_id=job.setup_id, definition_sha256=job.definition_sha256,
)
with sqlite3.connect(queue.database_path) as connection:
names = {
row[1] for row in connection.execute("PRAGMA index_list(observatory_recorded_jobs)")
}
assert "observatory_recorded_jobs_published_source" in names
assert "observatory_recorded_jobs_computation" in names
def test_legacy_sqlite_claim_schema_migrates_without_reusing_old_token(
tmp_path: Path,
) -> None: