SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(observatory): verify exact published result reuse

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-09-03 09:14:54 +03:00
parent 80fc0058cb
commit 1e4ddc2cff
10 changed files with 910 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/observatory/portable_queue_binding.py
+4
View File
@@ -11,6 +11,7 @@ from __future__ import annotations
import hashlib
import json
import re
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Final
@@ -109,6 +110,7 @@ class PortableRecordedQueueBindingService:
media_inspector: RecordedMediaInspector,
definitions: PortableRunDefinitionRegistry,
queue: ObservatoryRecordedJobQueue | None = None,
published_result_available: Callable[[ObservatoryRecordedJob], bool] | None = None,
) -> None:
self.data_dir = data_dir.expanduser().resolve()
if self.data_dir != session_store.data_dir:
@@ -123,6 +125,7 @@ class PortableRecordedQueueBindingService:
self._media_inspector = media_inspector
self._definitions = definitions
self._queue = queue
self._published_result_available = published_result_available
def probe(
self,
@@ -215,6 +218,7 @@ class PortableRecordedQueueBindingService:
preparation.intent(idempotency_key=idempotency_key),
enqueue=enqueue,
reject_duplicate_computation=True,
published_result_available=self._published_result_available,
)
def _resolve_definition(
src/k1link/observatory/portable_result_cache.py
+305
View File
@@ -0,0 +1,305 @@
"""Verified recorded-result reuse without inference or source replay preparation."""
from __future__ import annotations
import hashlib
import stat
from collections import OrderedDict
from dataclasses import fields
from pathlib import Path
from threading import RLock
from k1link.artifact_gateway import ArtifactGatewayError, CentralArtifactStore
from k1link.observatory.portable_result_contract import (
PORTABLE_RESULT_PACKAGE_SCHEMA,
RESULT_DOCUMENT_ROLE,
RESULT_PACKAGE_MANIFEST_ROLE,
PortableCalculationProfileRegistry,
PortableResultPackageManifest,
PortableResultPublisherError,
job_identity_document,
result_identity_document,
run_definition_document,
source_identity_document,
)
from k1link.observatory.portable_result_publisher import (
resolve_published_portable_calculation_profile,
)
from k1link.observatory.portable_run_definitions import (
PortableRunDefinition,
PortableRunDefinitionRegistry,
PortableRunDefinitionRegistryError,
)
from k1link.observatory.recorded_jobs import (
ObservatoryRecordedJob,
ObservatoryRecordedJobQueue,
ObservatoryRecordedQueueConflictError,
)
from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore
class PortableResultCacheCheckRequired(ObservatoryRecordedQueueConflictError):
"""Refresh verifies a newly published/changed object outside the queue write lock."""
class PortableResultCache:
"""Join the durable publication receipt, Session binding and immutable CAS.
Content checks are memoized by filesystem fingerprint, not unconditional availability.
Every lookup rechecks metadata. First use/changed metadata streams SHA-256
with bounded memory; no videos, RRDs or model tensors are kept in RAM.
"""
def __init__(
self,
*,
sessions: SessionStore,
artifacts: CentralArtifactStore,
queue: ObservatoryRecordedJobQueue,
definitions: PortableRunDefinitionRegistry,
calculation_profiles: PortableCalculationProfileRegistry,
) -> None:
if sessions.data_dir != queue.data_dir:
raise ValueError("portable result cache and queue roots disagree")
self._sessions = sessions
self._artifacts = artifacts
self._queue = queue
self._definitions = definitions
self._profiles = calculation_profiles
self._verified: OrderedDict[tuple[Path, str], tuple[tuple[int, ...], bool]] = OrderedDict()
self._lock = RLock()
def find(
self,
source_session_id: str,
definition: PortableRunDefinition,
) -> tuple[dict[str, object], ...]:
"""Return the newest usable exact result; older evidence remains untouched."""
try:
source, catalog_sha256 = self._sessions.get_session_with_catalog_snapshot(
source_session_id
)
except (SessionNotFoundError, SessionIntegrityError):
return ()
if source.summary.lab is not None:
return ()
candidates = self._queue.published_results(
source_session_id=source_session_id,
source_catalog_sha256=catalog_sha256,
setup_id=definition.setup_id,
definition_sha256=definition.definition_sha256,
)
for job in candidates:
result = self._projection(job)
if result is not None:
return (result,)
return ()
def available(self, job: ObservatoryRecordedJob) -> bool:
"""Queue INSERT fence: verify this row without re-entering the queue DB."""
if not self._lock.acquire(blocking=False):
raise PortableResultCacheCheckRequired("published result verification is in progress")
try:
return self._projection(job, warm_only=True) is not None
finally:
self._lock.release()
def _projection(
self,
job: ObservatoryRecordedJob,
*,
warm_only: bool = False,
) -> dict[str, object] | None:
if (
job.state != "succeeded"
or job.publication_state != "published"
or job.result_id is None
or job.result_sha256 is None
or job.terminal_code != "result-sealed"
or job.terminal_claim_token_sha256 is None
or job.claim_generation < 1
or job.active_claim_token is not None
or job.active_claimant_id is not None
):
return None
try:
definition = self._definitions.resolve(job.setup_id, job.definition_sha256)
recorded = definition.to_recorded_run_definition()
if any(
getattr(recorded, field.name) != getattr(job, field.name)
for field in fields(recorded)
):
return None
summary = self._sessions.get_session(job.result_id).summary
profile = self._profiles.resolve(definition)
if (
resolve_published_portable_calculation_profile(
summary,
definitions=self._definitions,
calculation_profiles=self._profiles,
)
!= profile.as_dict()
):
return None
binding = summary.lab
assert binding is not None
provenance = binding.provenance
published_package = provenance.get("result_package")
if (
provenance.get("job") != job_identity_document(job)
or provenance.get("source") != source_identity_document(job)
or not isinstance(published_package, dict)
):
return None
manifest_id = published_package.get("artifact_manifest_id")
if not isinstance(manifest_id, str):
return None
manifest = self._artifacts.read_manifest(manifest_id)
package_member = manifest.member(RESULT_PACKAGE_MANIFEST_ROLE)
if (
package_member.sha256 != job.result_sha256
or package_member.media_type != "application/json"
or not 0 < package_member.byte_length <= 1024 * 1024
):
return None
package_path = self._artifacts.object_path(package_member.sha256)
# This small contract is parsed on every lookup, never trusted by stat alone.
_fingerprint(package_path, package_member.byte_length)
with package_path.open("rb") as stream:
payload = stream.read(1024 * 1024 + 1)
package = PortableResultPackageManifest.from_bytes(payload)
if (
len(payload) != package_member.byte_length
or package.manifest_sha256 != job.result_sha256
or package.job != job_identity_document(job)
or package.source != source_identity_document(job)
or package.run_definition != run_definition_document(definition)
or package.result != result_identity_document(definition, job.result_id)
or manifest.artifact_type != "observatory-portable-result"
or manifest.subject_id != job.result_id
or dict(manifest.metadata)
!= {
"package-sha256": job.result_sha256,
"package-identity-sha256": package.identity_sha256,
"job-id": job.job_id,
"job-identity-sha256": job.identity_sha256,
"definition-sha256": job.definition_sha256,
"source-bundle-sha256": job.source_bundle_sha256,
"result-contract-sha256": definition.result_contract.contract_sha256,
"calculation-profile-sha256": profile.identity_sha256,
}
):
return None
result_document = next(
item for item in package.artifacts if item.role == RESULT_DOCUMENT_ROLE
)
if published_package != {
"schema_version": PORTABLE_RESULT_PACKAGE_SCHEMA,
"manifest_sha256": job.result_sha256,
"identity_sha256": package.identity_sha256,
"artifact_manifest_id": manifest_id,
"result_document_sha256": result_document.sha256,
"artifacts": [item.as_dict() for item in package.artifacts],
}:
return None
expected_members = {
item.role: (item.sha256, item.byte_length, item.media_type)
for item in package.artifacts
}
expected_members[RESULT_PACKAGE_MANIFEST_ROLE] = (
job.result_sha256,
len(payload),
"application/json",
)
if expected_members != {
item.role: (item.sha256, item.byte_length, item.media_type)
for item in manifest.members
}:
return None
for member in manifest.members:
self._verify_file(
self._artifacts.object_path(member.sha256),
member.sha256,
member.byte_length,
warm_only=warm_only,
)
return {
"result_id": job.result_id,
"result_kind": binding.result_kind,
"relation": "exact-recorded-computation",
"access": "observatory",
"created_at_utc": binding.run_created_at_utc,
"observatory_projection_available": True,
"identity": {
"job_id": job.job_id,
"source_session_id": job.source_session_id,
"source_catalog_sha256": job.source_catalog_sha256,
"source_bundle_sha256": job.source_bundle_sha256,
"source_capability_manifest_sha256": job.source_capability_manifest_sha256,
"setup_id": job.setup_id,
"definition_sha256": job.definition_sha256,
"package_sha256": job.result_sha256,
"artifact_manifest_id": manifest_id,
},
}
except (
ArtifactGatewayError,
PortableResultPublisherError,
PortableRunDefinitionRegistryError,
SessionNotFoundError,
SessionIntegrityError,
OSError,
ValueError,
KeyError,
TypeError,
):
# Missing/corrupt cache is not proof of computation coverage.
return None
def _verify_file(
self,
path: Path,
digest: str,
byte_length: int,
*,
warm_only: bool,
) -> None:
with self._lock:
before = _fingerprint(path, byte_length)
key = (path, digest)
cached = self._verified.get(key)
if cached is not None and cached[0] == before:
self._verified.move_to_end(key)
if not cached[1]:
raise ValueError("cached result object digest is invalid")
return
self._verified.pop(key, None)
if warm_only:
raise PortableResultCacheCheckRequired("refresh published result verification")
checksum = hashlib.sha256()
with path.open("rb") as stream:
while chunk := stream.read(1024 * 1024):
checksum.update(chunk)
if _fingerprint(path, byte_length) != before:
raise ValueError("cached result object changed")
valid = checksum.hexdigest() == digest
self._verified[key] = (before, valid)
while len(self._verified) > 4096:
self._verified.popitem(last=False)
if not valid:
raise ValueError("cached result object digest is invalid")
def _fingerprint(path: Path, byte_length: int) -> tuple[int, ...]:
metadata = path.lstat()
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size != byte_length:
raise ValueError("cached result object is unavailable or not regular")
return (
metadata.st_dev,
metadata.st_ino,
metadata.st_size,
metadata.st_mtime_ns,
metadata.st_ctime_ns,
)
src/k1link/observatory/portable_setup_projection.py
+38 -14
View File
@@ -22,6 +22,7 @@ from k1link.observatory.portable_run_definitions import (
PortableRunDefinitionRegistry,
PortableRunDefinitionRegistryError,
)
from k1link.observatory.recorded_jobs import ObservatoryRecordedQueueError
from k1link.observatory.source_admission import (
PortableRecordedSourceCapability,
PortableSourceAdmissionError,
@@ -90,6 +91,14 @@ class PortableDefinitionCapabilityProbe(Protocol):
type PortableSourceCapabilityProbe = Callable[[str], PortableRecordedSourceCapability]
class PortablePublishedResultLookup(Protocol):
def find(
self,
source_session_id: str,
definition: PortableRunDefinition,
) -> tuple[dict[str, object], ...]: ...
@dataclass(frozen=True, slots=True)
class _SourceCompatibility:
compatible: bool
@@ -116,6 +125,7 @@ class PortableSetupProjector:
capability_probe: PortableDefinitionCapabilityProbe,
dispatch_available: bool = False,
equipment_capture_registry: EquipmentCaptureRegistry | None = None,
result_cache: PortablePublishedResultLookup | None = None,
) -> None:
if not hasattr(capability_probe, "probe"):
raise PortableSetupProjectionError("portable source capability probe is unavailable")
@@ -123,6 +133,7 @@ class PortableSetupProjector:
self._capability_probe = capability_probe
self._dispatch_available = dispatch_available
self._equipment_capture_registry = equipment_capture_registry
self._result_cache = result_cache
for definition in registry.definitions:
_validate_model_presentation(definition)
if definition.authority.as_dict() != _OBSERVATION_ONLY_AUTHORITY:
@@ -166,8 +177,19 @@ class PortableSetupProjector:
presentation = _presentation(definition)
compatibility = self._probe_source(definition, source, presentation)
executor = definition.executor
try:
existing = (
()
if self._result_cache is None
else self._result_cache.find(source.session_id, definition)
)
except (ObservatoryRecordedQueueError, OSError) as exc:
raise PortableSetupProjectionError("portable result cache is unavailable") from exc
submission_allowed = (
compatibility.compatible and executor.ready and self._dispatch_available
not existing
and compatibility.compatible
and executor.ready
and self._dispatch_available
)
return {
"setup_id": definition.setup_id,
@@ -191,17 +213,23 @@ class PortableSetupProjector:
"reason_code": executor.reason_code,
"reason": executor.reason,
},
"existing_results": [],
"existing_results": list(existing),
"preflight": {
"outcome": "ready" if submission_allowed else "blocked",
"action": "check" if submission_allowed else "blocked",
"reason": self._preflight_reason(
"outcome": "existing" if existing else "ready" if submission_allowed else "blocked",
"action": "open-existing"
if existing
else "check"
if submission_allowed
else "blocked",
"reason": "Точный расчёт сохранён и проверен; доступен без запуска моделей."
if existing
else self._preflight_reason(
definition,
compatibility,
presentation,
),
"submission_allowed": submission_allowed,
"existing_result_ids": [],
"existing_result_ids": [result["result_id"] for result in existing],
},
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
}
@@ -214,10 +242,8 @@ class PortableSetupProjector:
) -> _SourceCompatibility:
source_session_id = source.session_id
if self._equipment_capture_registry is not None:
expected_profile = (
self._equipment_capture_registry.compatible_profile_for_requirements(
definition.source_requirements.as_dict()
)
expected_profile = self._equipment_capture_registry.compatible_profile_for_requirements(
definition.source_requirements.as_dict()
)
if expected_profile is None:
raise PortableSetupProjectionError(
@@ -232,8 +258,7 @@ class PortableSetupProjector:
reason="У записи нет проверенной привязки к оборудованию и профилю записи.",
)
if (
attestation.equipment_model_id
!= expected_profile.equipment.equipment_model_id
attestation.equipment_model_id != expected_profile.equipment.equipment_model_id
or attestation.equipment_model_sha256
!= expected_profile.equipment.equipment_model_sha256
):
@@ -245,8 +270,7 @@ class PortableSetupProjector:
)
if (
attestation.capture_profile_id != expected_profile.capture_profile_id
or attestation.capture_profile_sha256
!= expected_profile.capture_profile_sha256
or attestation.capture_profile_sha256 != expected_profile.capture_profile_sha256
):
return _SourceCompatibility(
compatible=False,
src/k1link/observatory/recorded_jobs.py
+48
View File
@@ -1202,6 +1202,7 @@ class ObservatoryRecordedJobQueue:
*,
enqueue: bool = False,
reject_duplicate_computation: bool = False,
published_result_available: Callable[[ObservatoryRecordedJob], bool] | None = None,
) -> tuple[ObservatoryRecordedJob, bool]:
"""Accept one immutable identity and optionally queue it atomically."""
@@ -1242,6 +1243,20 @@ class ObservatoryRecordedJobQueue:
).fetchone()
if duplicate is not None:
raise ObservatoryRecordedQueueDuplicateError(duplicate["job_id"])
if published_result_available is not None:
# Recheck under the INSERT lock: publication may have completed
# after catalog/preflight. The verifier must not re-enter this DB.
published = connection.execute(
"SELECT * FROM observatory_recorded_jobs "
"WHERE identity_sha256 = ? AND state = 'succeeded' "
"AND publication_state = 'published' "
"ORDER BY created_at_utc DESC, job_id DESC",
(identity_sha256,),
).fetchall()
for row in published:
candidate = _job_from_row(row)
if published_result_available(candidate):
raise ObservatoryRecordedQueueDuplicateError(candidate.job_id)
self._require_capacity(
connection,
table="observatory_recorded_jobs",
@@ -2091,6 +2106,27 @@ class ObservatoryRecordedJobQueue:
).fetchall()
return tuple(_job_from_row(row) for row in rows)
def published_results(
self, *, source_session_id: str, source_catalog_sha256: str,
setup_id: str, definition_sha256: str,
) -> tuple[ObservatoryRecordedJob, ...]:
"""Exact cache candidates, never inferred from labels or a truncated job page."""
_validate_pattern(source_session_id, _SESSION_ID, "source session id")
_validate_pattern(setup_id, _IDENTIFIER, "setup id")
_validate_pattern(source_catalog_sha256, _SHA256, "source catalog sha256")
_validate_pattern(definition_sha256, _SHA256, "definition sha256")
with self._read_connection() as connection:
rows = connection.execute(
"SELECT * FROM observatory_recorded_jobs "
"WHERE source_session_id = ? AND source_catalog_sha256 = ? "
"AND setup_id = ? AND definition_sha256 = ? "
"AND state = 'succeeded' AND publication_state = 'published' "
"ORDER BY created_at_utc DESC, job_id DESC",
(source_session_id, source_catalog_sha256, setup_id, definition_sha256),
).fetchall()
return tuple(_job_from_row(row) for row in rows)
def request_live(self, intent: ObservatoryLiveLeaseIntent) -> tuple[ObservatoryLiveLease, bool]:
"""Close recorded admission without allowing a monolith to delay live K1."""
@@ -2746,6 +2782,18 @@ class ObservatoryRecordedJobQueue:
connection.executescript(_SCHEMA_SQL)
self._migrate_claim_lease_schema(connection)
self._migrate_publication_schema(connection)
# A legacy database may not have publication columns until
# the migration above. Never create the partial index earlier.
connection.execute(
"CREATE INDEX IF NOT EXISTS observatory_recorded_jobs_published_source "
"ON observatory_recorded_jobs "
"(source_session_id, source_catalog_sha256, setup_id, definition_sha256) "
"WHERE state = 'succeeded' AND publication_state = 'published'"
)
connection.execute(
"CREATE INDEX IF NOT EXISTS observatory_recorded_jobs_computation "
"ON observatory_recorded_jobs (identity_sha256)"
)
self._validate_schema(connection)
self._validate_existing_capacity(connection)
connection.commit()
src/k1link/web/app.py
+18
View File
@@ -54,6 +54,7 @@ from k1link.observatory.portable_queue_binding import (
PortableQueueBindingError,
PortableRecordedQueueBindingService,
)
from k1link.observatory.portable_result_cache import PortableResultCache
from k1link.observatory.portable_result_contract import (
PortableCalculationProfileRegistry,
PortableResultContractValidatorRegistry,
@@ -538,24 +539,41 @@ else:
OBSERVATORY_PORTABLE_BINDING_SERVICE: PortableRecordedQueueBindingService | None
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableSetupProjector | None
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
OBSERVATORY_PORTABLE_RESULT_CACHE: PortableResultCache | None = None
try:
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None:
raise PortableSetupProjectionError(
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR
or "portable definition registry is unavailable"
)
if (
session_artifact_gateway is not None
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
and OBSERVATORY_PORTABLE_CALCULATION_PROFILES is not None
):
OBSERVATORY_PORTABLE_RESULT_CACHE = PortableResultCache(
sessions=session_store, artifacts=session_artifact_gateway.store,
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
)
OBSERVATORY_PORTABLE_BINDING_SERVICE = PortableRecordedQueueBindingService(
data_dir=session_store.data_dir,
session_store=session_store,
media_inspector=session_recorded_media_inspector,
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
published_result_available=(
None if OBSERVATORY_PORTABLE_RESULT_CACHE is None
else OBSERVATORY_PORTABLE_RESULT_CACHE.available
),
)
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableSetupProjector(
registry=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
capability_probe=OBSERVATORY_PORTABLE_BINDING_SERVICE,
dispatch_available=OBSERVATORY_WORKER_DISPATCH_READY,
equipment_capture_registry=session_store.equipment_capture_registry,
result_cache=OBSERVATORY_PORTABLE_RESULT_CACHE,
)
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None
except (
src/k1link/web/observatory_api.py
+35 -1
View File
@@ -33,6 +33,7 @@ from k1link.observatory.portable_queue_binding import (
PortableQueueBindingStaleCheckError,
PortableRecordedQueueBindingService,
)
from k1link.observatory.portable_result_cache import PortableResultCacheCheckRequired
from k1link.observatory.portable_result_contract import PortableResultPublisherError
from k1link.observatory.portable_result_publisher import (
PortableObservatoryResultPublisher,
@@ -298,6 +299,29 @@ def build_observatory_router(
)
compatible = compatibility.get("compatible") is True
executor_ready = executor.get("state") == "ready" and executor.get("ready") is True
projected_preflight = projected.get("preflight")
if (
isinstance(projected_preflight, dict)
and projected_preflight.get("outcome") == "existing"
):
return {
"schema_version": OBSERVATORY_RUN_PREFLIGHT_SCHEMA,
"source_session_id": request.source_session_id,
"setup_id": request.setup_id,
"definition_sha256": expected_digest,
"check_sha256": None,
"outcome": "existing",
"submission_allowed": False,
"checks": [{
"check_id": "published-result",
"outcome": "pass",
"reason_code": "exact-published-result-verified",
"message": "Точный расчёт сохранён; повторный inference не требуется.",
}],
"existing_result_ids": projected_preflight["existing_result_ids"],
"executor": executor,
"authority": projected.get("authority", dict(_OBSERVATION_ONLY_AUTHORITY)),
}
checked = None
check_reason: str | None = None
if (
@@ -895,6 +919,11 @@ def build_observatory_router(
status_code=409,
detail="Идентичность RunDefinition изменилась; повторите preflight.",
)
if portable_projection.get("existing_results"):
raise HTTPException(
status_code=409,
detail="Точный расчёт уже сохранён. Обновите каталог и откройте результат.",
)
if (
projected_executor.get("state") != "ready"
or projected_executor.get("ready") is not True
@@ -930,11 +959,16 @@ def build_observatory_router(
status_code=409,
detail="Portable-привязка источника не прошла проверку целостности.",
) from exc
except PortableResultCacheCheckRequired as exc:
raise HTTPException(
status_code=409,
detail="Опубликованный результат требует проверки. Обновите каталог.",
) from exc
except ObservatoryRecordedQueueDuplicateError as exc:
raise HTTPException(
status_code=409,
detail=(
"Такой расчёт уже выполняется или ожидает публикации. "
"Такой расчёт уже выполняется, ожидает публикации или сохранён. "
"Обновите список расчётов; повторный запуск не создан."
),
) from exc