feat(observatory): add portable calculation profiles

This commit is contained in:
DCCONSTRUCTIONS
2026-08-31 15:42:56 +03:00
parent d1b75efcea
commit 9beb534108
75 changed files with 24419 additions and 348 deletions
+2
View File
@@ -24,6 +24,7 @@ from k1link.observatory.run_preparations import (
from k1link.observatory.setups import (
LABORATORY_SETUP_CATALOG_SCHEMA,
LABORATORY_SETUP_REGISTRY_SCHEMA,
OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
LaboratorySetupRegistry,
LaboratorySetupRegistryError,
)
@@ -31,6 +32,7 @@ from k1link.observatory.setups import (
__all__ = [
"LABORATORY_SETUP_CATALOG_SCHEMA",
"LABORATORY_SETUP_REGISTRY_SCHEMA",
"OBSERVATORY_CALCULATION_PROFILE_SCHEMA",
"MAX_RUN_PREPARATION_RECORDS",
"MAX_RUN_PREPARATION_STORAGE_BYTES",
"OBSERVATORY_RUN_PREPARATION_REQUEST_SCHEMA",
@@ -0,0 +1,523 @@
"""Exact local composition for the portable M4.9 TRAVEL/TGS executor.
The server supplies only a sealed recorded-job identity. Source delivery and
result upload remain implementations of the shared portable Worker ports; all
filesystem locations, the compiled runner and its build seal are selected by
reviewed Worker-local configuration.
This module does not register an executor or make a blocked runtime ready. It
is the adapter that an installer may bind only after the release archive,
compiled runner, executor image and local admission have all been sealed.
"""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import stat
import subprocess
import tempfile
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Final, Protocol, cast
from k1link.observatory.m49_portable_result import (
M49_PORTABLE_PROFILE_SHA256,
M49_PORTABLE_RESULT_CONTRACT_SHA256,
assemble_m49_portable_result,
)
from k1link.observatory.m49_portable_source import (
M49_PORTABLE_STAGE_SCHEDULE,
M49_PORTABLE_TGS_SEQUENCE,
M49PortableSourceStage,
materialize_m49_portable_source_from_worker_stage,
validate_m49_portable_source_stage,
)
from k1link.observatory.portable_result_contract import (
OBSERVATION_ONLY_AUTHORITY,
canonical_json,
)
from k1link.observatory.portable_run_definitions import PortableRunDefinition
from k1link.observatory.portable_worker_runtime import (
PortableWorkerExecutorAdapter,
PortableWorkerResultDraft,
PortableWorkerResultPublisher,
PortableWorkerRuntimeAdmission,
PortableWorkerRuntimeCandidate,
PortableWorkerRuntimeJobRejectedError,
PortableWorkerRuntimePlan,
PortableWorkerRuntimeUnavailableError,
PortableWorkerSourceMaterializer,
PortableWorkerSourceStage,
)
from k1link.observatory.worker_agent import SealedObservatoryRecordedJob
M49_COMPILED_RUNNER_BUILD_SCHEMA: Final = "missioncore.m49-tgs-portable-compiled-runner-build/v1"
M49_PORTABLE_RUNNER_SOURCE_SHA256: Final = (
"52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9"
)
M49_PORTABLE_RUNNER_WRAPPER_SHA256: Final = (
"2d6c32560682647f868e4ce4c2605749f17c60482a609f8c03ff951411f48ffb"
)
M49_PORTABLE_TRAVEL_BUILD_IMAGE_SHA256: Final = (
"7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f"
)
M49_PORTABLE_RUNTIME_PHASES: Final = (
"source-delivery",
"camera-lidar-timeline-materializer",
"portable-tgs-input-materializer",
"portable-tgs-runner",
"result-v2-assembler",
"observatory-result-publisher",
)
M49_PORTABLE_COMPILED_RUNNER_ASSET_ID: Final = "m49-portable-compiled-runner"
M49_PORTABLE_COMPILED_RUNNER_BUILD_SEAL_ASSET_ID: Final = "m49-portable-compiled-runner-build-seal"
M49_PORTABLE_PROFILE_ASSET_ID: Final = "m49-portable-profile"
M49_PORTABLE_TRAVEL_IMAGE_ASSET_ID: Final = "travel-tgs-image"
M49_PORTABLE_COMPILER_CONTRACT: Final = {
"compiler": "g++",
"language_standard": "c++17",
"flags": ["-O3", "-DNDEBUG", "-pthread"],
"travel_include": "/opt/travel/src/TRAVEL/cpp/travel/core",
"eigen_include": "/usr/include/eigen3",
}
_MAX_BUILD_SEAL_BYTES: Final = 128 * 1024
_MAX_RUN_SECONDS: Final = 24 * 60 * 60
class M49PortableExecutorError(PortableWorkerRuntimeUnavailableError):
"""The local M4.9 executor installation or invocation is not exact."""
class M49PortableRunnerInvoker(Protocol):
def __call__(
self,
*,
binary: Path,
sequence: Path,
schedule: Path,
output: Path,
timing: Path,
workspace: Path,
timeout_seconds: int,
) -> None: ...
@dataclass(frozen=True, slots=True)
class M49PortableRunnerInstallation:
"""Worker-local paths bound to an exact build-only runner seal."""
profile_path: Path
runner_binary_path: Path
runner_build_seal_path: Path
runner_build_seal_sha256: str
output_parent: Path
timeout_seconds: int = _MAX_RUN_SECONDS
def __post_init__(self) -> None:
profile = _exact_file(
self.profile_path,
M49_PORTABLE_PROFILE_SHA256,
"portable M4.9 profile",
)
seal = _exact_file(
self.runner_build_seal_path,
self.runner_build_seal_sha256,
"portable M4.9 runner build seal",
)
binary = _regular_file(self.runner_binary_path, "portable M4.9 compiled runner")
document = _read_build_seal(seal)
binary_row = _object(document["binary"], "portable M4.9 build-seal binary")
if binary_row != {
"file_name": "run_m49_tgs_portable",
"format": "elf",
"byte_length": binary.stat().st_size,
"sha256": _sha256_file(binary),
} or document["profile_sha256"] != _sha256_file(profile):
raise M49PortableExecutorError(
"portable M4.9 compiled runner differs from its build seal"
)
if not os.access(binary, os.X_OK):
raise M49PortableExecutorError("portable M4.9 compiled runner is not executable")
if not _is_elf(binary):
raise M49PortableExecutorError("portable M4.9 compiled runner is not ELF")
parent = self.output_parent.expanduser().absolute()
parent.mkdir(mode=0o700, parents=True, exist_ok=True)
if parent.is_symlink() or not parent.is_dir():
raise M49PortableExecutorError("portable M4.9 output root is unsafe")
if (
isinstance(self.timeout_seconds, bool)
or not isinstance(self.timeout_seconds, int)
or not 1 <= self.timeout_seconds <= _MAX_RUN_SECONDS
):
raise ValueError("portable M4.9 runner timeout is invalid")
object.__setattr__(self, "profile_path", profile)
object.__setattr__(self, "runner_binary_path", binary)
object.__setattr__(self, "runner_build_seal_path", seal)
object.__setattr__(self, "output_parent", parent)
@property
def runner_binary_sha256(self) -> str:
binary = _object(
_read_build_seal(self.runner_build_seal_path)["binary"],
"portable M4.9 build-seal binary",
)
return cast(str, binary["sha256"])
@dataclass(frozen=True, slots=True)
class M49PortableBoundSourceStage(PortableWorkerSourceStage):
"""Claim-bound in-memory extension of the shared source-stage port."""
job: SealedObservatoryRecordedJob
m49_stage: M49PortableSourceStage
def __post_init__(self) -> None:
PortableWorkerSourceStage.__post_init__(self)
if (
self.root != self.m49_stage.root
or self.source_bundle_sha256 != self.job.source_bundle_sha256
or self.source_capability_manifest_sha256 != self.job.source_capability_manifest_sha256
or self.source_adapter_sha256 != self.job.source_adapter_sha256
):
raise M49PortableExecutorError("portable M4.9 bound source differs from its sealed job")
@dataclass(frozen=True, slots=True)
class M49PortableSourceMaterializerAdapter:
"""Adapt exact Worker transport delivery to the M4.9 TGS source port."""
upstream: PortableWorkerSourceMaterializer
profile_path: Path
output_parent: Path
def __post_init__(self) -> None:
profile = _exact_file(
self.profile_path,
M49_PORTABLE_PROFILE_SHA256,
"portable M4.9 profile",
)
parent = self.output_parent.expanduser().absolute()
parent.mkdir(mode=0o700, parents=True, exist_ok=True)
if parent.is_symlink() or not parent.is_dir():
raise M49PortableExecutorError("portable M4.9 source output root is unsafe")
object.__setattr__(self, "profile_path", profile)
object.__setattr__(self, "output_parent", parent)
def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage:
delivered = self.upstream.materialize(job)
materialized = materialize_m49_portable_source_from_worker_stage(
worker_stage=delivered,
job=job,
profile_path=self.profile_path,
output_parent=self.output_parent,
)
return M49PortableBoundSourceStage(
root=materialized.root,
source_bundle_sha256=job.source_bundle_sha256,
source_capability_manifest_sha256=job.source_capability_manifest_sha256,
source_adapter_sha256=job.source_adapter_sha256,
job=job,
m49_stage=materialized,
)
@dataclass(frozen=True, slots=True)
class M49PortableProfileRunnerAdapter:
"""Run the exact installed binary and assemble the deterministic result-v2."""
definition: PortableRunDefinition
installation: M49PortableRunnerInstallation
created_at_utc: Callable[[], str]
invoker: M49PortableRunnerInvoker | None = None
def __post_init__(self) -> None:
_verify_definition(self.definition)
def run(
self,
plan: PortableWorkerRuntimePlan,
source: PortableWorkerSourceStage,
) -> PortableWorkerResultDraft:
if not isinstance(source, M49PortableBoundSourceStage):
raise PortableWorkerRuntimeJobRejectedError(
"portable M4.9 runner requires its claim-bound source stage"
)
job = source.job
_verify_plan(plan, job=job, definition=self.definition)
stage = validate_m49_portable_source_stage(source.root)
_verify_installation_unchanged(self.installation)
workspace = Path(
tempfile.mkdtemp(prefix=".m49-portable-run-", dir=self.installation.output_parent)
)
try:
output = workspace / "outputs"
timing = workspace / "timing.tsv"
invoker = self.invoker or _invoke_exact_runner
invoker(
binary=self.installation.runner_binary_path,
sequence=stage.root / M49_PORTABLE_TGS_SEQUENCE,
schedule=stage.root / M49_PORTABLE_STAGE_SCHEDULE,
output=output,
timing=timing,
workspace=workspace,
timeout_seconds=self.installation.timeout_seconds,
)
package = assemble_m49_portable_result(
source_stage_root=stage.root,
runner_output_root=output,
runner_timing_path=timing,
profile_path=self.installation.profile_path,
output_parent=self.installation.output_parent / "result-packages",
job=job,
definition=self.definition,
created_at_utc=self.created_at_utc(),
)
return PortableWorkerResultDraft(
root=package.root,
result_id=package.result_id,
result_sha256=package.manifest.manifest_sha256,
result_contract_sha256=M49_PORTABLE_RESULT_CONTRACT_SHA256,
)
finally:
shutil.rmtree(workspace, ignore_errors=True)
def compose_m49_portable_executor_adapter(
*,
candidate: PortableWorkerRuntimeCandidate,
definition: PortableRunDefinition,
admission: PortableWorkerRuntimeAdmission,
source_transport: PortableWorkerSourceMaterializer,
result_transport: PortableWorkerResultPublisher,
installation: M49PortableRunnerInstallation,
source_output_parent: Path,
created_at_utc: Callable[[], str],
invoker: M49PortableRunnerInvoker | None = None,
) -> PortableWorkerExecutorAdapter:
"""Compose shared Worker ports without changing their API or registry state."""
_verify_candidate_assets(candidate, installation)
return PortableWorkerExecutorAdapter(
candidate=candidate,
definition=definition,
admission=admission,
source_materializer=M49PortableSourceMaterializerAdapter(
upstream=source_transport,
profile_path=installation.profile_path,
output_parent=source_output_parent,
),
runner=M49PortableProfileRunnerAdapter(
definition=definition,
installation=installation,
created_at_utc=created_at_utc,
invoker=invoker,
),
publisher=result_transport,
)
def _invoke_exact_runner(
*,
binary: Path,
sequence: Path,
schedule: Path,
output: Path,
timing: Path,
workspace: Path,
timeout_seconds: int,
) -> None:
stdout = workspace / "runner.stdout.log"
stderr = workspace / "runner.stderr.log"
try:
with stdout.open("xb") as stdout_stream, stderr.open("xb") as stderr_stream:
completed = subprocess.run(
[str(binary), str(sequence), str(schedule), str(output), str(timing)],
cwd=workspace,
env={"LANG": "C", "LC_ALL": "C", "TZ": "UTC"},
stdin=subprocess.DEVNULL,
stdout=stdout_stream,
stderr=stderr_stream,
check=False,
timeout=timeout_seconds,
)
except (OSError, subprocess.SubprocessError) as exc:
raise M49PortableExecutorError("portable M4.9 runner invocation failed") from exc
if completed.returncode != 0:
raise M49PortableExecutorError("portable M4.9 runner rejected its exact source stage")
def _verify_definition(definition: PortableRunDefinition) -> None:
components = {component.component_id: component for component in definition.components}
profile = components.get("m49-tgs-portable-profile-v2")
if (
definition.setup_id != "m49-tgs-portable-v2"
or definition.definition_id != "m49-tgs-portable"
or definition.result_contract.contract_sha256 != M49_PORTABLE_RESULT_CONTRACT_SHA256
or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY
or profile is None
or profile.sha256 != M49_PORTABLE_PROFILE_SHA256
):
raise M49PortableExecutorError("portable M4.9 RunDefinition identity changed")
def _verify_plan(
plan: PortableWorkerRuntimePlan,
*,
job: SealedObservatoryRecordedJob,
definition: PortableRunDefinition,
) -> None:
if (
plan.job_id != job.job_id
or plan.setup_id != job.setup_id
or plan.definition_sha256 != job.definition_sha256
or plan.source_bundle_sha256 != job.source_bundle_sha256
or plan.source_capability_manifest_sha256 != job.source_capability_manifest_sha256
or plan.result_contract_sha256 != M49_PORTABLE_RESULT_CONTRACT_SHA256
or plan.phases != M49_PORTABLE_RUNTIME_PHASES
or job.definition_sha256 != definition.definition_sha256
):
raise PortableWorkerRuntimeJobRejectedError(
"portable M4.9 runtime plan differs from its sealed job"
)
def _verify_candidate_assets(
candidate: PortableWorkerRuntimeCandidate,
installation: M49PortableRunnerInstallation,
) -> None:
assets = {asset.asset_id: asset for asset in candidate.reusable_assets}
binary = assets.get(M49_PORTABLE_COMPILED_RUNNER_ASSET_ID)
build_seal = assets.get(M49_PORTABLE_COMPILED_RUNNER_BUILD_SEAL_ASSET_ID)
profile = assets.get(M49_PORTABLE_PROFILE_ASSET_ID)
image = assets.get(M49_PORTABLE_TRAVEL_IMAGE_ASSET_ID)
if (
binary is None
or binary.kind != "local-file"
or binary.sha256 != installation.runner_binary_sha256
or binary.byte_length != installation.runner_binary_path.stat().st_size
or build_seal is None
or build_seal.kind != "local-file"
or build_seal.sha256 != installation.runner_build_seal_sha256
or build_seal.byte_length != installation.runner_build_seal_path.stat().st_size
or profile is None
or profile.sha256 != M49_PORTABLE_PROFILE_SHA256
or image is None
or image.kind != "container-image"
or image.sha256 != M49_PORTABLE_TRAVEL_BUILD_IMAGE_SHA256
):
raise M49PortableExecutorError(
"portable M4.9 runtime candidate lacks exact installed runner assets"
)
def _verify_installation_unchanged(installation: M49PortableRunnerInstallation) -> None:
_exact_file(
installation.profile_path,
M49_PORTABLE_PROFILE_SHA256,
"portable M4.9 profile",
)
_exact_file(
installation.runner_build_seal_path,
installation.runner_build_seal_sha256,
"portable M4.9 runner build seal",
)
binary = _regular_file(
installation.runner_binary_path,
"portable M4.9 compiled runner",
)
if (
_sha256_file(binary) != installation.runner_binary_sha256
or not os.access(binary, os.X_OK)
or not _is_elf(binary)
):
raise M49PortableExecutorError("portable M4.9 installed runner changed")
def _read_build_seal(path: Path) -> dict[str, object]:
try:
payload = path.read_bytes()
decoded: object = json.loads(payload.decode("utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise M49PortableExecutorError("portable M4.9 build seal is unreadable") from exc
if not 0 < len(payload) <= _MAX_BUILD_SEAL_BYTES:
raise M49PortableExecutorError("portable M4.9 build seal size is invalid")
document = _object(decoded, "portable M4.9 build seal")
if (
set(document)
!= {
"schema_version",
"source_revision",
"source_state",
"build_image_sha256",
"profile_sha256",
"runner_source_sha256",
"runner_wrapper_sha256",
"compiler_contract",
"binary",
"authority",
}
or payload != canonical_json(document)
or document["schema_version"] != M49_COMPILED_RUNNER_BUILD_SCHEMA
or not isinstance(document["source_revision"], str)
or len(document["source_revision"]) != 40
or any(character not in "0123456789abcdef" for character in document["source_revision"])
or document["source_state"] != "committed-snapshot"
or document["build_image_sha256"] != M49_PORTABLE_TRAVEL_BUILD_IMAGE_SHA256
or document["profile_sha256"] != M49_PORTABLE_PROFILE_SHA256
or document["runner_source_sha256"] != M49_PORTABLE_RUNNER_SOURCE_SHA256
or document["runner_wrapper_sha256"] != M49_PORTABLE_RUNNER_WRAPPER_SHA256
or document["compiler_contract"] != M49_PORTABLE_COMPILER_CONTRACT
or document["authority"] != OBSERVATION_ONLY_AUTHORITY
):
raise M49PortableExecutorError("portable M4.9 build seal identity changed")
return document
def _exact_file(path: Path, expected_sha256: str, label: str) -> Path:
candidate = _regular_file(path, label)
if _sha256_file(candidate) != expected_sha256:
raise M49PortableExecutorError(f"{label} digest changed")
return candidate
def _regular_file(path: Path, label: str) -> Path:
candidate = path.expanduser().absolute()
try:
metadata = candidate.lstat()
resolved = candidate.resolve(strict=True)
except OSError as exc:
raise M49PortableExecutorError(f"{label} is unavailable") from exc
if (
stat.S_ISLNK(metadata.st_mode)
or not stat.S_ISREG(metadata.st_mode)
or not os.path.samefile(candidate, resolved)
):
raise M49PortableExecutorError(f"{label} is unsafe")
return resolved
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _is_elf(path: Path) -> bool:
try:
with path.open("rb") as stream:
return stream.read(4) == b"\x7fELF"
except OSError:
return False
def _object(value: object, label: str) -> dict[str, object]:
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
raise M49PortableExecutorError(f"{label} must be an object")
return cast(dict[str, object], value)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -133,9 +133,12 @@ class PortableRecordedQueueBindingService:
) -> PortableRecordedSourceCapability:
"""Return cheap authoritative compatibility without replay preparation."""
portable, recorded = self._resolve_definition(setup_id, definition_sha256)
# Catalog projection must remain available for not-yet-installed
# executors. Probe only resolves the immutable portable definition;
# check/admit/submit still require a ready executor before source reads.
portable = self._definitions.resolve(setup_id, definition_sha256)
capability = self._source_service(portable).probe(source_session_id)
if recorded.source_adapter_sha256 != capability.source_adapter_sha256:
if portable.source_adapter.contract_sha256 != capability.source_adapter_sha256:
raise PortableQueueBindingIntegrityError(
"portable registry and source capability adapter identities disagree"
)
@@ -0,0 +1,601 @@
"""Strict, path-free contracts for portable Observatory result packages."""
from __future__ import annotations
import hashlib
import json
import re
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Final, cast
from k1link.observatory.portable_run_definitions import (
PortableRunDefinition,
canonical_sha256,
)
from k1link.observatory.recorded_jobs import ObservatoryRecordedJob
PORTABLE_RESULT_PACKAGE_SCHEMA: Final = (
"missioncore.observatory-portable-result-package/v1"
)
PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA: Final = (
"missioncore.observatory-portable-result-package-identity/v1"
)
PORTABLE_RESULT_PUBLICATION_SCHEMA: Final = (
"missioncore.observatory-portable-result-publication/v1"
)
OBSERVATORY_CALCULATION_PROFILE_SCHEMA: Final = (
"missioncore.observatory-calculation-profile/v1"
)
RESULT_PACKAGE_MANIFEST_NAME: Final = "manifest.json"
RESULT_DOCUMENT_ROLE: Final = "result-document"
RESULT_PACKAGE_MANIFEST_ROLE: Final = "result-package-manifest"
_MAX_MANIFEST_BYTES: Final = 1024 * 1024
_MAX_RESULT_DOCUMENT_BYTES: Final = 8 * 1024 * 1024
_MAX_ARTIFACTS: Final = 128
_MAX_ARTIFACT_BYTES: Final = (1 << 63) - 1
_SHA256: Final = re.compile(r"^[a-f0-9]{64}$")
_IDENTIFIER: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
_SESSION_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
_ROLE: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
_PATH_COMPONENT: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
_LAB_ID: Final = re.compile(r"^LAB [A-Z][A-Z0-9._-]{0,31}$")
OBSERVATION_ONLY_AUTHORITY: Final = {
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
"production_accepted": False,
}
class PortableResultPublisherError(RuntimeError):
"""Base class for verified portable result publication failures."""
class PortableResultPackageIntegrityError(PortableResultPublisherError):
"""The supplied package or one of its content identities is invalid."""
class PortableResultPublicationBlockedError(PortableResultPublisherError):
"""A required server-owned definition, policy, or validator is absent."""
@dataclass(frozen=True, slots=True)
class PortableResultArtifact:
"""One confined, content-addressed package member."""
role: str
relative_path: str
media_type: str
byte_length: int
sha256: str
def __post_init__(self) -> None:
_pattern(self.role, _ROLE, "portable result artifact role")
if self.role == RESULT_PACKAGE_MANIFEST_ROLE:
raise ValueError("portable result artifact role is reserved")
relative_artifact_path(self.relative_path)
_media_type(self.media_type)
if (
not isinstance(self.byte_length, int)
or isinstance(self.byte_length, bool)
or not 0 <= self.byte_length <= _MAX_ARTIFACT_BYTES
):
raise ValueError("portable result artifact byte length is invalid")
digest(self.sha256, "portable result artifact sha256")
def as_dict(self) -> dict[str, object]:
return {
"role": self.role,
"relative_path": self.relative_path,
"media_type": self.media_type,
"byte_length": self.byte_length,
"sha256": self.sha256,
}
@dataclass(frozen=True, slots=True)
class PortableResultPackageManifest:
"""Strict Worker-produced manifest with a separately hashed identity."""
identity_sha256: str
created_at_utc: str
job: dict[str, object]
source: dict[str, object]
run_definition: dict[str, object]
result: dict[str, object]
authority: dict[str, object]
artifacts: tuple[PortableResultArtifact, ...]
def __post_init__(self) -> None:
digest(self.identity_sha256, "portable result package identity sha256")
_timestamp(self.created_at_utc, "portable result package creation time")
if self.authority != OBSERVATION_ONLY_AUTHORITY:
raise PortableResultPackageIntegrityError(
"portable result package is not observation-only"
)
if not 1 <= len(self.artifacts) <= _MAX_ARTIFACTS:
raise PortableResultPackageIntegrityError(
"portable result package artifact count is invalid"
)
roles = tuple(artifact.role for artifact in self.artifacts)
paths = tuple(artifact.relative_path for artifact in self.artifacts)
if roles != tuple(sorted(roles)) or len(set(roles)) != len(roles):
raise PortableResultPackageIntegrityError(
"portable result package artifacts are not canonically ordered"
)
if len(set(paths)) != len(paths):
raise PortableResultPackageIntegrityError(
"portable result package artifact paths are not unique"
)
result_documents = tuple(
artifact for artifact in self.artifacts if artifact.role == RESULT_DOCUMENT_ROLE
)
if (
len(result_documents) != 1
or result_documents[0].media_type != "application/json"
or not 0 < result_documents[0].byte_length <= _MAX_RESULT_DOCUMENT_BYTES
):
raise PortableResultPackageIntegrityError(
"portable result package requires one JSON result document"
)
if self.identity_sha256 != canonical_sha256(self.identity_document()):
raise PortableResultPackageIntegrityError(
"portable result package identity digest changed"
)
@classmethod
def create(
cls,
*,
job: ObservatoryRecordedJob,
definition: PortableRunDefinition,
result_id: str,
created_at_utc: str,
artifacts: Sequence[PortableResultArtifact],
) -> PortableResultPackageManifest:
"""Build the canonical package envelope used by a future Worker assembler."""
_pattern(result_id, _SESSION_ID, "portable result id")
job_document = job_identity_document(job)
source_document = source_identity_document(job)
definition_document = run_definition_document(definition)
result_document = result_identity_document(definition, result_id)
authority: dict[str, object] = dict(OBSERVATION_ONLY_AUTHORITY)
normalized_artifacts = tuple(
sorted(artifacts, key=lambda artifact: artifact.role)
)
identity_document = _package_identity_document(
created_at_utc=created_at_utc,
job=job_document,
source=source_document,
run_definition=definition_document,
result=result_document,
authority=authority,
artifacts=normalized_artifacts,
)
return cls(
identity_sha256=canonical_sha256(identity_document),
created_at_utc=created_at_utc,
job=job_document,
source=source_document,
run_definition=definition_document,
result=result_document,
authority=authority,
artifacts=normalized_artifacts,
)
@classmethod
def from_bytes(cls, payload: bytes) -> PortableResultPackageManifest:
if not 0 < len(payload) <= _MAX_MANIFEST_BYTES:
raise PortableResultPackageIntegrityError(
"portable result package manifest size is invalid"
)
try:
decoded: object = json.loads(payload.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise PortableResultPackageIntegrityError(
"portable result package manifest is not valid JSON"
) from exc
document = object_document(decoded, "portable result package manifest")
exact_keys(
document,
{
"schema_version",
"identity_sha256",
"created_at_utc",
"job",
"source",
"run_definition",
"result",
"authority",
"artifacts",
},
"portable result package manifest",
)
if document["schema_version"] != PORTABLE_RESULT_PACKAGE_SCHEMA:
raise PortableResultPackageIntegrityError(
"portable result package manifest schema is invalid"
)
if payload != canonical_json(document):
raise PortableResultPackageIntegrityError(
"portable result package manifest is not canonical JSON"
)
artifacts_value = document["artifacts"]
if not isinstance(artifacts_value, list):
raise PortableResultPackageIntegrityError(
"portable result package artifacts are not an array"
)
artifacts = tuple(_artifact(value) for value in artifacts_value)
try:
return cls(
identity_sha256=string(
document["identity_sha256"],
"portable result package identity sha256",
),
created_at_utc=string(
document["created_at_utc"],
"portable result package creation time",
),
job=object_document(document["job"], "portable result job identity"),
source=object_document(
document["source"], "portable result source identity"
),
run_definition=object_document(
document["run_definition"],
"portable result RunDefinition",
),
result=object_document(document["result"], "portable result identity"),
authority=object_document(
document["authority"], "portable result authority"
),
artifacts=artifacts,
)
except ValueError as exc:
raise PortableResultPackageIntegrityError(
"portable result package manifest field is invalid"
) from exc
def identity_document(self) -> dict[str, object]:
return _package_identity_document(
created_at_utc=self.created_at_utc,
job=self.job,
source=self.source,
run_definition=self.run_definition,
result=self.result,
authority=self.authority,
artifacts=self.artifacts,
)
def as_dict(self) -> dict[str, object]:
identity = self.identity_document()
identity.pop("schema_version")
return {
"schema_version": PORTABLE_RESULT_PACKAGE_SCHEMA,
"identity_sha256": self.identity_sha256,
**identity,
}
@property
def canonical_bytes(self) -> bytes:
return canonical_json(self.as_dict())
@property
def manifest_sha256(self) -> str:
return hashlib.sha256(self.canonical_bytes).hexdigest()
@dataclass(frozen=True, slots=True)
class PortableCalculationProfilePolicy:
"""Server-owned presentation identity bound to one exact RunDefinition."""
setup_id: str
definition_id: str
definition_version: int
definition_sha256: str
lab_id: str
display_name: str
include_recorded_media: bool = False
def __post_init__(self) -> None:
_pattern(self.setup_id, _IDENTIFIER, "portable calculation profile setup id")
_pattern(
self.definition_id,
_IDENTIFIER,
"portable calculation profile definition id",
)
if (
not isinstance(self.definition_version, int)
or isinstance(self.definition_version, bool)
or self.definition_version < 1
):
raise ValueError("portable calculation profile version is invalid")
digest(
self.definition_sha256,
"portable calculation profile definition sha256",
)
if _LAB_ID.fullmatch(self.lab_id) is None:
raise ValueError("portable calculation profile LAB id is invalid")
_text(self.display_name, "portable calculation profile display name", maximum=160)
if not isinstance(self.include_recorded_media, bool):
raise ValueError("portable calculation profile media policy is invalid")
def as_dict(self) -> dict[str, object]:
return {
"schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
"setup_id": self.setup_id,
"display_name": self.display_name,
"origin": "archived-definition",
"definition_id": self.definition_id,
"definition_version": self.definition_version,
"definition_sha256": self.definition_sha256,
}
@property
def identity_sha256(self) -> str:
return canonical_sha256(self.as_dict())
@dataclass(frozen=True, slots=True)
class PortableCalculationProfileRegistry:
policies: tuple[PortableCalculationProfilePolicy, ...]
def __post_init__(self) -> None:
keys = tuple(
(policy.setup_id, policy.definition_sha256) for policy in self.policies
)
if len(keys) != len(set(keys)):
raise ValueError("portable calculation profile policies are not unique")
def resolve(
self,
definition: PortableRunDefinition,
) -> PortableCalculationProfilePolicy:
for policy in self.policies:
if (
policy.setup_id == definition.setup_id
and policy.definition_sha256 == definition.definition_sha256
):
if (
policy.definition_id != definition.definition_id
or policy.definition_version != definition.version
):
raise PortableResultPublicationBlockedError(
"calculation profile policy disagrees with the RunDefinition"
)
return policy
raise PortableResultPublicationBlockedError(
"exact calculation profile policy is not registered"
)
@dataclass(frozen=True, slots=True)
class PortableResultValidationContext:
"""Read-only inputs supplied to one exact result-contract validator."""
manifest: PortableResultPackageManifest
job: ObservatoryRecordedJob
definition: PortableRunDefinition
result_document: Mapping[str, object]
artifact_paths: Mapping[str, Path]
type PortableResultContractValidator = Callable[[PortableResultValidationContext], None]
@dataclass(frozen=True, slots=True)
class PortableResultContractValidatorRegistration:
contract_sha256: str
validator: PortableResultContractValidator
def __post_init__(self) -> None:
digest(self.contract_sha256, "portable result validator contract sha256")
if not callable(self.validator):
raise ValueError("portable result contract validator is not callable")
@dataclass(frozen=True, slots=True)
class PortableResultContractValidatorRegistry:
registrations: tuple[PortableResultContractValidatorRegistration, ...]
def __post_init__(self) -> None:
digests = tuple(registration.contract_sha256 for registration in self.registrations)
if len(digests) != len(set(digests)):
raise ValueError("portable result contract validators are not unique")
def resolve(self, contract_sha256: str) -> PortableResultContractValidator:
digest(contract_sha256, "portable result contract sha256")
for registration in self.registrations:
if registration.contract_sha256 == contract_sha256:
return registration.validator
raise PortableResultPublicationBlockedError(
"exact portable result-contract validator is not installed"
)
def job_identity_document(job: ObservatoryRecordedJob) -> dict[str, object]:
return {
"job_id": job.job_id,
"request_sha256": job.request_sha256,
"identity_sha256": job.identity_sha256,
"submission_receipt_sha256": job.submission_receipt_sha256,
"claim_generation": job.claim_generation,
}
def source_identity_document(job: ObservatoryRecordedJob) -> dict[str, object]:
return {
"session_id": job.source_session_id,
"catalog_sha256": job.source_catalog_sha256,
"bundle_sha256": job.source_bundle_sha256,
"capability_manifest_sha256": job.source_capability_manifest_sha256,
"adapter": {
"adapter_id": job.source_adapter_id,
"version": job.source_adapter_version,
"adapter_sha256": job.source_adapter_sha256,
},
}
def run_definition_document(definition: PortableRunDefinition) -> dict[str, object]:
return {
**definition.identity_document(),
"definition_sha256": definition.definition_sha256,
}
def result_identity_document(
definition: PortableRunDefinition,
result_id: str,
) -> dict[str, object]:
contract = definition.result_contract
return {
"result_id": result_id,
"result_schema": contract.result_schema,
"result_kind": contract.result_kind,
"result_contract_sha256": contract.contract_sha256,
}
def _package_identity_document(
*,
created_at_utc: str,
job: Mapping[str, object],
source: Mapping[str, object],
run_definition: Mapping[str, object],
result: Mapping[str, object],
authority: Mapping[str, object],
artifacts: Sequence[PortableResultArtifact],
) -> dict[str, object]:
return {
"schema_version": PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
"created_at_utc": created_at_utc,
"job": dict(job),
"source": dict(source),
"run_definition": dict(run_definition),
"result": dict(result),
"authority": dict(authority),
"artifacts": [artifact.as_dict() for artifact in artifacts],
}
def _artifact(value: object) -> PortableResultArtifact:
row = object_document(value, "portable result artifact")
exact_keys(
row,
{"role", "relative_path", "media_type", "byte_length", "sha256"},
"portable result artifact",
)
byte_length = row["byte_length"]
if not isinstance(byte_length, int) or isinstance(byte_length, bool):
raise PortableResultPackageIntegrityError(
"portable result artifact byte length is invalid"
)
return PortableResultArtifact(
role=string(row["role"], "portable result artifact role"),
relative_path=string(
row["relative_path"],
"portable result artifact relative path",
),
media_type=string(row["media_type"], "portable result artifact media type"),
byte_length=byte_length,
sha256=string(row["sha256"], "portable result artifact sha256"),
)
def relative_artifact_path(value: object) -> PurePosixPath:
if not isinstance(value, str) or not 1 <= len(value) <= 512:
raise ValueError("portable result artifact path is invalid")
path = PurePosixPath(value)
if (
path.is_absolute()
or path.as_posix() != value
or len(path.parts) < 2
or path.parts[0] != "artifacts"
or any(
part in {"", ".", ".."} or _PATH_COMPONENT.fullmatch(part) is None
for part in path.parts
)
):
raise ValueError("portable result artifact path is invalid")
return path
def _media_type(value: object) -> str:
if (
not isinstance(value, str)
or not 3 <= len(value) <= 255
or "/" not in value
or value != value.strip()
or any(ord(character) < 32 or ord(character) > 126 for character in value)
):
raise ValueError("portable result artifact media type is invalid")
return value
def canonical_json(value: object) -> bytes:
try:
return json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode("utf-8")
except (TypeError, ValueError) as exc:
raise PortableResultPackageIntegrityError(
"portable result package is not JSON-compatible"
) from exc
def object_document(value: object, label: str) -> dict[str, object]:
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
raise PortableResultPackageIntegrityError(f"{label} must be an object")
return cast(dict[str, object], value)
def exact_keys(value: Mapping[str, object], expected: set[str], label: str) -> None:
if set(value) != expected:
raise PortableResultPackageIntegrityError(f"{label} fields are invalid")
def string(value: object, label: str) -> str:
if not isinstance(value, str):
raise PortableResultPackageIntegrityError(f"{label} must be text")
return value
def _pattern(value: object, pattern: re.Pattern[str], label: str) -> str:
if not isinstance(value, str) or pattern.fullmatch(value) is None:
raise ValueError(f"{label} is invalid")
return value
def digest(value: object, label: str) -> str:
return _pattern(value, _SHA256, label)
def _text(value: object, label: str, *, maximum: int) -> str:
if (
not isinstance(value, str)
or not 1 <= len(value) <= maximum
or value != value.strip()
):
raise ValueError(f"{label} is invalid")
return value
def _timestamp(value: object, label: str) -> str:
from datetime import datetime
if not isinstance(value, str):
raise ValueError(f"{label} is invalid")
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as exc:
raise ValueError(f"{label} is invalid") from exc
if parsed.tzinfo is None:
raise ValueError(f"{label} must include a timezone")
return value
@@ -0,0 +1,809 @@
"""Verified publication boundary for portable recorded Observatory results.
A Worker success acknowledgement is only a transport receipt. This module
admits a result into the Session catalog only after a canonical result package
is bound to the exact durable job, persisted source contracts, portable
RunDefinition, result-contract validator, and observation-only authority.
The publisher deliberately has no production default validators or presentation
policy. An unknown result contract or definition-bound calculation profile is
therefore a hard publication blocker rather than an invitation to infer one from
the current UI selection or a historical LAB label.
"""
from __future__ import annotations
import hashlib
import json
import re
import stat
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import cast
from k1link.artifact_gateway import (
ArtifactGatewayError,
ArtifactManifest,
ArtifactMember,
CentralArtifactStore,
)
from k1link.observatory.portable_result_contract import (
OBSERVATION_ONLY_AUTHORITY,
PORTABLE_RESULT_PACKAGE_SCHEMA,
PORTABLE_RESULT_PUBLICATION_SCHEMA,
RESULT_DOCUMENT_ROLE,
RESULT_PACKAGE_MANIFEST_NAME,
RESULT_PACKAGE_MANIFEST_ROLE,
PortableCalculationProfilePolicy,
PortableCalculationProfileRegistry,
PortableResultArtifact,
PortableResultContractValidatorRegistry,
PortableResultPackageIntegrityError,
PortableResultPackageManifest,
PortableResultPublicationBlockedError,
PortableResultPublisherError,
PortableResultValidationContext,
canonical_json,
exact_keys,
job_identity_document,
object_document,
relative_artifact_path,
result_identity_document,
run_definition_document,
source_identity_document,
string,
)
from k1link.observatory.portable_result_contract import (
digest as validate_digest,
)
from k1link.observatory.portable_run_definitions import (
PortableRunDefinition,
PortableRunDefinitionRegistry,
PortableRunDefinitionRegistryError,
canonical_sha256,
)
from k1link.observatory.recorded_jobs import ObservatoryRecordedJob
from k1link.observatory.source_admission import (
PORTABLE_SOURCE_BUNDLE_SCHEMA,
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
PORTABLE_SOURCE_DOCUMENT_DIRECTORY,
)
from k1link.sessions.models import LabSessionBinding, SessionIntegrityError, SessionSummary
from k1link.sessions.store import SessionStore
_COPY_CHUNK_BYTES = 1024 * 1024
_FULL_ROUTE_LABEL = "полный маршрут и воспроизведение"
_LEGACY_CANONICAL_RESULT = re.compile(
r"^lab-v1-vegetation-shadow-[a-f0-9]{64}$"
)
@dataclass(frozen=True, slots=True)
class PublishedPortableObservatoryResult:
binding: LabSessionBinding
package: PortableResultPackageManifest
artifact_manifest: ArtifactManifest
class PortableObservatoryResultPublisher:
"""Validate, archive, and project one exact portable Worker result."""
def __init__(
self,
*,
session_store: SessionStore,
artifact_store: CentralArtifactStore,
definitions: PortableRunDefinitionRegistry,
calculation_profiles: PortableCalculationProfileRegistry,
validators: PortableResultContractValidatorRegistry,
) -> None:
self._session_store = session_store
self._artifact_store = artifact_store
self._definitions = definitions
self._calculation_profiles = calculation_profiles
self._validators = validators
def publish(
self,
*,
job: ObservatoryRecordedJob,
package_root: Path,
) -> PublishedPortableObservatoryResult:
"""Publish only a verified terminal package; exact retries are idempotent."""
_verify_terminal_job(job)
try:
definition = self._definitions.resolve(job.setup_id, job.definition_sha256)
except (PortableRunDefinitionRegistryError, ValueError) as exc:
raise PortableResultPublicationBlockedError(
"recorded job RunDefinition is not in the portable registry"
) from exc
_verify_definition_job_identity(definition, job)
profile = self._calculation_profiles.resolve(definition)
validator = self._validators.resolve(definition.result_contract.contract_sha256)
root, manifest_path, package = _read_package(package_root)
_verify_package_identity(package, job=job, definition=definition)
if package.manifest_sha256 != job.result_sha256 or root.name != job.result_sha256:
raise PortableResultPackageIntegrityError(
"portable result package content address disagrees with queue success"
)
source_summary = self._verify_source(job, definition)
artifact_paths = _verify_package_artifacts(root, package.artifacts)
result_document = _read_canonical_result_document(
artifact_paths[RESULT_DOCUMENT_ROLE]
)
context = PortableResultValidationContext(
manifest=package,
job=job,
definition=definition,
result_document=result_document,
artifact_paths=artifact_paths,
)
try:
validator(context)
except PortableResultPublisherError:
raise
except Exception as exc:
raise PortableResultPackageIntegrityError(
"portable result document failed its exact contract validator"
) from exc
artifact_manifest = self._archive_package(
job=job,
package=package,
manifest_path=manifest_path,
artifact_paths=artifact_paths,
profile=profile,
)
provenance = _publication_provenance(
job=job,
definition=definition,
package=package,
artifact_manifest=artifact_manifest,
profile=profile,
)
try:
binding = self._session_store.publish_lab_instance(
session_id=cast(str, job.result_id),
source_session_id=job.source_session_id,
display_name=_portable_result_display_name(source_summary),
lab_id=profile.lab_id,
result_kind=definition.result_contract.result_kind,
result_id=cast(str, job.result_id),
config_sha256=definition.definition_sha256,
run_created_at_utc=job.updated_at_utc,
replay_capability=None,
provenance=provenance,
include_recorded_media=profile.include_recorded_media,
expected_source_catalog_sha256=job.source_catalog_sha256,
)
except (SessionIntegrityError, ValueError) as exc:
raise PortableResultPackageIntegrityError(
"portable result could not be projected as immutable LAB provenance"
) from exc
return PublishedPortableObservatoryResult(
binding=binding,
package=package,
artifact_manifest=artifact_manifest,
)
def _verify_source(
self,
job: ObservatoryRecordedJob,
definition: PortableRunDefinition,
) -> SessionSummary:
try:
detail, catalog_sha256 = (
self._session_store.get_session_with_catalog_snapshot(
job.source_session_id
)
)
except Exception as exc:
raise PortableResultPackageIntegrityError(
"portable result source session is unavailable"
) from exc
if (
detail.summary.session_id != job.source_session_id
or detail.summary.lab is not None
or catalog_sha256 != job.source_catalog_sha256
):
raise PortableResultPackageIntegrityError(
"portable result source catalog changed after admission"
)
_verify_source_documents(
self._session_store.data_dir,
job=job,
definition=definition,
)
return detail.summary
def _archive_package(
self,
*,
job: ObservatoryRecordedJob,
package: PortableResultPackageManifest,
manifest_path: Path,
artifact_paths: Mapping[str, Path],
profile: PortableCalculationProfilePolicy,
) -> ArtifactManifest:
members = [
_publish_exact_member(
self._artifact_store,
role=RESULT_PACKAGE_MANIFEST_ROLE,
media_type="application/json",
source=manifest_path,
expected_sha256=cast(str, job.result_sha256),
expected_byte_length=len(package.canonical_bytes),
)
]
for artifact in package.artifacts:
members.append(
_publish_exact_member(
self._artifact_store,
role=artifact.role,
media_type=artifact.media_type,
source=artifact_paths[artifact.role],
expected_sha256=artifact.sha256,
expected_byte_length=artifact.byte_length,
)
)
try:
archived = self._artifact_store.publish_manifest(
artifact_type="observatory-portable-result",
subject_id=cast(str, job.result_id),
members=members,
metadata={
"package-sha256": cast(str, job.result_sha256),
"package-identity-sha256": package.identity_sha256,
"job-id": job.job_id,
"job-identity-sha256": job.identity_sha256,
"definition-sha256": job.definition_sha256,
"source-bundle-sha256": job.source_bundle_sha256,
"result-contract-sha256": string(
package.result["result_contract_sha256"],
"portable result contract sha256",
),
"calculation-profile-sha256": profile.identity_sha256,
},
created_at_utc=job.updated_at_utc,
)
verified = self._artifact_store.read_manifest(archived.manifest_id)
except (ArtifactGatewayError, OSError, ValueError) as exc:
raise PortableResultPackageIntegrityError(
"portable result package could not be archived immutably"
) from exc
if verified != archived:
raise PortableResultPackageIntegrityError(
"portable result artifact manifest changed after publication"
)
return archived
def _portable_result_display_name(source: SessionSummary) -> str:
"""Keep the result label source-owned; profile provenance is rendered separately."""
candidate = f"{source.display_name} · {_FULL_ROUTE_LABEL}"
return candidate if len(candidate) <= 160 else source.display_name
def resolve_published_portable_calculation_profile(
summary: SessionSummary,
*,
definitions: PortableRunDefinitionRegistry,
calculation_profiles: PortableCalculationProfileRegistry,
) -> dict[str, object] | None:
"""Resolve only an exact, immutable portable publication profile.
Catalog projection must never infer profile identity from a selected setup,
a display label, or a current definition with the same setup id. Any drift
in the stored publication provenance therefore makes the profile absent.
"""
binding = summary.lab
if binding is None:
return None
try:
provenance = object_document(
binding.provenance,
"portable result publication provenance",
)
exact_keys(
provenance,
{
"schema_version",
"authority",
"calculation_profile",
"calculation_profile_sha256",
"job",
"source",
"run_definition",
"result_package",
"storage",
"method",
},
"portable result publication provenance",
)
profile_document = object_document(
provenance["calculation_profile"],
"portable calculation profile",
)
run_definition = object_document(
provenance["run_definition"],
"portable result RunDefinition",
)
source = object_document(provenance["source"], "portable result source")
setup_id = string(profile_document.get("setup_id"), "portable setup id")
definition_sha256 = string(
run_definition.get("definition_sha256"),
"portable RunDefinition sha256",
)
definition = definitions.resolve(setup_id, definition_sha256)
profile = calculation_profiles.resolve(definition)
if (
provenance["schema_version"] != PORTABLE_RESULT_PUBLICATION_SCHEMA
or provenance["authority"] != OBSERVATION_ONLY_AUTHORITY
or provenance["calculation_profile_sha256"] != profile.identity_sha256
or profile_document != profile.as_dict()
or canonical_sha256(profile_document) != profile.identity_sha256
or run_definition != run_definition_document(definition)
or binding.session_id != summary.session_id
or binding.result_id != summary.session_id
or binding.source_session_id != source.get("session_id")
or binding.lab_id != profile.lab_id
or binding.config_sha256 != definition.definition_sha256
or binding.result_kind != definition.result_contract.result_kind
):
return None
return profile.as_dict()
except (
KeyError,
PortableResultPublisherError,
PortableRunDefinitionRegistryError,
TypeError,
ValueError,
):
return None
def _verify_terminal_job(job: ObservatoryRecordedJob) -> None:
if (
job.state != "succeeded"
or job.result_id is None
or job.result_sha256 is None
or job.terminal_code != "result-sealed"
or job.terminal_claim_token_sha256 is None
or job.claim_generation < 1
or job.active_claim_token is not None
or job.active_claimant_id is not None
):
raise PortableResultPublicationBlockedError(
"recorded job has no exact terminal Worker result receipt"
)
if _LEGACY_CANONICAL_RESULT.fullmatch(job.result_id) is not None:
raise PortableResultPublicationBlockedError(
"legacy canonical result namespace is immutable and reserved"
)
def _verify_definition_job_identity(
definition: PortableRunDefinition,
job: ObservatoryRecordedJob,
) -> None:
if not definition.executor.ready:
raise PortableResultPublicationBlockedError(
"portable result RunDefinition executor is not installed"
)
try:
recorded = definition.to_recorded_run_definition()
except Exception as exc:
raise PortableResultPublicationBlockedError(
"portable result RunDefinition cannot produce a queue identity"
) from exc
expected = (
recorded.setup_id,
recorded.definition_id,
recorded.definition_version,
recorded.definition_sha256,
recorded.source_adapter_id,
recorded.source_adapter_version,
recorded.source_adapter_sha256,
recorded.executor_release_id,
recorded.executor_release_sha256,
recorded.executor_image_sha256,
recorded.model_release_ids,
recorded.model_manifest_sha256,
recorded.resource_profile_id,
recorded.resource_profile_sha256,
recorded.checkpoint_policy,
recorded.allowed_checkpoints,
)
actual = (
job.setup_id,
job.definition_id,
job.definition_version,
job.definition_sha256,
job.source_adapter_id,
job.source_adapter_version,
job.source_adapter_sha256,
job.executor_release_id,
job.executor_release_sha256,
job.executor_image_sha256,
job.model_release_ids,
job.model_manifest_sha256,
job.resource_profile_id,
job.resource_profile_sha256,
job.checkpoint_policy,
job.allowed_checkpoints,
)
if actual != expected or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY:
raise PortableResultPackageIntegrityError(
"recorded job and portable RunDefinition identities disagree"
)
def _read_package(
package_root: Path,
) -> tuple[Path, Path, PortableResultPackageManifest]:
candidate = package_root.expanduser().absolute()
try:
root_metadata = candidate.lstat()
root = candidate.resolve(strict=True)
except OSError as exc:
raise PortableResultPackageIntegrityError(
"portable result package root is unavailable"
) from exc
if stat.S_ISLNK(root_metadata.st_mode) or not stat.S_ISDIR(root_metadata.st_mode):
raise PortableResultPackageIntegrityError(
"portable result package root must be a regular directory"
)
manifest_path = root / RESULT_PACKAGE_MANIFEST_NAME
try:
metadata = manifest_path.lstat()
payload = manifest_path.read_bytes()
except OSError as exc:
raise PortableResultPackageIntegrityError(
"portable result package manifest is unavailable"
) from exc
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
raise PortableResultPackageIntegrityError(
"portable result package manifest must be a regular file"
)
package = PortableResultPackageManifest.from_bytes(payload)
if hashlib.sha256(payload).hexdigest() != package.manifest_sha256:
raise PortableResultPackageIntegrityError(
"portable result package manifest digest changed"
)
return root, manifest_path, package
def _verify_package_identity(
package: PortableResultPackageManifest,
*,
job: ObservatoryRecordedJob,
definition: PortableRunDefinition,
) -> None:
if package.job != job_identity_document(job):
raise PortableResultPackageIntegrityError(
"portable result package is bound to another queue job"
)
if package.source != source_identity_document(job):
raise PortableResultPackageIntegrityError(
"portable result package is bound to another source"
)
if package.run_definition != run_definition_document(definition):
raise PortableResultPackageIntegrityError(
"portable result package is bound to another RunDefinition"
)
if package.result != result_identity_document(definition, cast(str, job.result_id)):
raise PortableResultPackageIntegrityError(
"portable result package violates its sealed result contract"
)
if package.authority != definition.authority.as_dict():
raise PortableResultPackageIntegrityError(
"portable result package authority changed"
)
def _verify_source_documents(
data_dir: Path,
*,
job: ObservatoryRecordedJob,
definition: PortableRunDefinition,
) -> None:
root = data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY
try:
metadata = root.lstat()
except OSError as exc:
raise PortableResultPublicationBlockedError(
"admitted portable source documents are unavailable"
) from exc
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode):
raise PortableResultPackageIntegrityError(
"portable source document root is unsafe"
)
bundle = _read_content_addressed_document(root, job.source_bundle_sha256)
capability = _read_content_addressed_document(
root,
job.source_capability_manifest_sha256,
)
exact_keys(
bundle,
{
"schema_version",
"source_session_id",
"source_catalog_sha256",
"plugin_id",
"archive_id",
"source_adapter",
"sources",
"spatial_replay",
"camera",
"authority",
},
"portable source bundle",
)
exact_keys(
capability,
{
"schema_version",
"source_session_id",
"source_catalog_sha256",
"source_bundle_sha256",
"source_adapter_sha256",
"modalities",
"camera_profile",
"calibration",
"authority",
},
"portable source capability",
)
expected_adapter = {
"id": job.source_adapter_id,
"version": job.source_adapter_version,
"sha256": job.source_adapter_sha256,
}
if (
bundle["schema_version"] != PORTABLE_SOURCE_BUNDLE_SCHEMA
or bundle["source_session_id"] != job.source_session_id
or bundle["source_catalog_sha256"] != job.source_catalog_sha256
or bundle["plugin_id"] != definition.source_requirements.plugin_id
or bundle["archive_id"] != definition.source_requirements.archive_id
or bundle["source_adapter"] != expected_adapter
or bundle["authority"] != OBSERVATION_ONLY_AUTHORITY
or capability["schema_version"] != PORTABLE_SOURCE_CAPABILITY_SCHEMA
or capability["source_session_id"] != job.source_session_id
or capability["source_catalog_sha256"] != job.source_catalog_sha256
or capability["source_bundle_sha256"] != job.source_bundle_sha256
or capability["source_adapter_sha256"] != job.source_adapter_sha256
or capability["authority"] != OBSERVATION_ONLY_AUTHORITY
):
raise PortableResultPackageIntegrityError(
"persisted portable source contract disagrees with the queue job"
)
def _read_content_addressed_document(root: Path, document_sha256: str) -> dict[str, object]:
validate_digest(document_sha256, "portable source document sha256")
path = root / f"{document_sha256}.json"
try:
metadata = path.lstat()
payload = path.read_bytes()
except OSError as exc:
raise PortableResultPublicationBlockedError(
"an admitted portable source document is unavailable"
) from exc
if (
stat.S_ISLNK(metadata.st_mode)
or not stat.S_ISREG(metadata.st_mode)
or hashlib.sha256(payload).hexdigest() != document_sha256
):
raise PortableResultPackageIntegrityError(
"admitted portable source document digest changed"
)
try:
decoded: object = json.loads(payload.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise PortableResultPackageIntegrityError(
"admitted portable source document is not valid JSON"
) from exc
document = object_document(decoded, "portable source document")
if payload != canonical_json(document):
raise PortableResultPackageIntegrityError(
"admitted portable source document is not canonical JSON"
)
return document
def _verify_package_artifacts(
root: Path,
artifacts: Sequence[PortableResultArtifact],
) -> dict[str, Path]:
resolved: dict[str, Path] = {}
for artifact in artifacts:
path = _resolve_package_member(root, artifact.relative_path)
digest, byte_length = _hash_file(path)
if digest != artifact.sha256 or byte_length != artifact.byte_length:
raise PortableResultPackageIntegrityError(
f"portable result artifact content changed: {artifact.role}"
)
resolved[artifact.role] = path
return resolved
def _resolve_package_member(root: Path, relative_path: str) -> Path:
portable = relative_artifact_path(relative_path)
candidate = root.joinpath(*portable.parts)
current = root
try:
for part in portable.parts:
current = current / part
metadata = current.lstat()
if stat.S_ISLNK(metadata.st_mode):
raise PortableResultPackageIntegrityError(
"portable result artifact path contains a symlink"
)
resolved = candidate.resolve(strict=True)
metadata = candidate.lstat()
except PortableResultPublisherError:
raise
except OSError as exc:
raise PortableResultPackageIntegrityError(
"portable result artifact is unavailable"
) from exc
if not resolved.is_relative_to(root) or not stat.S_ISREG(metadata.st_mode):
raise PortableResultPackageIntegrityError(
"portable result artifact escapes its package"
)
return resolved
def _read_canonical_result_document(path: Path) -> dict[str, object]:
try:
payload = path.read_bytes()
decoded: object = json.loads(payload.decode("utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise PortableResultPackageIntegrityError(
"portable result document is not valid JSON"
) from exc
document = object_document(decoded, "portable result document")
if payload != canonical_json(document):
raise PortableResultPackageIntegrityError(
"portable result document is not canonical JSON"
)
return document
def _publish_exact_member(
store: CentralArtifactStore,
*,
role: str,
media_type: str,
source: Path,
expected_sha256: str,
expected_byte_length: int,
) -> ArtifactMember:
try:
published = store.publish_file(source)
except (ArtifactGatewayError, OSError, ValueError) as exc:
raise PortableResultPackageIntegrityError(
f"portable result artifact could not be archived: {role}"
) from exc
if (
published.sha256 != expected_sha256
or published.byte_length != expected_byte_length
):
raise PortableResultPackageIntegrityError(
f"portable result artifact changed while it was archived: {role}"
)
return ArtifactMember(
role=role,
media_type=media_type,
sha256=published.sha256,
byte_length=published.byte_length,
)
def _publication_provenance(
*,
job: ObservatoryRecordedJob,
definition: PortableRunDefinition,
package: PortableResultPackageManifest,
artifact_manifest: ArtifactManifest,
profile: PortableCalculationProfilePolicy,
) -> dict[str, object]:
result_document = next(
artifact for artifact in package.artifacts if artifact.role == RESULT_DOCUMENT_ROLE
)
return {
"schema_version": PORTABLE_RESULT_PUBLICATION_SCHEMA,
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
"calculation_profile": profile.as_dict(),
"calculation_profile_sha256": profile.identity_sha256,
"job": job_identity_document(job),
"source": source_identity_document(job),
"run_definition": run_definition_document(definition),
"result_package": {
"schema_version": PORTABLE_RESULT_PACKAGE_SCHEMA,
"manifest_sha256": package.manifest_sha256,
"identity_sha256": package.identity_sha256,
"artifact_manifest_id": artifact_manifest.manifest_id,
"result_document_sha256": result_document.sha256,
"artifacts": [artifact.as_dict() for artifact in package.artifacts],
},
"storage": {
"mode": "central-content-addressed-artifact-store",
"include_recorded_media": profile.include_recorded_media,
"replay_capability": None,
},
"method": _laboratory_method(job, definition),
}
def _laboratory_method(
job: ObservatoryRecordedJob,
definition: PortableRunDefinition,
) -> dict[str, object]:
components: list[dict[str, object]] = [
{
"kind": "source",
"name": job.source_session_id,
"version": f"{job.source_adapter_id}/v{job.source_adapter_version}",
"role": "immutable admitted K1 source bundle",
"identity_sha256": job.source_bundle_sha256,
},
{
"kind": "algorithm",
"name": definition.definition_id,
"version": f"v{definition.version}",
"role": "portable laboratory RunDefinition",
"identity_sha256": definition.definition_sha256,
},
{
"kind": "runtime",
"name": job.executor_release_id,
"version": "sealed-release",
"role": "portable Worker executor",
"identity_sha256": job.executor_release_sha256,
},
{
"kind": "runtime",
"name": job.resource_profile_id,
"version": "sealed-resource-profile",
"role": "exclusive Worker resource contract",
"identity_sha256": job.resource_profile_sha256,
},
]
components.extend(
{
"kind": "model",
"name": model.release_id,
"version": model.revision or "sealed-artifacts",
"role": "portable inference model",
"identity_sha256": model.identity_sha256,
}
for model in definition.models
)
return {
"schema_version": "missioncore.laboratory-method/v1",
"completeness": "complete",
"execution_class": "hybrid" if definition.models else "deterministic",
"pipeline_id": definition.definition_id,
"components": components,
}
def _hash_file(path: Path) -> tuple[str, int]:
digest = hashlib.sha256()
byte_length = 0
try:
with path.open("rb") as stream:
while chunk := stream.read(_COPY_CHUNK_BYTES):
digest.update(chunk)
byte_length += len(chunk)
except OSError as exc:
raise PortableResultPackageIntegrityError(
"portable result artifact could not be read"
) from exc
return digest.hexdigest(), byte_length
@@ -510,26 +510,46 @@ class PortableRunDefinition:
by_kind: dict[str, list[ImmutableComponentIdentity]] = {}
for component in self.components:
by_kind.setdefault(component.kind, []).append(component)
for required_kind in (
"calibration",
# Calibration is common to every admitted K1 source. Learned-model
# definitions additionally keep the original profile/runner/FOV seals;
# algorithm-only definitions such as M4.9 may remain projectable while
# their portable runner is not installed yet.
required_kinds: tuple[str, ...] = ("calibration",)
if self.models:
required_kinds += (
"profile",
"runner",
"valid-fov-identity",
"valid-fov-mask",
)
for required_kind in required_kinds:
if len(by_kind.get(required_kind, [])) != 1:
raise PortableRunDefinitionRegistryError(
f"portable definition requires exactly one {required_kind} component"
)
for singleton_kind in (
"profile",
"runner",
"valid-fov-identity",
"valid-fov-mask",
):
if len(by_kind.get(required_kind, [])) != 1:
if len(by_kind.get(singleton_kind, [])) > 1:
raise PortableRunDefinitionRegistryError(
f"portable definition requires exactly one {required_kind} component"
f"portable definition allows at most one {singleton_kind} component"
)
if not self.models and self.executor.ready and len(by_kind.get("runner", [])) != 1:
raise PortableRunDefinitionRegistryError(
"ready portable definition requires exactly one runner component"
)
calibration = by_kind["calibration"][0]
if calibration.sha256 != self.source_requirements.calibration_identity_sha256:
raise PortableRunDefinitionRegistryError(
"source capability and runtime calibration identities disagree"
)
model_ids = [model.release_id for model in self.models]
if not model_ids or model_ids != sorted(model_ids) or len(model_ids) != len(set(model_ids)):
if model_ids != sorted(model_ids) or len(model_ids) != len(set(model_ids)):
raise PortableRunDefinitionRegistryError(
"models must be non-empty, unique, and canonically ordered"
"models must be unique and canonically ordered"
)
if self.executor.contour_id != self.resource_profile.contour_id:
raise PortableRunDefinitionRegistryError(
@@ -720,13 +740,38 @@ class PortableRunDefinitionRegistry:
"portable setup and definition identity are not allowlisted"
)
def to_recorded_registry(self) -> RecordedRunDefinitionRegistry:
"""Convert the complete registry and fail if any definition is blocked."""
def resolve_setup(self, setup_id: str) -> PortableRunDefinition:
"""Resolve the single current definition projected for one setup."""
return RecordedRunDefinitionRegistry(
tuple(definition.to_recorded_run_definition() for definition in self.definitions)
_pattern(setup_id, _IDENTIFIER, "setup id")
for definition in self.definitions:
if definition.setup_id == setup_id:
return definition
raise PortableRunDefinitionRegistryError("portable setup is not allowlisted")
def ready_recorded_definitions(self) -> tuple[RecordedRunDefinition, ...]:
"""Return only definitions with fully sealed, installed executors.
Blocked definitions remain valid catalog entries and do not prevent an
unrelated ready definition from entering the durable queue allowlist.
"""
return tuple(
definition.to_recorded_run_definition()
for definition in self.definitions
if definition.executor.ready
)
def to_recorded_registry(self) -> RecordedRunDefinitionRegistry:
"""Build a queue registry from ready definitions, ignoring blocked ones."""
definitions = self.ready_recorded_definitions()
if not definitions:
raise PortableRunDefinitionUnavailableError(
"portable registry has no sealed and installed executor"
)
return RecordedRunDefinitionRegistry(definitions)
def canonical_sha256(value: object) -> str:
"""Return the repository-wide canonical JSON SHA-256 identity."""
@@ -1,15 +1,9 @@
"""UI-ready Observatory projection for the portable LAB V1 definition.
"""UI-ready Observatory projection for source-independent portable setups.
This module deliberately does not extend the legacy setup registry and does
not submit work. It projects two independent facts for one selected source:
* the result of a lightweight recorded-source capability probe;
* the executor state sealed by the portable RunDefinition;
Keeping those facts separate prevents a compatible new recording from being
described as incompatible merely because the executor is not installed yet.
Historical vegetation-shadow results belong only to the legacy catalog and
never become exact results of the generic portable v2 definition.
The projector keeps source capability, executor availability, and dispatch
availability separate. One blocked definition therefore remains visible
without hiding another definition or weakening either definition's admission
contract. Historical LAB results remain exclusively in the legacy catalog.
"""
from __future__ import annotations
@@ -17,11 +11,16 @@ from __future__ import annotations
import re
from collections.abc import Callable
from dataclasses import dataclass
from typing import Final, Protocol, runtime_checkable
from typing import Final, Protocol
from k1link.observatory.portable_result_contract import (
PortableCalculationProfilePolicy,
PortableCalculationProfileRegistry,
)
from k1link.observatory.portable_run_definitions import (
PortableRunDefinition,
PortableRunDefinitionRegistry,
PortableRunDefinitionRegistryError,
)
from k1link.observatory.source_admission import (
PortableRecordedSourceCapability,
@@ -34,6 +33,8 @@ PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA: Final = (
)
PORTABLE_LAB_V1_SETUP_ID: Final = "lab-v1-eomt-ddrnet-portable-v1"
PORTABLE_LAB_V1_DISPLAY_NAME: Final = "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39"
PORTABLE_M49_SETUP_ID: Final = "m49-tgs-portable-v2"
PORTABLE_M49_DISPLAY_NAME: Final = "M4.9T5 · TRAVEL TGS · CPU-only, без ML"
_SOURCE_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
_OBSERVATION_ONLY_AUTHORITY: Final = {
@@ -43,6 +44,25 @@ _OBSERVATION_ONLY_AUTHORITY: Final = {
"production_accepted": False,
}
_SETUP_PRESENTATION: Final = {
PORTABLE_LAB_V1_SETUP_ID: {
"lab_id": "LAB V1",
"display_name": PORTABLE_LAB_V1_DISPLAY_NAME,
"description": ("Проверка записанной K1-сессии моделями EoMT и DDRNet; только наблюдение."),
"compatible": "Запись соответствует требованиям EoMT + DDRNet.",
"incompatible": "Запись не соответствует требованиям EoMT + DDRNet.",
"executor_unavailable": "Вычислительный контур LAB V1 пока недоступен.",
},
PORTABLE_M49_SETUP_ID: {
"lab_id": "LAB M4.9T5",
"display_name": PORTABLE_M49_DISPLAY_NAME,
"description": ("Динамический TGS-разбор записанной K1-сессии без ML; только наблюдение."),
"compatible": "Запись соответствует требованиям TRAVEL TGS.",
"incompatible": "Запись не соответствует требованиям TRAVEL TGS.",
"executor_unavailable": "Переносимый вычислительный контур M4.9T5 пока недоступен.",
},
}
_MODEL_PRESENTATION: Final = {
"eomt-cityscapes-large-1024-v1": "EoMT Cityscapes Large 1024",
"lab-v1-ddrnet-39-goose-fine-64-v1": "DDRNet-39",
@@ -50,82 +70,104 @@ _MODEL_PRESENTATION: Final = {
class PortableSetupProjectionError(RuntimeError):
"""The portable setup cannot be projected without weakening its contract."""
"""A portable setup cannot be projected without weakening its contract."""
@runtime_checkable
class PortableSourceCapabilityProbeService(Protocol):
"""A definition-bound lightweight source-capability service."""
class PortableDefinitionCapabilityProbe(Protocol):
"""Definition-bound lightweight capability probe."""
def probe(self, source_session_id: str) -> PortableRecordedSourceCapability:
"""Probe one source without preparing media or persisting documents."""
def probe(
self,
*,
source_session_id: str,
setup_id: str,
definition_sha256: str,
) -> PortableRecordedSourceCapability:
"""Probe one source against one exact portable definition."""
type PortableSourceCapabilityProbe = (
Callable[[str], PortableRecordedSourceCapability] | PortableSourceCapabilityProbeService
)
type PortableSourceCapabilityProbe = Callable[[str], PortableRecordedSourceCapability]
@dataclass(frozen=True, slots=True)
class _SourceCompatibility:
compatible: bool
capability: PortableRecordedSourceCapability | None
reason: str
def as_dict(self) -> dict[str, object]:
return {
"outcome": "pass" if self.compatible else "blocked",
"compatible": self.compatible,
"reason": (
"Запись соответствует требованиям EoMT + DDRNet."
if self.compatible
else "Запись не соответствует требованиям EoMT + DDRNet."
),
"reason": self.reason,
}
class PortableLabV1SetupProjector:
"""Project the generic portable LAB V1 setup for one selected source."""
class PortableSetupProjector:
"""Project every allowlisted portable setup for one selected source."""
def __init__(
self,
*,
registry: PortableRunDefinitionRegistry,
capability_probe: PortableSourceCapabilityProbe,
capability_probe: PortableDefinitionCapabilityProbe,
dispatch_available: bool = False,
) -> None:
self._definition = _resolve_lab_v1_definition(registry)
if not isinstance(
capability_probe,
PortableSourceCapabilityProbeService,
) and not callable(capability_probe):
if not hasattr(capability_probe, "probe"):
raise PortableSetupProjectionError("portable source capability probe is unavailable")
self._registry = registry
self._capability_probe = capability_probe
_validate_model_presentation(self._definition)
if self._definition.authority.as_dict() != _OBSERVATION_ONLY_AUTHORITY:
raise PortableSetupProjectionError("portable LAB V1 authority is not observation-only")
self._dispatch_available = dispatch_available
for definition in registry.definitions:
_validate_model_presentation(definition)
if definition.authority.as_dict() != _OBSERVATION_ONLY_AUTHORITY:
raise PortableSetupProjectionError(
"portable setup authority is not observation-only"
)
def has_setup(self, setup_id: str) -> bool:
try:
self._registry.resolve_setup(setup_id)
except (PortableRunDefinitionRegistryError, ValueError):
return False
return True
def catalog(self, source: SessionSummary) -> dict[str, object]:
"""Return a one-setup v2 catalog projection for ``source``."""
"""Return all independent portable setup projections for ``source``."""
return {
"schema_version": PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA,
"source_session_id": source.session_id,
"setups": [self.project(source)],
"setups": [
self.project(source, setup_id=definition.setup_id)
for definition in self._registry.definitions
],
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
}
def project(self, source: SessionSummary) -> dict[str, object]:
"""Return a strict, observation-only setup projection."""
def project(
self,
source: SessionSummary,
*,
setup_id: str,
) -> dict[str, object]:
"""Return one strict observation-only portable setup projection."""
_validate_source_id(source.session_id)
compatibility = self._probe_source(source.session_id)
definition = self._definition
try:
definition = self._registry.resolve_setup(setup_id)
except PortableRunDefinitionRegistryError as exc:
raise PortableSetupProjectionError("portable setup is unavailable") from exc
presentation = _presentation(definition)
compatibility = self._probe_source(definition, source.session_id, presentation)
executor = definition.executor
submission_allowed = (
compatibility.compatible and executor.ready and self._dispatch_available
)
return {
"setup_id": definition.setup_id,
"display_name": PORTABLE_LAB_V1_DISPLAY_NAME,
"description": (
"Проверка записанной K1-сессии моделями EoMT и DDRNet; только наблюдение."
),
"display_name": presentation["display_name"],
"description": presentation["description"],
"origin": "portable-definition",
"source_requirements": definition.source_requirements.as_dict(),
"run_definition": {
@@ -141,65 +183,169 @@ class PortableLabV1SetupProjector:
"contour_id": executor.contour_id,
"state": executor.state,
"ready": executor.ready,
"reason_code": executor.reason_code,
"reason": executor.reason,
},
"existing_results": [],
"preflight": {
"outcome": "blocked",
"action": "blocked",
"reason": self._preflight_reason(compatibility),
"submission_allowed": False,
"outcome": "ready" if submission_allowed else "blocked",
"action": "check" if submission_allowed else "blocked",
"reason": self._preflight_reason(
definition,
compatibility,
presentation,
),
"submission_allowed": submission_allowed,
"existing_result_ids": [],
},
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
}
def _probe_source(self, source_session_id: str) -> _SourceCompatibility:
def _probe_source(
self,
definition: PortableRunDefinition,
source_session_id: str,
presentation: dict[str, str],
) -> _SourceCompatibility:
try:
if isinstance(
self._capability_probe,
PortableSourceCapabilityProbeService,
):
capability = self._capability_probe.probe(source_session_id)
else:
capability = self._capability_probe(source_session_id)
capability = self._capability_probe.probe(
source_session_id=source_session_id,
setup_id=definition.setup_id,
definition_sha256=definition.definition_sha256,
)
except PortableSourceAdmissionError:
return _SourceCompatibility(compatible=False, capability=None)
return _SourceCompatibility(
compatible=False,
capability=None,
reason=presentation["incompatible"],
)
if not isinstance(capability, PortableRecordedSourceCapability):
raise PortableSetupProjectionError("capability probe returned an invalid result")
if capability.source_session_id != source_session_id:
raise PortableSetupProjectionError(
"capability probe is bound to another source session"
)
if capability.source_adapter_sha256 != self._definition.source_adapter.contract_sha256:
if capability.source_adapter_sha256 != definition.source_adapter.contract_sha256:
raise PortableSetupProjectionError("capability probe uses another source adapter")
return _SourceCompatibility(compatible=True, capability=capability)
def _preflight_reason(self, compatibility: _SourceCompatibility) -> str:
if not compatibility.compatible:
return "Запись не соответствует требованиям этого сетапа."
if not self._definition.executor.ready:
return "Вычислительный контур LAB V1 пока недоступен."
return (
"Server-side проверка definition/check SHA и постановка portable "
"LAB V1 в очередь пока недоступны."
return _SourceCompatibility(
compatible=True,
capability=capability,
reason=presentation["compatible"],
)
def _preflight_reason(
self,
definition: PortableRunDefinition,
compatibility: _SourceCompatibility,
presentation: dict[str, str],
) -> str:
if not compatibility.compatible:
return "Запись не соответствует требованиям этого сетапа."
if not definition.executor.ready:
return presentation["executor_unavailable"]
if not self._dispatch_available:
return "Server-side проверка и постановка portable-сетапа в очередь недоступны."
return "Сетап готов к server-side проверке источника перед постановкой в очередь."
def _resolve_lab_v1_definition(
class PortableLabV1SetupProjector(PortableSetupProjector):
"""Backward-compatible single-definition LAB V1 projector."""
def __init__(
self,
*,
registry: PortableRunDefinitionRegistry,
capability_probe: PortableSourceCapabilityProbe | object,
) -> None:
try:
definition = registry.resolve_setup(PORTABLE_LAB_V1_SETUP_ID)
except PortableRunDefinitionRegistryError as exc:
raise PortableSetupProjectionError(
"portable LAB V1 definition is unavailable or ambiguous"
) from exc
class _LegacyProbeAdapter:
def probe(
adapter_self,
*,
source_session_id: str,
setup_id: str,
definition_sha256: str,
) -> PortableRecordedSourceCapability:
del adapter_self
if (
setup_id != definition.setup_id
or definition_sha256 != definition.definition_sha256
):
raise PortableSetupProjectionError("portable LAB V1 definition changed")
probe_method = getattr(capability_probe, "probe", None)
if callable(probe_method):
capability = probe_method(source_session_id)
elif callable(capability_probe):
capability = capability_probe(source_session_id)
else:
raise PortableSetupProjectionError(
"portable source capability probe is unavailable"
)
if not isinstance(capability, PortableRecordedSourceCapability):
raise PortableSetupProjectionError(
"capability probe returned an invalid result"
)
return capability
super().__init__(
registry=PortableRunDefinitionRegistry((definition,)),
capability_probe=_LegacyProbeAdapter(),
dispatch_available=False,
)
def project(
self,
source: SessionSummary,
*,
setup_id: str = PORTABLE_LAB_V1_SETUP_ID,
) -> dict[str, object]:
return super().project(source, setup_id=setup_id)
def _presentation(definition: PortableRunDefinition) -> dict[str, str]:
configured = _SETUP_PRESENTATION.get(definition.setup_id)
if configured is not None:
return dict(configured)
return {
"lab_id": "LAB PORTABLE",
"display_name": definition.setup_id,
"description": "Переносимый анализ записанной K1-сессии; только наблюдение.",
"compatible": "Запись соответствует требованиям сетапа.",
"incompatible": "Запись не соответствует требованиям сетапа.",
"executor_unavailable": "Вычислительный контур сетапа пока недоступен.",
}
def portable_calculation_profile_registry(
registry: PortableRunDefinitionRegistry,
) -> PortableRunDefinition:
matching = tuple(
definition
for definition in registry.definitions
if definition.setup_id == PORTABLE_LAB_V1_SETUP_ID
)
if len(matching) != 1:
raise PortableSetupProjectionError("portable LAB V1 definition is unavailable or ambiguous")
return matching[0]
) -> PortableCalculationProfileRegistry:
"""Build exact publication policies from the server-owned presentation map."""
policies: list[PortableCalculationProfilePolicy] = []
for definition in registry.definitions:
presentation = _presentation(definition)
policies.append(
PortableCalculationProfilePolicy(
setup_id=definition.setup_id,
definition_id=definition.definition_id,
definition_version=definition.version,
definition_sha256=definition.definition_sha256,
lab_id=presentation["lab_id"],
display_name=presentation["display_name"],
)
)
return PortableCalculationProfileRegistry(tuple(policies))
def _validate_model_presentation(definition: PortableRunDefinition) -> None:
if definition.setup_id != PORTABLE_LAB_V1_SETUP_ID:
return
releases = {model.release_id: model for model in definition.models}
if set(releases) != set(_MODEL_PRESENTATION):
raise PortableSetupProjectionError(
@@ -217,15 +363,14 @@ def _validate_model_presentation(definition: PortableRunDefinition) -> None:
def _project_models(definition: PortableRunDefinition) -> list[dict[str, object]]:
by_release = {model.release_id: model for model in definition.models}
return [
{
"name": _MODEL_PRESENTATION[release_id],
"release_id": release_id,
"model_id": by_release[release_id].model_id,
"architecture": by_release[release_id].architecture,
"name": _MODEL_PRESENTATION.get(model.release_id, model.model_id),
"release_id": model.release_id,
"model_id": model.model_id,
"architecture": model.architecture,
}
for release_id in _MODEL_PRESENTATION
for model in definition.models
]
@@ -0,0 +1,375 @@
"""Fail-closed server composition for portable Observatory Worker jobs.
This module is deliberately only a composition boundary. It does not enable
the Worker router, install an executor, select commands, or grant production
authority. It binds the two admitted portable profiles to their exact result
contracts, then constructs the local artifact transport and verified result
publisher from server-owned dependencies.
"""
from __future__ import annotations
import os
import stat
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Final
from k1link.artifact_gateway import CentralArtifactStore
from k1link.observatory.m49_portable_result import (
M49_PORTABLE_RESULT_CONTRACT_SHA256,
M49_PORTABLE_RESULT_SCHEMA,
validate_m49_portable_result,
)
from k1link.observatory.portable_artifact_transport import (
PortableArtifactTransportError,
PortableObservatoryArtifactTransport,
)
from k1link.observatory.portable_lab_v1_executor import (
PORTABLE_LAB_V1_RESULT_SCHEMA,
validate_lab_v1_result_v2,
)
from k1link.observatory.portable_result_contract import (
OBSERVATION_ONLY_AUTHORITY,
PortableCalculationProfileRegistry,
PortableResultContractValidator,
PortableResultContractValidatorRegistration,
PortableResultContractValidatorRegistry,
PortableResultPublicationBlockedError,
)
from k1link.observatory.portable_result_publisher import (
PortableObservatoryResultPublisher,
)
from k1link.observatory.portable_run_definitions import (
PORTABLE_RESULT_CONTRACT_SCHEMA,
PortableRunDefinition,
PortableRunDefinitionRegistry,
PortableRunDefinitionRegistryError,
)
from k1link.observatory.portable_setup_projection import (
PORTABLE_LAB_V1_SETUP_ID,
PORTABLE_M49_SETUP_ID,
portable_calculation_profile_registry,
)
from k1link.observatory.recorded_jobs import ObservatoryRecordedJobQueue
from k1link.sessions.media import RecordedMediaInspector
from k1link.sessions.store import SessionStore
PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256: Final = (
"b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a"
)
OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_SOURCE_CAS_ROOT"
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV: Final = (
"MISSIONCORE_OBSERVATORY_WORKER_RESULT_STAGING_ROOT"
)
class PortableWorkerIntegrationError(RuntimeError):
"""The exact portable server composition cannot be constructed."""
@dataclass(frozen=True, slots=True)
class PortableWorkerStorageRoots:
"""Pre-provisioned server-owned roots for large portable artifacts."""
source_cas_root: Path
result_staging_root: Path
@classmethod
def from_environment(
cls,
*,
artifact_store_root: Path,
environment: Mapping[str, str] | None = None,
) -> PortableWorkerStorageRoots:
"""Load both roots without creating anything on a missing mount."""
values = os.environ if environment is None else environment
return cls.from_paths(
artifact_store_root=artifact_store_root,
source_cas_root=_required_environment_path(
values,
OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV,
),
result_staging_root=_required_environment_path(
values,
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV,
),
)
@classmethod
def from_paths(
cls,
*,
artifact_store_root: Path,
source_cas_root: Path,
result_staging_root: Path,
) -> PortableWorkerStorageRoots:
"""Validate existing, disjoint roots inside the central store boundary."""
artifact_store = _existing_canonical_directory(
artifact_store_root,
"central artifact store",
)
storage_boundary = _existing_canonical_directory(
artifact_store.parent,
"central artifact storage boundary",
)
_require_mounted_volume(storage_boundary)
source_cas = _existing_canonical_directory(
source_cas_root,
"portable source CAS",
)
result_staging = _existing_canonical_directory(
result_staging_root,
"portable result staging",
)
for label, root in (
("portable source CAS", source_cas),
("portable result staging", result_staging),
):
if root == storage_boundary or not root.is_relative_to(storage_boundary):
raise PortableWorkerIntegrationError(
f"{label} must be inside the central artifact storage boundary"
)
if _paths_overlap(root, artifact_store):
raise PortableWorkerIntegrationError(
f"{label} must be disjoint from the central artifact store"
)
if _paths_overlap(source_cas, result_staging):
raise PortableWorkerIntegrationError(
"portable source CAS and result staging roots must be disjoint"
)
return cls(
source_cas_root=source_cas,
result_staging_root=result_staging,
)
@dataclass(frozen=True, slots=True)
class PortableObservatoryWorkerIntegration:
"""Server-owned objects required by the optional Worker router."""
calculation_profiles: PortableCalculationProfileRegistry
validators: PortableResultContractValidatorRegistry
artifact_transport: PortableObservatoryArtifactTransport
result_publisher: PortableObservatoryResultPublisher
supported_setup_ids: tuple[str, ...]
@dataclass(frozen=True, slots=True)
class _ValidatorSpec:
setup_id: str
definition_id: str
definition_version: int
contract_id: str
contract_version: int
result_schema: str
result_kind: str
contract_sha256: str
validator: PortableResultContractValidator
def expected_contract(self) -> dict[str, object]:
return {
"schema_version": PORTABLE_RESULT_CONTRACT_SCHEMA,
"contract_id": self.contract_id,
"version": self.contract_version,
"result_schema": self.result_schema,
"result_kind": self.result_kind,
"publication": "observatory",
"contract_sha256": self.contract_sha256,
}
_VALIDATOR_SPECS: Final = (
_ValidatorSpec(
setup_id=PORTABLE_LAB_V1_SETUP_ID,
definition_id="lab-v1-eomt-ddrnet-portable",
definition_version=2,
contract_id="recorded-eomt-ddrnet-review-v2",
contract_version=2,
result_schema=PORTABLE_LAB_V1_RESULT_SCHEMA,
result_kind="recorded-perception-qualification",
contract_sha256=PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256,
validator=validate_lab_v1_result_v2,
),
_ValidatorSpec(
setup_id=PORTABLE_M49_SETUP_ID,
definition_id="m49-tgs-portable",
definition_version=2,
contract_id="m49-tgs-portable-review-v2",
contract_version=2,
result_schema=M49_PORTABLE_RESULT_SCHEMA,
result_kind="recorded-perception-qualification",
contract_sha256=M49_PORTABLE_RESULT_CONTRACT_SHA256,
validator=validate_m49_portable_result,
),
)
def portable_result_validator_registry(
definitions: PortableRunDefinitionRegistry,
) -> PortableResultContractValidatorRegistry:
"""Bind both product profiles to fixed result contracts and validators."""
registrations: list[PortableResultContractValidatorRegistration] = []
for spec in _VALIDATOR_SPECS:
try:
definition = definitions.resolve_setup(spec.setup_id)
except PortableRunDefinitionRegistryError as exc:
raise PortableWorkerIntegrationError(
f"required portable setup is unavailable: {spec.setup_id}"
) from exc
_verify_validator_definition(definition, spec)
registrations.append(
PortableResultContractValidatorRegistration(
contract_sha256=spec.contract_sha256,
validator=spec.validator,
)
)
return PortableResultContractValidatorRegistry(tuple(registrations))
def build_portable_observatory_worker_integration(
*,
queue: ObservatoryRecordedJobQueue,
session_store: SessionStore,
media_inspector: RecordedMediaInspector,
definitions: PortableRunDefinitionRegistry,
artifact_store: CentralArtifactStore,
calculation_profiles: PortableCalculationProfileRegistry | None = None,
validators: PortableResultContractValidatorRegistry | None = None,
source_cas_root: Path | None = None,
result_staging_root: Path | None = None,
) -> PortableObservatoryWorkerIntegration:
"""Construct the dormant server foundation without enabling any route."""
try:
if source_cas_root is None or result_staging_root is None:
raise PortableWorkerIntegrationError(
"portable Worker storage roots must be explicitly configured"
)
storage_roots = PortableWorkerStorageRoots.from_paths(
artifact_store_root=artifact_store.root,
source_cas_root=source_cas_root,
result_staging_root=result_staging_root,
)
profiles = calculation_profiles or portable_calculation_profile_registry(definitions)
validator_registry = validators or portable_result_validator_registry(definitions)
_verify_composition(definitions, profiles, validator_registry)
transport = PortableObservatoryArtifactTransport(
queue=queue,
session_store=session_store,
media_inspector=media_inspector,
definitions=definitions,
source_cas_root=storage_roots.source_cas_root,
result_staging_root=storage_roots.result_staging_root,
)
except PortableWorkerIntegrationError:
raise
except (
PortableArtifactTransportError,
PortableResultPublicationBlockedError,
PortableRunDefinitionRegistryError,
OSError,
ValueError,
) as exc:
raise PortableWorkerIntegrationError(
"portable Worker server composition is unavailable"
) from exc
publisher = PortableObservatoryResultPublisher(
session_store=session_store,
artifact_store=artifact_store,
definitions=definitions,
calculation_profiles=profiles,
validators=validator_registry,
)
return PortableObservatoryWorkerIntegration(
calculation_profiles=profiles,
validators=validator_registry,
artifact_transport=transport,
result_publisher=publisher,
supported_setup_ids=tuple(spec.setup_id for spec in _VALIDATOR_SPECS),
)
def _verify_validator_definition(
definition: PortableRunDefinition,
spec: _ValidatorSpec,
) -> None:
if (
definition.definition_id != spec.definition_id
or definition.version != spec.definition_version
or definition.result_contract.as_dict() != spec.expected_contract()
or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY
):
raise PortableWorkerIntegrationError(
f"portable result contract changed for setup: {spec.setup_id}"
)
def _verify_composition(
definitions: PortableRunDefinitionRegistry,
profiles: PortableCalculationProfileRegistry,
validators: PortableResultContractValidatorRegistry,
) -> None:
expected_validators = {spec.contract_sha256: spec.validator for spec in _VALIDATOR_SPECS}
for definition in definitions.definitions:
profiles.resolve(definition)
if definition.executor.ready:
try:
validators.resolve(definition.result_contract.contract_sha256)
except PortableResultPublicationBlockedError as exc:
raise PortableWorkerIntegrationError(
"a ready portable definition has no exact result validator"
) from exc
for contract_sha256, expected in expected_validators.items():
if validators.resolve(contract_sha256) is not expected:
raise PortableWorkerIntegrationError(
"portable result validator registration changed identity"
)
def _required_environment_path(
environment: Mapping[str, str],
name: str,
) -> Path:
raw = environment.get(name)
if not isinstance(raw, str) or not raw or raw != raw.strip():
raise PortableWorkerIntegrationError(f"{name} is required")
path = Path(raw)
if not path.is_absolute():
raise PortableWorkerIntegrationError(f"{name} must be an absolute path")
return path
def _existing_canonical_directory(path: Path, label: str) -> Path:
candidate = path.expanduser().absolute()
try:
metadata = candidate.lstat()
resolved = candidate.resolve(strict=True)
except OSError as exc:
raise PortableWorkerIntegrationError(f"{label} is unavailable") from exc
if (
stat.S_ISLNK(metadata.st_mode)
or not stat.S_ISDIR(metadata.st_mode)
or resolved != candidate
):
raise PortableWorkerIntegrationError(f"{label} is not a canonical directory")
return resolved
def _paths_overlap(left: Path, right: Path) -> bool:
return left == right or left.is_relative_to(right) or right.is_relative_to(left)
def _require_mounted_volume(path: Path) -> None:
parts = path.parts
if len(parts) < 3 or parts[0] != os.sep or parts[1] != "Volumes":
return
mount_point = Path(os.sep, "Volumes", parts[2])
if not os.path.ismount(mount_point):
raise PortableWorkerIntegrationError(
f"central artifact volume is not mounted: {mount_point}"
)
@@ -0,0 +1,969 @@
"""Fail-closed local runtime contract for portable Observatory executors.
This module belongs on Worker 006, not in the browser or the K1 control path.
It binds one source-independent portable RunDefinition to locally verified
assets and to local Python adapters. A queued job can select an adapter only
through the four server-sealed executor digests; it can never supply a command,
path, environment variable, image reference, or priority.
The production candidate registry is deliberately allowed to contain blocked
candidates. Reusable historical assets are evidence, not an installed
executor. A blocked candidate cannot yield a Worker registration or execute a
job even when every reusable asset is present.
"""
from __future__ import annotations
import hashlib
import json
import re
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Final, Literal, Protocol
from k1link.observatory.portable_run_definitions import (
PortableRunDefinition,
PortableRunDefinitionRegistry,
canonical_sha256,
)
from k1link.observatory.worker_agent import (
ObservatoryWorkerExecutionResult,
ObservatoryWorkerExecutorIdentity,
SealedObservatoryRecordedJob,
)
PORTABLE_WORKER_RUNTIME_REGISTRY_SCHEMA: Final = (
"missioncore.observatory-portable-worker-runtime-registry/v1"
)
PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA: Final = (
"missioncore.observatory-portable-worker-runtime-candidate/v1"
)
PORTABLE_WORKER_RUNTIME_PLAN_SCHEMA: Final = (
"missioncore.observatory-portable-worker-runtime-plan/v1"
)
_MAX_REGISTRY_BYTES: Final = 256 * 1024
_SHA256: Final = re.compile(r"^[a-f0-9]{64}$")
_IDENTIFIER: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
_ASSET_ID: Final = re.compile(r"^[a-z][a-z0-9.-]{2,127}$")
_SESSION_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
_JOB_ID: Final = re.compile(r"^observatory-run-[a-f0-9]{32}$")
_AUTHORITY: Final = {
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
"production_accepted": False,
}
type CandidateState = Literal["blocked", "ready"]
type PhaseState = Literal["implemented", "missing"]
type RuntimeAssetKind = Literal[
"container-image",
"definition-component",
"local-file",
"model-artifact",
]
type AssetVerificationState = Literal["matched", "missing", "mismatched"]
class PortableWorkerRuntimeError(RuntimeError):
"""Base error for the local portable Worker runtime boundary."""
class PortableWorkerRuntimeRegistryError(PortableWorkerRuntimeError):
"""A candidate registry is malformed or drifts from a RunDefinition."""
class PortableWorkerRuntimeUnavailableError(PortableWorkerRuntimeError):
"""A candidate is not a complete, sealed, locally admitted executor."""
class PortableWorkerRuntimeJobRejectedError(PortableWorkerRuntimeError):
"""A Worker job differs from the exact local candidate identity."""
@dataclass(frozen=True, slots=True)
class PortableWorkerRuntimePhase:
phase_id: str
state: PhaseState
def __post_init__(self) -> None:
_pattern(self.phase_id, _IDENTIFIER, "runtime phase id")
if self.state not in ("implemented", "missing"):
raise PortableWorkerRuntimeRegistryError("runtime phase state is invalid")
def as_dict(self) -> dict[str, str]:
return {"phase_id": self.phase_id, "state": self.state}
@dataclass(frozen=True, slots=True)
class PortableWorkerAssetRequirement:
"""One exact reusable local asset; its locator is intentionally absent."""
asset_id: str
kind: RuntimeAssetKind
sha256: str
byte_length: int | None
component_id: str | None
model_release_id: str | None
model_artifact_role: str | None
def __post_init__(self) -> None:
_pattern(self.asset_id, _ASSET_ID, "runtime asset id")
if self.kind not in (
"container-image",
"definition-component",
"local-file",
"model-artifact",
):
raise PortableWorkerRuntimeRegistryError("runtime asset kind is invalid")
_digest(self.sha256, "runtime asset sha256")
if self.byte_length is not None and (
isinstance(self.byte_length, bool)
or not isinstance(self.byte_length, int)
or self.byte_length < 1
):
raise PortableWorkerRuntimeRegistryError("runtime asset byte length is invalid")
if self.kind == "container-image":
if any(
value is not None
for value in (
self.byte_length,
self.component_id,
self.model_release_id,
self.model_artifact_role,
)
):
raise PortableWorkerRuntimeRegistryError(
"container image requirement cannot impersonate a definition asset"
)
elif self.kind == "definition-component":
_optional_identifier(self.component_id, "definition component id")
if self.component_id is None or any(
value is not None
for value in (self.model_release_id, self.model_artifact_role)
):
raise PortableWorkerRuntimeRegistryError(
"definition component requirement is incomplete"
)
elif self.kind == "model-artifact":
_optional_identifier(self.model_release_id, "model release id")
_optional_identifier(self.model_artifact_role, "model artifact role")
if (
self.model_release_id is None
or self.model_artifact_role is None
or self.component_id is not None
or self.byte_length is None
):
raise PortableWorkerRuntimeRegistryError(
"model artifact requirement is incomplete"
)
elif any(
value is not None
for value in (
self.component_id,
self.model_release_id,
self.model_artifact_role,
)
):
raise PortableWorkerRuntimeRegistryError(
"local file requirement cannot impersonate a definition asset"
)
def as_dict(self) -> dict[str, object]:
return {
"asset_id": self.asset_id,
"kind": self.kind,
"sha256": self.sha256,
"byte_length": self.byte_length,
"component_id": self.component_id,
"model_release_id": self.model_release_id,
"model_artifact_role": self.model_artifact_role,
}
@dataclass(frozen=True, slots=True)
class PortableWorkerExecutorSeal:
release_id: str
release_sha256: str
image_sha256: str
def __post_init__(self) -> None:
_pattern(self.release_id, _IDENTIFIER, "executor release id")
_digest(self.release_sha256, "executor release sha256")
_digest(self.image_sha256, "executor image sha256")
def as_dict(self) -> dict[str, str]:
return {
"release_id": self.release_id,
"release_sha256": self.release_sha256,
"image_sha256": self.image_sha256,
}
@dataclass(frozen=True, slots=True)
class PortableWorkerRuntimeCandidate:
adapter_id: str
setup_id: str
definition_id: str
definition_version: int
definition_sha256: str
source_adapter_sha256: str
model_manifest_sha256: str
resource_profile_sha256: str
result_contract_sha256: str
state: CandidateState
executor: PortableWorkerExecutorSeal | None
reusable_assets: tuple[PortableWorkerAssetRequirement, ...]
phases: tuple[PortableWorkerRuntimePhase, ...]
blockers: tuple[str, ...]
candidate_sha256: str
def __post_init__(self) -> None:
for value, label in (
(self.adapter_id, "runtime adapter id"),
(self.setup_id, "setup id"),
(self.definition_id, "definition id"),
):
_pattern(value, _IDENTIFIER, label)
if (
isinstance(self.definition_version, bool)
or not isinstance(self.definition_version, int)
or self.definition_version < 1
):
raise PortableWorkerRuntimeRegistryError("definition version is invalid")
for value, label in (
(self.definition_sha256, "definition sha256"),
(self.source_adapter_sha256, "source adapter sha256"),
(self.model_manifest_sha256, "model manifest sha256"),
(self.resource_profile_sha256, "resource profile sha256"),
(self.result_contract_sha256, "result contract sha256"),
(self.candidate_sha256, "runtime candidate sha256"),
):
_digest(value, label)
if self.state not in ("blocked", "ready"):
raise PortableWorkerRuntimeRegistryError("runtime candidate state is invalid")
asset_ids = [asset.asset_id for asset in self.reusable_assets]
if asset_ids != sorted(asset_ids) or len(asset_ids) != len(set(asset_ids)):
raise PortableWorkerRuntimeRegistryError(
"runtime assets must be unique and canonically ordered"
)
phase_ids = [phase.phase_id for phase in self.phases]
if not phase_ids or len(phase_ids) != len(set(phase_ids)):
raise PortableWorkerRuntimeRegistryError("runtime phases must be non-empty and unique")
if self.blockers != tuple(sorted(self.blockers)) or len(self.blockers) != len(
set(self.blockers)
):
raise PortableWorkerRuntimeRegistryError(
"runtime blockers must be unique and canonically ordered"
)
for blocker in self.blockers:
_pattern(blocker, _IDENTIFIER, "runtime blocker")
missing_phases = tuple(
phase.phase_id for phase in self.phases if phase.state == "missing"
)
if self.state == "ready":
if self.executor is None or self.blockers or missing_phases:
raise PortableWorkerRuntimeRegistryError(
"ready runtime requires a sealed executor and complete phases"
)
elif self.executor is not None or not self.blockers or not missing_phases:
raise PortableWorkerRuntimeRegistryError(
"blocked runtime must keep its executor unsealed and missing phases explicit"
)
if self.candidate_sha256 != canonical_sha256(self.identity_document()):
raise PortableWorkerRuntimeRegistryError("runtime candidate digest changed")
@property
def ready(self) -> bool:
return self.state == "ready"
def identity_document(self) -> dict[str, object]:
return {
"schema_version": PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA,
"adapter_id": self.adapter_id,
"setup_id": self.setup_id,
"definition_id": self.definition_id,
"definition_version": self.definition_version,
"definition_sha256": self.definition_sha256,
"source_adapter_sha256": self.source_adapter_sha256,
"model_manifest_sha256": self.model_manifest_sha256,
"resource_profile_sha256": self.resource_profile_sha256,
"result_contract_sha256": self.result_contract_sha256,
"state": self.state,
"executor": self.executor.as_dict() if self.executor is not None else None,
"reusable_assets": [asset.as_dict() for asset in self.reusable_assets],
"phases": [phase.as_dict() for phase in self.phases],
"blockers": list(self.blockers),
"authority": dict(_AUTHORITY),
}
def bind_definition(self, definition: PortableRunDefinition) -> None:
if (
definition.setup_id != self.setup_id
or definition.definition_id != self.definition_id
or definition.version != self.definition_version
or definition.definition_sha256 != self.definition_sha256
or definition.source_adapter.contract_sha256 != self.source_adapter_sha256
or definition.model_manifest_sha256 != self.model_manifest_sha256
or definition.resource_profile.profile_sha256 != self.resource_profile_sha256
or definition.result_contract.contract_sha256 != self.result_contract_sha256
):
raise PortableWorkerRuntimeRegistryError(
"runtime candidate and portable RunDefinition identities disagree"
)
components = {component.component_id: component for component in definition.components}
models = {model.release_id: model for model in definition.models}
for requirement in self.reusable_assets:
if requirement.kind == "definition-component":
component = components.get(requirement.component_id or "")
if component is None or component.sha256 != requirement.sha256:
raise PortableWorkerRuntimeRegistryError(
"runtime component requirement differs from its RunDefinition"
)
elif requirement.kind == "model-artifact":
model = models.get(requirement.model_release_id or "")
artifacts = {
artifact.role: artifact for artifact in model.artifacts
} if model is not None else {}
artifact = artifacts.get(requirement.model_artifact_role or "")
if (
artifact is None
or artifact.sha256 != requirement.sha256
or artifact.byte_length != requirement.byte_length
):
raise PortableWorkerRuntimeRegistryError(
"runtime model artifact differs from its RunDefinition"
)
if self.state == "blocked":
if definition.executor.ready:
raise PortableWorkerRuntimeRegistryError(
"blocked local runtime cannot bind a ready RunDefinition"
)
return
if not definition.executor.ready or self.executor is None:
raise PortableWorkerRuntimeRegistryError(
"ready local runtime requires a ready RunDefinition"
)
if (
definition.executor.release_id != self.executor.release_id
or definition.executor.release_sha256 != self.executor.release_sha256
or definition.executor.image_sha256 != self.executor.image_sha256
):
raise PortableWorkerRuntimeRegistryError(
"local executor seal differs from its RunDefinition"
)
def executor_identity(self) -> ObservatoryWorkerExecutorIdentity:
if not self.ready or self.executor is None:
raise PortableWorkerRuntimeUnavailableError(
"blocked runtime candidate has no executor identity"
)
return ObservatoryWorkerExecutorIdentity(
release_sha256=self.executor.release_sha256,
image_sha256=self.executor.image_sha256,
model_manifest_sha256=self.model_manifest_sha256,
resource_profile_sha256=self.resource_profile_sha256,
)
@dataclass(frozen=True, slots=True)
class PortableWorkerRuntimeRegistry:
candidates: tuple[PortableWorkerRuntimeCandidate, ...]
def __post_init__(self) -> None:
if not self.candidates:
raise PortableWorkerRuntimeRegistryError("runtime registry cannot be empty")
for label, values in (
("runtime adapter ids", [candidate.adapter_id for candidate in self.candidates]),
("runtime setup ids", [candidate.setup_id for candidate in self.candidates]),
(
"runtime candidate digests",
[candidate.candidate_sha256 for candidate in self.candidates],
),
):
if len(values) != len(set(values)):
raise PortableWorkerRuntimeRegistryError(f"{label} must be unique")
@classmethod
def from_file(
cls,
path: Path,
*,
definitions: PortableRunDefinitionRegistry,
) -> PortableWorkerRuntimeRegistry:
candidate_path = path.expanduser().absolute()
if candidate_path.is_symlink() or not candidate_path.is_file():
raise PortableWorkerRuntimeRegistryError(
"runtime registry must be a regular file"
)
try:
if candidate_path.stat().st_size > _MAX_REGISTRY_BYTES:
raise PortableWorkerRuntimeRegistryError("runtime registry is too large")
document: object = json.loads(candidate_path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise PortableWorkerRuntimeRegistryError("runtime registry is unreadable") from exc
_reject_unsafe_keys(document)
root = _object(document, "runtime registry")
_exact_keys(root, {"schema_version", "candidates"}, "runtime registry")
if root["schema_version"] != PORTABLE_WORKER_RUNTIME_REGISTRY_SCHEMA:
raise PortableWorkerRuntimeRegistryError("runtime registry schema is invalid")
rows = _array(root["candidates"], "runtime candidates")
registry = cls(tuple(_candidate(row) for row in rows))
for candidate in registry.candidates:
candidate.bind_definition(
definitions.resolve(candidate.setup_id, candidate.definition_sha256)
)
return registry
def resolve(self, setup_id: str, definition_sha256: str) -> PortableWorkerRuntimeCandidate:
_pattern(setup_id, _IDENTIFIER, "setup id")
_digest(definition_sha256, "definition sha256")
for candidate in self.candidates:
if (
candidate.setup_id == setup_id
and candidate.definition_sha256 == definition_sha256
):
return candidate
raise PortableWorkerRuntimeRegistryError(
"portable Worker candidate identity is not allowlisted"
)
@dataclass(frozen=True, slots=True)
class PortableWorkerLocalAssetBinding:
"""Worker-local binding populated by reviewed local configuration only."""
asset_id: str
file_path: Path | None = None
image_sha256: str | None = None
def __post_init__(self) -> None:
_pattern(self.asset_id, _ASSET_ID, "local asset id")
if (self.file_path is None) == (self.image_sha256 is None):
raise ValueError("local asset binding must select exactly one local locator")
if self.image_sha256 is not None:
_digest(self.image_sha256, "local image sha256")
@dataclass(frozen=True, slots=True)
class PortableWorkerAssetVerification:
asset_id: str
state: AssetVerificationState
reason: str | None
def __post_init__(self) -> None:
_pattern(self.asset_id, _ASSET_ID, "verified asset id")
if self.state not in ("matched", "missing", "mismatched"):
raise ValueError("verified asset state is invalid")
if (self.state == "matched") != (self.reason is None):
raise ValueError("verified asset reason disagrees with its state")
def verify_local_assets(
candidate: PortableWorkerRuntimeCandidate,
bindings: Mapping[str, PortableWorkerLocalAssetBinding],
) -> tuple[PortableWorkerAssetVerification, ...]:
"""Hash locally bound assets without accepting locators from a job."""
checks: list[PortableWorkerAssetVerification] = []
for requirement in candidate.reusable_assets:
binding = bindings.get(requirement.asset_id)
if binding is None or binding.asset_id != requirement.asset_id:
checks.append(
PortableWorkerAssetVerification(requirement.asset_id, "missing", "not-bound")
)
continue
if requirement.kind == "container-image":
matched = binding.file_path is None and binding.image_sha256 == requirement.sha256
else:
matched = _matches_file(requirement, binding.file_path)
checks.append(
PortableWorkerAssetVerification(
requirement.asset_id,
"matched" if matched else "mismatched",
None if matched else "identity-mismatch",
)
)
return tuple(checks)
@dataclass(frozen=True, slots=True)
class PortableWorkerRuntimeAdmission:
candidate_sha256: str
ready: bool
blockers: tuple[str, ...]
assets: tuple[PortableWorkerAssetVerification, ...]
def __post_init__(self) -> None:
_digest(self.candidate_sha256, "runtime admission candidate sha256")
if self.blockers != tuple(sorted(self.blockers)) or len(self.blockers) != len(
set(self.blockers)
):
raise ValueError("runtime admission blockers must be canonical")
if self.ready and (
self.blockers or any(item.state != "matched" for item in self.assets)
):
raise ValueError("ready runtime admission cannot contain an unresolved asset")
def inspect_runtime_candidate(
candidate: PortableWorkerRuntimeCandidate,
bindings: Mapping[str, PortableWorkerLocalAssetBinding],
) -> PortableWorkerRuntimeAdmission:
assets = verify_local_assets(candidate, bindings)
asset_blockers = tuple(
sorted(f"asset-{item.asset_id}-{item.state}" for item in assets if item.state != "matched")
)
blockers = tuple(sorted((*candidate.blockers, *asset_blockers)))
return PortableWorkerRuntimeAdmission(
candidate_sha256=candidate.candidate_sha256,
ready=candidate.ready and not blockers,
blockers=blockers,
assets=assets,
)
@dataclass(frozen=True, slots=True)
class PortableWorkerSourceStage:
"""A locally materialized source; transport/materialization owns its path."""
root: Path
source_bundle_sha256: str
source_capability_manifest_sha256: str
source_adapter_sha256: str
def __post_init__(self) -> None:
for value, label in (
(self.source_bundle_sha256, "source bundle sha256"),
(self.source_capability_manifest_sha256, "source capability sha256"),
(self.source_adapter_sha256, "source adapter sha256"),
):
_digest(value, label)
if self.root.is_symlink() or not self.root.is_dir():
raise PortableWorkerRuntimeUnavailableError(
"portable source stage must be a real local directory"
)
@dataclass(frozen=True, slots=True)
class PortableWorkerResultDraft:
root: Path
result_id: str
result_sha256: str
result_contract_sha256: str
def __post_init__(self) -> None:
_pattern(self.result_id, _SESSION_ID, "result id")
_digest(self.result_sha256, "result sha256")
_digest(self.result_contract_sha256, "result contract sha256")
if self.root.is_symlink() or not self.root.is_dir():
raise PortableWorkerRuntimeUnavailableError(
"portable result draft must be a real local directory"
)
@dataclass(frozen=True, slots=True)
class PortableWorkerRuntimePlan:
job_id: str
adapter_id: str
candidate_sha256: str
setup_id: str
definition_sha256: str
source_bundle_sha256: str
source_capability_manifest_sha256: str
result_contract_sha256: str
phases: tuple[str, ...]
def __post_init__(self) -> None:
_pattern(self.job_id, _JOB_ID, "runtime plan job id")
for value, label in (
(self.adapter_id, "runtime plan adapter id"),
(self.setup_id, "runtime plan setup id"),
):
_pattern(value, _IDENTIFIER, label)
for value, label in (
(self.candidate_sha256, "runtime plan candidate sha256"),
(self.definition_sha256, "runtime plan definition sha256"),
(self.source_bundle_sha256, "runtime plan source bundle sha256"),
(
self.source_capability_manifest_sha256,
"runtime plan source capability sha256",
),
(self.result_contract_sha256, "runtime plan result contract sha256"),
):
_digest(value, label)
if not self.phases or len(self.phases) != len(set(self.phases)):
raise PortableWorkerRuntimeRegistryError(
"runtime plan phases must be non-empty and unique"
)
for phase in self.phases:
_pattern(phase, _IDENTIFIER, "runtime plan phase id")
def as_dict(self) -> dict[str, object]:
return {
"schema_version": PORTABLE_WORKER_RUNTIME_PLAN_SCHEMA,
"job_id": self.job_id,
"adapter_id": self.adapter_id,
"candidate_sha256": self.candidate_sha256,
"setup_id": self.setup_id,
"definition_sha256": self.definition_sha256,
"source_bundle_sha256": self.source_bundle_sha256,
"source_capability_manifest_sha256": self.source_capability_manifest_sha256,
"result_contract_sha256": self.result_contract_sha256,
"phases": list(self.phases),
"authority": dict(_AUTHORITY),
}
class PortableWorkerSourceMaterializer(Protocol):
def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage: ...
class PortableWorkerProfileRunner(Protocol):
def run(
self,
plan: PortableWorkerRuntimePlan,
source: PortableWorkerSourceStage,
) -> PortableWorkerResultDraft: ...
class PortableWorkerResultPublisher(Protocol):
def publish(
self,
job: SealedObservatoryRecordedJob,
draft: PortableWorkerResultDraft,
) -> ObservatoryWorkerExecutionResult: ...
@dataclass(frozen=True, slots=True)
class PortableWorkerExecutorAdapter:
"""Local adapter composition; none of its dependencies come from a job."""
candidate: PortableWorkerRuntimeCandidate
definition: PortableRunDefinition
admission: PortableWorkerRuntimeAdmission
source_materializer: PortableWorkerSourceMaterializer
runner: PortableWorkerProfileRunner
publisher: PortableWorkerResultPublisher
def __post_init__(self) -> None:
self.candidate.bind_definition(self.definition)
if not self.candidate.ready or not self.admission.ready:
raise PortableWorkerRuntimeUnavailableError(
"portable Worker adapter cannot bind a blocked runtime candidate"
)
if self.admission.candidate_sha256 != self.candidate.candidate_sha256:
raise PortableWorkerRuntimeUnavailableError(
"runtime admission belongs to another candidate"
)
expected_assets = tuple(
requirement.asset_id for requirement in self.candidate.reusable_assets
)
admitted_assets = tuple(item.asset_id for item in self.admission.assets)
if admitted_assets != expected_assets or any(
item.state != "matched" for item in self.admission.assets
):
raise PortableWorkerRuntimeUnavailableError(
"runtime admission does not prove every candidate asset"
)
def execute(
self,
job: SealedObservatoryRecordedJob,
) -> ObservatoryWorkerExecutionResult:
self._verify_job(job)
source = self.source_materializer.materialize(job)
if (
source.source_bundle_sha256 != job.source_bundle_sha256
or source.source_capability_manifest_sha256
!= job.source_capability_manifest_sha256
or source.source_adapter_sha256 != job.source_adapter_sha256
):
raise PortableWorkerRuntimeJobRejectedError(
"materialized source differs from the sealed job"
)
plan = PortableWorkerRuntimePlan(
job_id=job.job_id,
adapter_id=self.candidate.adapter_id,
candidate_sha256=self.candidate.candidate_sha256,
setup_id=job.setup_id,
definition_sha256=job.definition_sha256,
source_bundle_sha256=job.source_bundle_sha256,
source_capability_manifest_sha256=job.source_capability_manifest_sha256,
result_contract_sha256=self.candidate.result_contract_sha256,
phases=tuple(phase.phase_id for phase in self.candidate.phases),
)
draft = self.runner.run(plan, source)
if draft.result_contract_sha256 != self.candidate.result_contract_sha256:
raise PortableWorkerRuntimeJobRejectedError(
"runtime result uses another result contract"
)
published = self.publisher.publish(job, draft)
if (
published.result_id != draft.result_id
or published.result_sha256 != draft.result_sha256
):
raise PortableWorkerRuntimeJobRejectedError(
"publisher receipt differs from the validated result draft"
)
return published
def _verify_job(self, job: SealedObservatoryRecordedJob) -> None:
executor = self.candidate.executor
expected_identity = self.candidate.executor_identity()
if (
executor is None
or job.setup_id != self.definition.setup_id
or job.definition_id != self.definition.definition_id
or job.definition_version != self.definition.version
or job.definition_sha256 != self.definition.definition_sha256
or job.source_adapter_id != self.definition.source_adapter.adapter_id
or job.source_adapter_version != self.definition.source_adapter.version
or job.source_adapter_sha256 != self.definition.source_adapter.contract_sha256
or job.executor_release_id != executor.release_id
or job.executor_identity != expected_identity
or job.model_release_ids != self.definition.learned_models
or job.resource_profile_id != self.definition.resource_profile.profile_id
or job.checkpoint_policy != self.definition.resource_profile.checkpoint_policy
or job.allowed_checkpoints != self.definition.resource_profile.allowed_checkpoints
):
raise PortableWorkerRuntimeJobRejectedError(
"Worker job differs from the exact local runtime identity"
)
def _matches_file(
requirement: PortableWorkerAssetRequirement,
path: Path | None,
) -> bool:
if path is None:
return False
candidate = path.expanduser().absolute()
try:
if candidate.is_symlink() or not candidate.is_file():
return False
if requirement.byte_length is not None and candidate.stat().st_size != (
requirement.byte_length
):
return False
digest = hashlib.sha256()
with candidate.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest() == requirement.sha256
except OSError:
return False
def _candidate(value: object) -> PortableWorkerRuntimeCandidate:
row = _object(value, "runtime candidate")
_exact_keys(
row,
{
"schema_version",
"adapter_id",
"setup_id",
"definition_id",
"definition_version",
"definition_sha256",
"source_adapter_sha256",
"model_manifest_sha256",
"resource_profile_sha256",
"result_contract_sha256",
"state",
"executor",
"reusable_assets",
"phases",
"blockers",
"authority",
"candidate_sha256",
},
"runtime candidate",
)
if row["schema_version"] != PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA:
raise PortableWorkerRuntimeRegistryError("runtime candidate schema is invalid")
if row["authority"] != _AUTHORITY:
raise PortableWorkerRuntimeRegistryError(
"runtime candidate authority must remain observation-only"
)
state = row["state"]
if state not in ("blocked", "ready"):
raise PortableWorkerRuntimeRegistryError("runtime candidate state is invalid")
executor_row = row["executor"]
executor = None if executor_row is None else _executor(executor_row)
assets = _array(row["reusable_assets"], "runtime reusable assets")
phases = _array(row["phases"], "runtime phases")
blockers = _array(row["blockers"], "runtime blockers")
return PortableWorkerRuntimeCandidate(
adapter_id=_string(row["adapter_id"], "runtime adapter id"),
setup_id=_string(row["setup_id"], "setup id"),
definition_id=_string(row["definition_id"], "definition id"),
definition_version=_integer(row["definition_version"], "definition version"),
definition_sha256=_string(row["definition_sha256"], "definition sha256"),
source_adapter_sha256=_string(
row["source_adapter_sha256"], "source adapter sha256"
),
model_manifest_sha256=_string(
row["model_manifest_sha256"], "model manifest sha256"
),
resource_profile_sha256=_string(
row["resource_profile_sha256"], "resource profile sha256"
),
result_contract_sha256=_string(
row["result_contract_sha256"], "result contract sha256"
),
state=state,
executor=executor,
reusable_assets=tuple(_asset(item) for item in assets),
phases=tuple(_phase(item) for item in phases),
blockers=tuple(_string(item, "runtime blocker") for item in blockers),
candidate_sha256=_string(row["candidate_sha256"], "runtime candidate sha256"),
)
def _asset(value: object) -> PortableWorkerAssetRequirement:
row = _object(value, "runtime asset")
_exact_keys(
row,
{
"asset_id",
"kind",
"sha256",
"byte_length",
"component_id",
"model_release_id",
"model_artifact_role",
},
"runtime asset",
)
kind = row["kind"]
if kind not in (
"container-image",
"definition-component",
"local-file",
"model-artifact",
):
raise PortableWorkerRuntimeRegistryError("runtime asset kind is invalid")
return PortableWorkerAssetRequirement(
asset_id=_string(row["asset_id"], "runtime asset id"),
kind=kind,
sha256=_string(row["sha256"], "runtime asset sha256"),
byte_length=_optional_integer(row["byte_length"], "runtime asset byte length"),
component_id=_optional_string(row["component_id"], "definition component id"),
model_release_id=_optional_string(row["model_release_id"], "model release id"),
model_artifact_role=_optional_string(
row["model_artifact_role"], "model artifact role"
),
)
def _phase(value: object) -> PortableWorkerRuntimePhase:
row = _object(value, "runtime phase")
_exact_keys(row, {"phase_id", "state"}, "runtime phase")
state = row["state"]
if state not in ("implemented", "missing"):
raise PortableWorkerRuntimeRegistryError("runtime phase state is invalid")
return PortableWorkerRuntimePhase(
phase_id=_string(row["phase_id"], "runtime phase id"),
state=state,
)
def _executor(value: object) -> PortableWorkerExecutorSeal:
row = _object(value, "runtime executor")
_exact_keys(
row,
{"release_id", "release_sha256", "image_sha256"},
"runtime executor",
)
return PortableWorkerExecutorSeal(
release_id=_string(row["release_id"], "executor release id"),
release_sha256=_string(row["release_sha256"], "executor release sha256"),
image_sha256=_string(row["image_sha256"], "executor image sha256"),
)
def _reject_unsafe_keys(value: object, *, parent: str = "registry") -> None:
"""Keep executable instructions and local locators out of shared config."""
if isinstance(value, dict):
for key, child in value.items():
if not isinstance(key, str):
raise PortableWorkerRuntimeRegistryError(
"runtime registry object keys must be strings"
)
normalized = key.lower().replace("-", "_")
if (
normalized == "path"
or normalized.endswith("_path")
or normalized in {"command", "commands", "argv", "env", "environment"}
or normalized.startswith("command_")
or normalized.endswith("_command")
or "priority" in normalized
):
raise PortableWorkerRuntimeRegistryError(
f"runtime registry forbids {key!r} in {parent}"
)
_reject_unsafe_keys(child, parent=key)
elif isinstance(value, list):
for child in value:
_reject_unsafe_keys(child, parent=parent)
def _object(value: object, label: str) -> dict[str, object]:
if not isinstance(value, dict) or not all(isinstance(key, str) for key in value):
raise PortableWorkerRuntimeRegistryError(f"{label} must be an object")
return value
def _array(value: object, label: str) -> list[object]:
if not isinstance(value, list):
raise PortableWorkerRuntimeRegistryError(f"{label} must be an array")
return value
def _exact_keys(row: Mapping[str, object], expected: set[str], label: str) -> None:
if set(row) != expected:
raise PortableWorkerRuntimeRegistryError(f"{label} fields are invalid")
def _string(value: object, label: str) -> str:
if not isinstance(value, str):
raise PortableWorkerRuntimeRegistryError(f"{label} must be a string")
return value
def _optional_string(value: object, label: str) -> str | None:
if value is None:
return None
return _string(value, label)
def _integer(value: object, label: str) -> int:
if isinstance(value, bool) or not isinstance(value, int):
raise PortableWorkerRuntimeRegistryError(f"{label} must be an integer")
return value
def _optional_integer(value: object, label: str) -> int | None:
if value is None:
return None
return _integer(value, label)
def _pattern(value: str, pattern: re.Pattern[str], label: str) -> None:
if pattern.fullmatch(value) is None:
raise PortableWorkerRuntimeRegistryError(f"{label} is invalid")
def _optional_identifier(value: str | None, label: str) -> None:
if value is not None:
_pattern(value, _IDENTIFIER, label)
def _digest(value: object, label: str) -> None:
if not isinstance(value, str) or _SHA256.fullmatch(value) is None:
raise PortableWorkerRuntimeRegistryError(f"{label} is invalid")
+436 -22
View File
@@ -45,6 +45,9 @@ MAX_LIVE_LEASES: Final = 10_000
MAX_RECORDED_JOB_STORAGE_BYTES: Final = 128 * 1024 * 1024
RECORDED_JOB_SQLITE_LOCK_TIMEOUT_SECONDS: Final = 0.1
_SQLITE_BUSY_TIMEOUT_MILLISECONDS: Final = 100
DEFAULT_RECORDED_CLAIM_LEASE_SECONDS: Final = 120
MIN_RECORDED_CLAIM_LEASE_SECONDS: Final = 5
MAX_RECORDED_CLAIM_LEASE_SECONDS: Final = 3_600
LIVE_K1_PRIORITY_RANK: Final = 0
RECORDED_PRIORITY_RANK: Final = 100
@@ -120,6 +123,11 @@ CREATE TABLE IF NOT EXISTS observatory_recorded_jobs (
claim_generation INTEGER NOT NULL CHECK (claim_generation >= 0),
active_claim_token TEXT,
active_claimant_id TEXT,
claimed_at_utc TEXT,
claim_expires_at_utc TEXT,
claim_heartbeat_at_utc TEXT,
claim_renewal_count INTEGER NOT NULL DEFAULT 0
CHECK (claim_renewal_count >= 0),
last_checkpoint_id TEXT,
restart_from_zero INTEGER NOT NULL CHECK (restart_from_zero IN (0, 1)),
preemption_receipt_sha256 TEXT,
@@ -386,6 +394,10 @@ class ObservatoryRecordedJob:
claim_generation: int
active_claim_token: str | None
active_claimant_id: str | None
claimed_at_utc: str | None
claim_expires_at_utc: str | None
claim_heartbeat_at_utc: str | None
claim_renewal_count: int
last_checkpoint_id: str | None
restart_from_zero: bool
preemption_receipt_sha256: str | None
@@ -461,6 +473,46 @@ class ObservatoryRecordedJob:
raise ObservatoryRecordedQueueIntegrityError("recorded-job claim generation is invalid")
_validate_optional_pattern(self.active_claim_token, _TOKEN, "active claim token")
_validate_optional_pattern(self.active_claimant_id, _IDENTIFIER, "claimant id")
lease_values = (
self.claimed_at_utc,
self.claim_expires_at_utc,
self.claim_heartbeat_at_utc,
)
if (self.active_claim_token is None) != (self.active_claimant_id is None):
raise ObservatoryRecordedQueueIntegrityError(
"recorded-job claim ownership is partial"
)
if self.active_claim_token is None:
if any(value is not None for value in lease_values) or self.claim_renewal_count != 0:
raise ObservatoryRecordedQueueIntegrityError(
"inactive recorded-job claim retains lease state"
)
else:
if self.state not in {"claimed", "running", "paused", "preemption-pending"}:
raise ObservatoryRecordedQueueIntegrityError(
"recorded-job claim is active outside an owned state"
)
if self.claim_generation < 1 or any(value is None for value in lease_values):
raise ObservatoryRecordedQueueIntegrityError(
"active recorded-job claim has no complete lease"
)
if self.claim_renewal_count < 0:
raise ObservatoryRecordedQueueIntegrityError(
"recorded-job claim renewal count is invalid"
)
assert self.claimed_at_utc is not None
assert self.claim_expires_at_utc is not None
assert self.claim_heartbeat_at_utc is not None
claimed_at = _parse_timestamp(self.claimed_at_utc, "claim timestamp")
expires_at = _parse_timestamp(self.claim_expires_at_utc, "claim expiry")
heartbeat_at = _parse_timestamp(
self.claim_heartbeat_at_utc,
"claim heartbeat timestamp",
)
if not claimed_at <= heartbeat_at < expires_at:
raise ObservatoryRecordedQueueIntegrityError(
"recorded-job claim lease chronology is invalid"
)
_validate_optional_pattern(self.last_checkpoint_id, _CHECKPOINT_ID, "last checkpoint id")
if not isinstance(self.restart_from_zero, bool):
raise ObservatoryRecordedQueueIntegrityError("recorded-job restart marker is invalid")
@@ -532,6 +584,16 @@ class ObservatoryRecordedJob:
"restart_from_zero": self.restart_from_zero,
"preemption_receipt_sha256": self.preemption_receipt_sha256,
"claim_generation": self.claim_generation,
"claim_lease": (
None
if self.active_claim_token is None
else {
"claimed_at_utc": self.claimed_at_utc,
"expires_at_utc": self.claim_expires_at_utc,
"heartbeat_at_utc": self.claim_heartbeat_at_utc,
"renewal_count": self.claim_renewal_count,
}
),
"result": (
None
if self.result_id is None
@@ -794,10 +856,12 @@ class ObservatoryRecordedJobQueue:
max_jobs: int = MAX_RECORDED_JOBS,
max_claim_receipts: int = MAX_RECORDED_CLAIM_RECEIPTS,
max_live_leases: int = MAX_LIVE_LEASES,
claim_lease_seconds: int = DEFAULT_RECORDED_CLAIM_LEASE_SECONDS,
) -> None:
_validate_quota(max_jobs, MAX_RECORDED_JOBS, "recorded job")
_validate_quota(max_claim_receipts, MAX_RECORDED_CLAIM_RECEIPTS, "claim receipt")
_validate_quota(max_live_leases, MAX_LIVE_LEASES, "live lease")
_validate_claim_lease_seconds(claim_lease_seconds)
self.data_dir = data_dir.expanduser().resolve()
self.database_path = self.data_dir / RECORDED_JOB_DATABASE_NAME
self._definitions = definitions
@@ -806,6 +870,7 @@ class ObservatoryRecordedJobQueue:
self._max_jobs = max_jobs
self._max_claim_receipts = max_claim_receipts
self._max_live_leases = max_live_leases
self._claim_lease_seconds = claim_lease_seconds
self._lock = threading.RLock()
self._initialize()
@@ -946,6 +1011,8 @@ class ObservatoryRecordedJobQueue:
_validate_pattern(claim_request_id, _IDEMPOTENCY_KEY, "claim request id")
request_sha256 = _claim_request_sha256(claimant_id, claim_request_id)
with self._transaction() as connection:
now = self._timestamp()
self._recover_stale_claims(connection, now=now)
receipt = connection.execute(
"SELECT * FROM observatory_recorded_claim_receipts WHERE claim_request_id = ?",
(claim_request_id,),
@@ -976,7 +1043,6 @@ class ObservatoryRecordedJobQueue:
"WHERE state = 'queued' "
"ORDER BY priority_rank, created_at_utc, job_id LIMIT 1"
).fetchone()
now = self._timestamp()
if row is None:
connection.execute(
"INSERT INTO observatory_recorded_claim_receipts "
@@ -989,12 +1055,26 @@ class ObservatoryRecordedJobQueue:
claim_token = hashlib.sha256(
f"{uuid4().hex}:{job_id}:{claim_request_id}".encode()
).hexdigest()
claim_expires_at = _timestamp_after_seconds(
now,
self._claim_lease_seconds,
)
updated = connection.execute(
"UPDATE observatory_recorded_jobs SET state = 'claimed', "
"claim_generation = claim_generation + 1, active_claim_token = ?, "
"active_claimant_id = ?, preemption_requested = 0, "
"active_claimant_id = ?, claimed_at_utc = ?, "
"claim_expires_at_utc = ?, claim_heartbeat_at_utc = ?, "
"claim_renewal_count = 0, preemption_requested = 0, "
"updated_at_utc = ? WHERE job_id = ? AND state = 'queued'",
(claim_token, claimant_id, now, job_id),
(
claim_token,
claimant_id,
now,
claim_expires_at,
now,
now,
job_id,
),
)
if updated.rowcount != 1:
raise ObservatoryRecordedQueueIntegrityError(
@@ -1021,12 +1101,125 @@ class ObservatoryRecordedJobQueue:
job=self._get_job(connection, job_id),
)
def renew_claim(
self,
job_id: str,
*,
claim_token: str,
claim_generation: int,
heartbeat_sequence: int,
) -> ObservatoryRecordedJob:
"""Renew one exact active claim using an idempotent heartbeat sequence."""
_validate_pattern(job_id, _JOB_ID, "recorded job id")
_validate_pattern(claim_token, _TOKEN, "claim token")
_validate_positive_int(claim_generation, "claim generation")
_validate_positive_int(heartbeat_sequence, "heartbeat sequence")
self.recover_stale_claims()
with self._transaction() as connection:
job = self._get_job(connection, job_id)
now = self._timestamp()
self._require_active_claim(job, claim_token, now=now)
if job.claim_generation != claim_generation:
raise ObservatoryRecordedQueueStaleClaimError(
"recorded-job claim generation is stale"
)
if job.state not in {"claimed", "running", "paused", "preemption-pending"}:
raise ObservatoryRecordedQueueConflictError(
f"cannot renew recorded-job claim from {job.state}"
)
if heartbeat_sequence == job.claim_renewal_count:
return job
if heartbeat_sequence != job.claim_renewal_count + 1:
raise ObservatoryRecordedQueueConflictError(
"recorded-job heartbeat sequence is not contiguous"
)
expires_at = _timestamp_after_seconds(now, self._claim_lease_seconds)
connection.execute(
"UPDATE observatory_recorded_jobs SET claim_expires_at_utc = ?, "
"claim_heartbeat_at_utc = ?, claim_renewal_count = ?, "
"updated_at_utc = ? WHERE job_id = ? AND active_claim_token = ? "
"AND claim_generation = ?",
(
expires_at,
now,
heartbeat_sequence,
now,
job_id,
claim_token,
claim_generation,
),
)
return self._get_job(connection, job_id)
def authorize_claim_access(
self,
job_id: str,
*,
claim_token: str,
claim_generation: int,
claimant_id: str,
allowed_states: tuple[RecordedJobState, ...] = ("claimed", "running"),
) -> ObservatoryRecordedJob:
"""Authorize one bounded side-channel operation for the active lease.
Source downloads and result staging are deliberately not queue state
transitions, but they still must be fenced by the exact claimant,
token, generation, unexpired lease and an explicitly admitted queue
state. Keeping this check inside the queue transaction prevents an
artifact transport from reimplementing only part of claim semantics.
"""
_validate_pattern(job_id, _JOB_ID, "recorded job id")
_validate_pattern(claim_token, _TOKEN, "claim token")
_validate_positive_int(claim_generation, "claim generation")
_validate_pattern(claimant_id, _IDENTIFIER, "claimant id")
if (
not allowed_states
or len(set(allowed_states)) != len(allowed_states)
or any(state not in _recorded_job_states() for state in allowed_states)
):
raise ValueError("claim-access states are invalid")
self.recover_stale_claims()
with self._transaction() as connection:
job = self._get_job(connection, job_id)
self._require_active_claim(job, claim_token, now=self._timestamp())
if (
job.claim_generation != claim_generation
or job.active_claimant_id != claimant_id
):
raise ObservatoryRecordedQueueStaleClaimError(
"recorded-job claim ownership is stale"
)
if job.state not in allowed_states:
raise ObservatoryRecordedQueueConflictError(
f"claim-bound access is unavailable from {job.state}"
)
return job
def recover_stale_claims(self) -> tuple[ObservatoryRecordedJob, ...]:
"""Recover expired ownership without creating a second physical owner.
A never-started or durably paused claim is safe to requeue. Expired
running ownership is quarantined for scheduler reconciliation because
lease expiry alone does not prove that its Worker process stopped.
"""
with self._transaction() as connection:
recovered_ids = self._recover_stale_claims(
connection,
now=self._timestamp(),
)
return tuple(self._get_job(connection, job_id) for job_id in recovered_ids)
def start(self, job_id: str, *, claim_token: str) -> ObservatoryRecordedJob:
"""Enter running state, or yield before execution when live has priority."""
self.recover_stale_claims()
with self._transaction() as connection:
job = self._get_job(connection, job_id)
self._require_active_claim(job, claim_token)
now = self._timestamp()
self._require_active_claim(job, claim_token, now=now)
if job.state == "running":
return job
if job.state == "paused":
@@ -1039,7 +1232,7 @@ class ObservatoryRecordedJobQueue:
connection.execute(
"UPDATE observatory_recorded_jobs SET state = ?, "
"preemption_requested = ?, updated_at_utc = ? WHERE job_id = ?",
(state, int(state == "paused"), self._timestamp(), job_id),
(state, int(state == "paused"), now, job_id),
)
return self._get_job(connection, job_id)
@@ -1053,9 +1246,11 @@ class ObservatoryRecordedJobQueue:
"""Record an allowlisted cooperative boundary and yield if live is open."""
_validate_pattern(checkpoint_id, _CHECKPOINT_ID, "checkpoint id")
self.recover_stale_claims()
with self._transaction() as connection:
job = self._get_job(connection, job_id)
self._require_active_claim(job, claim_token)
now = self._timestamp()
self._require_active_claim(job, claim_token, now=now)
if job.checkpoint_policy != "cooperative":
raise ObservatoryRecordedCheckpointError(
"recorded RunDefinition is non-checkpointable"
@@ -1079,7 +1274,7 @@ class ObservatoryRecordedJobQueue:
connection.execute(
"UPDATE observatory_recorded_jobs SET state = ?, "
"last_checkpoint_id = ?, updated_at_utc = ? WHERE job_id = ?",
(state, checkpoint_id, self._timestamp(), job_id),
(state, checkpoint_id, now, job_id),
)
return self._get_job(connection, job_id)
@@ -1195,6 +1390,7 @@ class ObservatoryRecordedJobQueue:
def request_live(self, intent: ObservatoryLiveLeaseIntent) -> tuple[ObservatoryLiveLease, bool]:
"""Close recorded admission without allowing a monolith to delay live K1."""
self.recover_stale_claims()
created = False
with self._transaction() as connection:
existing = connection.execute(
@@ -1341,6 +1537,7 @@ class ObservatoryRecordedJobQueue:
"""Activate only after every recorded resource owner has yielded."""
_validate_pattern(lease_id, _LEASE_ID, "live lease id")
self.recover_stale_claims()
with self._transaction() as connection:
lease = self._get_live_lease(connection, lease_id)
if lease.state == "active":
@@ -1438,7 +1635,9 @@ class ObservatoryRecordedJobQueue:
connection.execute(
"UPDATE observatory_recorded_jobs SET state = 'queued', "
"preemption_requested = 0, active_claim_token = NULL, "
"active_claimant_id = NULL, updated_at_utc = ? "
"active_claimant_id = NULL, claimed_at_utc = NULL, "
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
"claim_renewal_count = 0, updated_at_utc = ? "
"WHERE state = 'paused'",
(now,),
)
@@ -1488,22 +1687,34 @@ class ObservatoryRecordedJobQueue:
_validate_pattern(job_id, _JOB_ID, "recorded job id")
_validate_pattern(claim_token, _TOKEN, "claim token")
token_sha256 = hashlib.sha256(claim_token.encode()).hexdigest()
self.recover_stale_claims()
with self._transaction() as connection:
job = self._get_job(connection, job_id)
if job.state in _TERMINAL_STATES:
exact_replay = (
job.state == state
and job.result_id == result_id
and job.result_sha256 == result_sha256
and job.terminal_code == terminal_code
and job.terminal_message == terminal_message
and job.terminal_claim_token_sha256 == token_sha256
)
if exact_replay:
return job
if (
job.state != state
or job.result_id != result_id
or job.result_sha256 != result_sha256
or job.terminal_code != terminal_code
or job.terminal_message != terminal_message
or job.terminal_claim_token_sha256 != token_sha256
job.terminal_claim_token_sha256 != token_sha256
or job.terminal_code
in {"claim-lease-expired", "claim-lease-migration"}
):
raise ObservatoryRecordedQueueStaleClaimError(
"recorded-job terminal acknowledgement is stale"
)
else:
raise ObservatoryRecordedQueueConflictError(
"recorded job is bound to another terminal outcome"
)
return job
self._require_active_claim(job, claim_token)
now = self._timestamp()
self._require_active_claim(job, claim_token, now=now)
allowed_states = (
("running",)
if state == "succeeded"
@@ -1523,7 +1734,9 @@ class ObservatoryRecordedJobQueue:
"UPDATE observatory_recorded_jobs SET state = ?, result_id = ?, "
"result_sha256 = ?, terminal_code = ?, terminal_message = ?, "
"terminal_claim_token_sha256 = ?, active_claim_token = NULL, "
"active_claimant_id = NULL, updated_at_utc = ? WHERE job_id = ?",
"active_claimant_id = NULL, claimed_at_utc = NULL, "
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
"claim_renewal_count = 0, updated_at_utc = ? WHERE job_id = ?",
(
state,
result_id,
@@ -1531,7 +1744,7 @@ class ObservatoryRecordedJobQueue:
terminal_code,
terminal_message,
token_sha256,
self._timestamp(),
now,
job_id,
),
)
@@ -1548,18 +1761,105 @@ class ObservatoryRecordedJobQueue:
raise ObservatoryRecordedQueueIntegrityError(
"stored claim receipt has a partial job identity"
)
job = self._get_job(connection, job_id)
if job.active_claim_token != claim_token:
raise ObservatoryRecordedQueueStaleClaimError(
"recorded-job claim receipt no longer owns the job"
)
return ObservatoryRecordedClaim(
claim_request_id=str(receipt["claim_request_id"]),
request_sha256=str(receipt["request_sha256"]),
claimant_id=str(receipt["claimant_id"]),
claim_token=claim_token,
job=self._get_job(connection, job_id),
job=job,
)
def _require_active_claim(self, job: ObservatoryRecordedJob, claim_token: str) -> None:
def _require_active_claim(
self,
job: ObservatoryRecordedJob,
claim_token: str,
*,
now: str | None = None,
) -> None:
_validate_pattern(claim_token, _TOKEN, "claim token")
if job.active_claim_token != claim_token:
raise ObservatoryRecordedQueueStaleClaimError("recorded-job claim token is stale")
if now is not None and (
job.claim_expires_at_utc is None
or _parse_timestamp(now, "queue timestamp")
>= _parse_timestamp(job.claim_expires_at_utc, "claim expiry")
):
raise ObservatoryRecordedQueueStaleClaimError(
"recorded-job claim lease expired"
)
def _recover_stale_claims(
self,
connection: sqlite3.Connection,
*,
now: str,
) -> tuple[str, ...]:
now_value = _parse_timestamp(now, "queue timestamp")
rows = connection.execute(
"SELECT * FROM observatory_recorded_jobs "
"WHERE active_claim_token IS NOT NULL "
"ORDER BY created_at_utc, job_id"
).fetchall()
recovered: list[str] = []
live_open = self._open_live_lease_row(connection) is not None
for row in rows:
job = _job_from_row(row)
if job.claim_expires_at_utc is None:
raise ObservatoryRecordedQueueIntegrityError(
"active recorded-job claim has no expiry"
)
if _parse_timestamp(job.claim_expires_at_utc, "claim expiry") > now_value:
continue
assert job.active_claim_token is not None
token_sha256 = hashlib.sha256(job.active_claim_token.encode()).hexdigest()
if job.state in {"claimed", "paused"}:
next_state = "paused" if job.state == "paused" and live_open else "queued"
restart_from_zero = job.restart_from_zero or job.state == "paused"
connection.execute(
"UPDATE observatory_recorded_jobs SET state = ?, "
"preemption_requested = ?, active_claim_token = NULL, "
"active_claimant_id = NULL, claimed_at_utc = NULL, "
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
"claim_renewal_count = 0, last_checkpoint_id = ?, "
"restart_from_zero = ?, updated_at_utc = ? WHERE job_id = ?",
(
next_state,
int(next_state == "paused"),
None if restart_from_zero else job.last_checkpoint_id,
int(restart_from_zero),
now,
job.job_id,
),
)
elif job.state in {"running", "preemption-pending"}:
connection.execute(
"UPDATE observatory_recorded_jobs "
"SET state = 'reconciliation-required', result_id = NULL, "
"result_sha256 = NULL, terminal_code = 'claim-lease-expired', "
"terminal_message = ?, terminal_claim_token_sha256 = ?, "
"active_claim_token = NULL, active_claimant_id = NULL, "
"claimed_at_utc = NULL, claim_expires_at_utc = NULL, "
"claim_heartbeat_at_utc = NULL, claim_renewal_count = 0, "
"updated_at_utc = ? WHERE job_id = ?",
(
"Worker claim lease expired after execution started; "
"physical resource ownership requires reconciliation.",
token_sha256,
now,
job.job_id,
),
)
else:
raise ObservatoryRecordedQueueIntegrityError(
"active recorded-job claim is stored in an invalid state"
)
recovered.append(job.job_id)
return tuple(recovered)
def _cancellation_request(
self,
@@ -1626,7 +1926,9 @@ class ObservatoryRecordedJobQueue:
connection.execute(
"UPDATE observatory_recorded_jobs SET state = 'paused', "
"preemption_requested = 1, active_claim_token = NULL, "
"active_claimant_id = NULL, last_checkpoint_id = NULL, "
"active_claimant_id = NULL, claimed_at_utc = NULL, "
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
"claim_renewal_count = 0, last_checkpoint_id = NULL, "
"restart_from_zero = 1, preemption_receipt_sha256 = ?, "
"updated_at_utc = ? WHERE job_id = ?",
(receipt.receipt_sha256, now, job.job_id),
@@ -1638,6 +1940,8 @@ class ObservatoryRecordedJobQueue:
"result_sha256 = NULL, terminal_code = 'preemption-race', "
"terminal_message = ?, terminal_claim_token_sha256 = ?, "
"active_claim_token = NULL, active_claimant_id = NULL, "
"claimed_at_utc = NULL, claim_expires_at_utc = NULL, "
"claim_heartbeat_at_utc = NULL, claim_renewal_count = 0, "
"preemption_receipt_sha256 = ?, updated_at_utc = ? "
"WHERE job_id = ?",
(
@@ -1707,6 +2011,7 @@ class ObservatoryRecordedJobQueue:
self.data_dir.chmod(0o700)
with self._connect() as connection:
connection.executescript(_SCHEMA_SQL)
self._migrate_claim_lease_schema(connection)
self._validate_schema(connection)
self._validate_existing_capacity(connection)
connection.commit()
@@ -1723,7 +2028,7 @@ class ObservatoryRecordedJobQueue:
def _validate_schema(self, connection: sqlite3.Connection) -> None:
expected = {
"observatory_recorded_jobs": 42,
"observatory_recorded_jobs": 46,
"observatory_recorded_claim_receipts": 6,
"observatory_live_leases": 13,
"observatory_recorded_preemptions": 14,
@@ -1734,6 +2039,21 @@ class ObservatoryRecordedJobQueue:
).fetchall()
if len(columns) != column_count:
raise ObservatoryRecordedQueueIntegrityError(f"{table} schema is incompatible")
job_columns = {
str(row["name"])
for row in connection.execute(
"SELECT name FROM pragma_table_info('observatory_recorded_jobs')"
).fetchall()
}
if not {
"claimed_at_utc",
"claim_expires_at_utc",
"claim_heartbeat_at_utc",
"claim_renewal_count",
}.issubset(job_columns):
raise ObservatoryRecordedQueueIntegrityError(
"recorded-job claim lease schema is unavailable"
)
indexes = connection.execute(
"SELECT name FROM sqlite_master WHERE type = 'index' "
"AND name = 'observatory_one_open_live_lease'"
@@ -1743,6 +2063,75 @@ class ObservatoryRecordedJobQueue:
"live K1 lease exclusivity index is unavailable"
)
def _migrate_claim_lease_schema(self, connection: sqlite3.Connection) -> None:
"""Add renewable claim columns and safely fence legacy active owners."""
columns = {
str(row["name"])
for row in connection.execute(
"SELECT name FROM pragma_table_info('observatory_recorded_jobs')"
).fetchall()
}
additions = (
("claimed_at_utc", "TEXT"),
("claim_expires_at_utc", "TEXT"),
("claim_heartbeat_at_utc", "TEXT"),
(
"claim_renewal_count",
"INTEGER NOT NULL DEFAULT 0 CHECK (claim_renewal_count >= 0)",
),
)
missing = [name for name, _definition in additions if name not in columns]
if not missing:
return
for name, definition in additions:
if name not in columns:
connection.execute(
f"ALTER TABLE observatory_recorded_jobs ADD COLUMN {name} {definition}"
)
now = self._timestamp()
rows = connection.execute(
"SELECT job_id, state, active_claim_token "
"FROM observatory_recorded_jobs WHERE active_claim_token IS NOT NULL"
).fetchall()
for row in rows:
job_id = str(row["job_id"])
state = str(row["state"])
claim_token = str(row["active_claim_token"])
if state in {"claimed", "paused"}:
connection.execute(
"UPDATE observatory_recorded_jobs SET state = 'queued', "
"preemption_requested = 0, active_claim_token = NULL, "
"active_claimant_id = NULL, claimed_at_utc = NULL, "
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
"claim_renewal_count = 0, last_checkpoint_id = NULL, "
"restart_from_zero = ?, updated_at_utc = ? WHERE job_id = ?",
(int(state == "paused"), now, job_id),
)
elif state in {"running", "preemption-pending"}:
connection.execute(
"UPDATE observatory_recorded_jobs "
"SET state = 'reconciliation-required', result_id = NULL, "
"result_sha256 = NULL, terminal_code = 'claim-lease-migration', "
"terminal_message = ?, terminal_claim_token_sha256 = ?, "
"active_claim_token = NULL, active_claimant_id = NULL, "
"claimed_at_utc = NULL, claim_expires_at_utc = NULL, "
"claim_heartbeat_at_utc = NULL, claim_renewal_count = 0, "
"updated_at_utc = ? WHERE job_id = ?",
(
"Legacy Worker execution had no expiring lease; physical "
"resource ownership requires reconciliation.",
hashlib.sha256(claim_token.encode()).hexdigest(),
now,
job_id,
),
)
else:
raise ObservatoryRecordedQueueIntegrityError(
"legacy active claim is stored in an invalid state"
)
def _validate_existing_capacity(self, connection: sqlite3.Connection) -> None:
for table, limit, label in (
("observatory_recorded_jobs", self._max_jobs, "recorded job"),
@@ -1901,6 +2290,10 @@ def _job_from_row(row: sqlite3.Row) -> ObservatoryRecordedJob:
claim_generation=row["claim_generation"],
active_claim_token=row["active_claim_token"],
active_claimant_id=row["active_claimant_id"],
claimed_at_utc=row["claimed_at_utc"],
claim_expires_at_utc=row["claim_expires_at_utc"],
claim_heartbeat_at_utc=row["claim_heartbeat_at_utc"],
claim_renewal_count=row["claim_renewal_count"],
last_checkpoint_id=row["last_checkpoint_id"],
restart_from_zero=bool(row["restart_from_zero"]),
preemption_receipt_sha256=row["preemption_receipt_sha256"],
@@ -2029,6 +2422,17 @@ def _validate_quota(value: object, maximum: int, label: str) -> None:
raise ValueError(f"{label} quota is invalid")
def _validate_claim_lease_seconds(value: object) -> None:
if (
not isinstance(value, int)
or isinstance(value, bool)
or not MIN_RECORDED_CLAIM_LEASE_SECONDS
<= value
<= MAX_RECORDED_CLAIM_LEASE_SECONDS
):
raise ValueError("recorded-job claim lease duration is invalid")
def _validate_positive_int(value: object, label: str) -> None:
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
raise ValueError(f"{label} must be positive")
@@ -2059,6 +2463,10 @@ def _validate_text(value: object, label: str, *, max_length: int) -> None:
def _validate_timestamp(value: object, label: str) -> None:
_parse_timestamp(value, label)
def _parse_timestamp(value: object, label: str) -> datetime:
_validate_text(value, label, max_length=64)
assert isinstance(value, str)
try:
@@ -2067,6 +2475,12 @@ def _validate_timestamp(value: object, label: str) -> None:
raise ValueError(f"{label} is invalid") from exc
if parsed.tzinfo is None or parsed.utcoffset() != timedelta(0) or not value.endswith("Z"):
raise ValueError(f"{label} must use UTC")
return parsed
def _timestamp_after_seconds(value: str, seconds: int) -> str:
expires_at = _parse_timestamp(value, "queue timestamp") + timedelta(seconds=seconds)
return expires_at.isoformat(timespec="milliseconds").replace("+00:00", "Z")
def _fsync_directory(path: Path) -> None:
+45 -3
View File
@@ -12,8 +12,11 @@ import json
import re
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Any, Final, Literal
from typing import Any, Final, Literal, cast
from k1link.observatory.canonical_result import (
is_admitted_observatory_recorded_result,
)
from k1link.sessions.models import SessionSummary
LABORATORY_SETUP_REGISTRY_SCHEMA: Final = (
@@ -22,6 +25,9 @@ LABORATORY_SETUP_REGISTRY_SCHEMA: Final = (
LABORATORY_SETUP_CATALOG_SCHEMA: Final = (
"missioncore.observatory-laboratory-setup-catalog/v1"
)
OBSERVATORY_CALCULATION_PROFILE_SCHEMA: Final = (
"missioncore.observatory-calculation-profile/v1"
)
_MAX_REGISTRY_BYTES: Final = 256 * 1024
_MAX_CONFIGURATION_BYTES: Final = 4 * 1024 * 1024
_IDENTIFIER: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
@@ -282,6 +288,42 @@ class LaboratorySetupRegistry:
return result.result_kind
return None
def observatory_calculation_profile(
self,
summary: SessionSummary,
) -> dict[str, object] | None:
"""Project an exact preserved legacy profile without inferring identity."""
for setup in self.setups:
for result in setup.preserved_results:
if (
result.access != "observatory"
or result.result_id != summary.session_id
):
continue
if (
setup.origin != "existing-result"
or setup.run_definition is not None
):
return None
if not is_admitted_observatory_recorded_result(
summary,
expected_result_id=result.result_id,
expected_source_session_id=setup.source_session_id,
expected_result_kind=result.result_kind,
):
return None
return {
"schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
"setup_id": setup.setup_id,
"display_name": setup.display_name,
"origin": setup.origin,
"definition_id": None,
"definition_version": None,
"definition_sha256": None,
}
return None
def _setup(value: object, *, repository_root: Path) -> LaboratorySetup:
row = _object(value, "setup")
@@ -330,7 +372,7 @@ def _setup(value: object, *, repository_root: Path) -> LaboratorySetup:
setup_id=_identifier(row["setup_id"], "setup_id"),
display_name=_text(row["display_name"], "display_name"),
description=_text(row["description"], "description"),
origin=origin,
origin=cast(SetupOrigin, origin),
source_session_id=_text(source["session_id"], "source session_id"),
source_label=_text(source["label"], "source label"),
required_modalities=modalities,
@@ -427,7 +469,7 @@ def _preserved_result(value: object) -> _PreservedResult:
result_id=result_id,
result_kind=_identifier(row["result_kind"], "result_kind"),
relation=_identifier(row["relation"], "result relation"),
access=access,
access=cast(ResultAccess, access),
created_at_utc=_text(row["created_at_utc"], "created_at_utc"),
)
+153 -4
View File
@@ -15,7 +15,7 @@ import re
import secrets
import stat
from contextlib import suppress
from dataclasses import dataclass
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Final
@@ -29,6 +29,7 @@ from k1link.sessions.media import (
)
from k1link.sessions.models import (
RecordedMediaArtifact,
ReplayArtifact,
ReplayCommand,
SessionArtifact,
SessionDetail,
@@ -40,6 +41,8 @@ PORTABLE_SOURCE_BUNDLE_SCHEMA: Final = "missioncore.portable-recorded-source-bun
PORTABLE_SOURCE_CAPABILITY_SCHEMA: Final = "missioncore.portable-recorded-source-capability/v1"
PORTABLE_SOURCE_ADAPTER_SCHEMA: Final = "missioncore.portable-source-adapter/v1"
PORTABLE_SOURCE_DOCUMENT_DIRECTORY: Final = "observatory-portable-source-contracts"
PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID: Final = "raw-transport-index"
PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE: Final = "application/x-ndjson"
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9.-]{2,127}$")
@@ -384,10 +387,14 @@ class RecordedK1SourceAdmissionService:
camera_source=camera_source,
recorded_media=recorded_media,
)
sealed_replay = _seal_replay_artifact_digests(
detail=detail,
replay=replay,
)
self._verify_replay(
detail=detail,
selected_sources=selected_sources,
replay=replay,
replay=sealed_replay,
)
try:
media = (
@@ -410,7 +417,7 @@ class RecordedK1SourceAdmissionService:
detail=detail,
catalog_sha256=catalog_sha256,
selected_sources=selected_sources,
replay=replay,
replay=sealed_replay,
media=media,
)
source_bundle = _canonical_json(source_bundle_document)
@@ -592,13 +599,31 @@ class RecordedK1SourceAdmissionService:
raise PortableSourceAdmissionIntegrityError("spatial replay members are not unique")
for replay_artifact in replay.artifacts:
catalog_artifact = catalog_artifacts.get(replay_artifact.artifact_id)
exact_metadata_member = (
catalog_artifact is not None
and replay_artifact.artifact_id
== PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
and catalog_artifact.kind
== PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
and replay_artifact.media_type
== PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
and catalog_artifact.media_type
== PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
)
digest_matches_catalog = (
replay_artifact.expected_sha256 == catalog_artifact.sha256
if catalog_artifact is not None and catalog_artifact.sha256 is not None
else exact_metadata_member
and isinstance(replay_artifact.expected_sha256, str)
and _SHA256.fullmatch(replay_artifact.expected_sha256) is not None
)
if (
catalog_artifact is None
or catalog_artifact.integrity_status not in _SEALED_ARTIFACT_STATES
or replay_artifact.media_type != catalog_artifact.media_type
or replay_artifact.file_byte_length != catalog_artifact.byte_length
or not 1 <= replay_artifact.replay_byte_length <= replay_artifact.file_byte_length
or replay_artifact.expected_sha256 != catalog_artifact.sha256
or not digest_matches_catalog
):
raise PortableSourceAdmissionIntegrityError(
"spatial replay member disagrees with the catalog"
@@ -799,6 +824,130 @@ class RecordedK1SourceAdmissionService:
}
def _seal_replay_artifact_digests(
*,
detail: SessionDetail,
replay: ReplayCommand,
) -> ReplayCommand:
"""Seal the one legacy replay member whose catalog has no stored digest.
Historical K1 catalogs explicitly register ``mqtt.metadata.jsonl`` as the
``raw-transport-index`` replay artifact, but the catalog row predates a
persisted SHA-256 column value. Portable admission may derive that one
digest from the already confined ReplayCommand handle. No sibling-name or
directory discovery is permitted, and every other missing digest remains
an integrity failure in ``_verify_replay``.
"""
catalog_artifacts = {artifact.artifact_id: artifact for artifact in detail.artifacts}
sealed: list[ReplayArtifact] = []
sealed_metadata_count = 0
for artifact in replay.artifacts:
if artifact.expected_sha256 is not None:
sealed.append(artifact)
continue
catalog_artifact = catalog_artifacts.get(artifact.artifact_id)
if (
catalog_artifact is None
or artifact.artifact_id
!= PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
or catalog_artifact.kind
!= PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
or artifact.media_type != PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
or catalog_artifact.media_type
!= PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
or catalog_artifact.sha256 is not None
or catalog_artifact.integrity_status not in _SEALED_ARTIFACT_STATES
or artifact.file_byte_length != catalog_artifact.byte_length
or artifact.replay_byte_length != catalog_artifact.byte_length
):
sealed.append(artifact)
continue
sealed_metadata_count += 1
if sealed_metadata_count != 1:
raise PortableSourceAdmissionIntegrityError(
"spatial replay metadata member is not unique"
)
sha256 = _hash_confined_replay_artifact(replay=replay, artifact=artifact)
sealed.append(replace(artifact, expected_sha256=sha256))
return replace(replay, artifacts=tuple(sealed))
def _hash_confined_replay_artifact(
*,
replay: ReplayCommand,
artifact: ReplayArtifact,
) -> str:
descriptor = -1
try:
allowed_root = replay.allowed_root.resolve(strict=True)
session_root = replay.session_root.resolve(strict=True)
source_metadata = artifact.path.lstat()
source_path = artifact.path.resolve(strict=True)
if (
not allowed_root.is_dir()
or not session_root.is_dir()
or not session_root.is_relative_to(allowed_root)
or not source_path.is_relative_to(session_root)
or stat.S_ISLNK(source_metadata.st_mode)
or not stat.S_ISREG(source_metadata.st_mode)
or not 1 <= artifact.file_byte_length <= MAX_SAFE_INTEGER
):
raise PortableSourceAdmissionIntegrityError(
"spatial replay metadata escapes its admitted session root"
)
descriptor = os.open(
source_path,
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
)
before = os.fstat(descriptor)
if (
not stat.S_ISREG(before.st_mode)
or before.st_size != artifact.file_byte_length
or before.st_size != artifact.replay_byte_length
):
raise PortableSourceAdmissionIntegrityError(
"spatial replay metadata is outside admitted bounds"
)
digest = hashlib.sha256()
byte_length = 0
while chunk := os.read(descriptor, 1024 * 1024):
byte_length += len(chunk)
if byte_length > artifact.file_byte_length:
raise PortableSourceAdmissionIntegrityError(
"spatial replay metadata grew while it was sealed"
)
digest.update(chunk)
after = os.fstat(descriptor)
stable_identity = (
before.st_dev,
before.st_ino,
before.st_size,
before.st_mtime_ns,
before.st_ctime_ns,
) == (
after.st_dev,
after.st_ino,
after.st_size,
after.st_mtime_ns,
after.st_ctime_ns,
)
if byte_length != artifact.file_byte_length or not stable_identity:
raise PortableSourceAdmissionIntegrityError(
"spatial replay metadata changed while it was sealed"
)
return digest.hexdigest()
except PortableSourceAdmissionIntegrityError:
raise
except OSError as exc:
raise PortableSourceAdmissionIntegrityError(
"spatial replay metadata is unavailable"
) from exc
finally:
if descriptor >= 0:
os.close(descriptor)
def _read_canonical_camera_summary(
source_path: Path,
) -> tuple[dict[str, object], int]:
+202 -1
View File
@@ -19,6 +19,7 @@ import re
import threading
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from datetime import datetime
from typing import Annotated, Final, Literal, Protocol
from uuid import uuid4
@@ -51,6 +52,7 @@ type WorkerCycleState = Literal[
"succeeded",
"failed",
"rejected",
"lease-lost",
]
type RecordedJobWireState = Literal[
"accepted",
@@ -122,6 +124,7 @@ class SealedObservatoryRecordedJob:
job_id: str
request_sha256: str
identity_sha256: str
submission_receipt_sha256: str
source_session_id: str
source_catalog_sha256: str
source_bundle_sha256: str
@@ -140,6 +143,10 @@ class SealedObservatoryRecordedJob:
checkpoint_policy: Literal["cooperative", "non-checkpointable"]
allowed_checkpoints: tuple[str, ...]
claim_generation: int
claim_claimed_at_utc: str | None
claim_expires_at_utc: str | None
claim_heartbeat_at_utc: str | None
claim_renewal_count: int
restart_from_zero: bool
@@ -213,6 +220,16 @@ class ObservatoryWorkerTransport(Protocol):
claim_token: str,
) -> Mapping[str, object]: ...
def renew_claim(
self,
*,
claimant_id: str,
job_id: str,
claim_token: str,
claim_generation: int,
heartbeat_sequence: int,
) -> Mapping[str, object]: ...
def succeed(
self,
*,
@@ -308,6 +325,13 @@ class _TerminalPayload(_StrictPayload):
message: str = Field(min_length=1, max_length=1_000)
class _ClaimLeasePayload(_StrictPayload):
claimed_at_utc: Timestamp
expires_at_utc: Timestamp
heartbeat_at_utc: Timestamp
renewal_count: int = Field(ge=0)
class _RecordedJobPayload(_StrictPayload):
schema_version: Literal["missioncore.observatory-recorded-job/v1"]
job_id: str = Field(pattern=_JOB_ID_PATTERN)
@@ -329,6 +353,7 @@ class _RecordedJobPayload(_StrictPayload):
restart_from_zero: bool
preemption_receipt_sha256: Sha256 | None
claim_generation: int = Field(ge=0)
claim_lease: _ClaimLeasePayload | None
result: _ResultPayload | None
terminal: _TerminalPayload | None
created_at_utc: Timestamp
@@ -365,10 +390,20 @@ class ObservatoryWorkerAgent:
transport: ObservatoryWorkerTransport,
executors: ObservatoryWorkerExecutorRegistry,
claim_request_id_factory: Callable[[], str] | None = None,
heartbeat_interval_seconds: float | None = None,
heartbeat_stop_timeout_seconds: float = 5.0,
) -> None:
if heartbeat_interval_seconds is not None and not (
0.01 <= heartbeat_interval_seconds <= 300.0
):
raise ValueError("Worker heartbeat interval is invalid")
if not 0.1 <= heartbeat_stop_timeout_seconds <= 300.0:
raise ValueError("Worker heartbeat stop timeout is invalid")
self._transport = transport
self._executors = executors
self._claim_request_id_factory = claim_request_id_factory or _default_claim_request_id
self._heartbeat_interval_seconds = heartbeat_interval_seconds
self._heartbeat_stop_timeout_seconds = heartbeat_stop_timeout_seconds
self._cycle_lock = threading.Lock()
def run_once(self) -> ObservatoryWorkerCycleReport:
@@ -443,11 +478,33 @@ class ObservatoryWorkerAgent:
job_id=claim.job.job_id,
)
active_job = _seal_job(started)
heartbeat = _ClaimHeartbeat(
transport=self._transport,
job=active_job,
claim_token=claim.claim_token,
interval_seconds=(
self._heartbeat_interval_seconds
if self._heartbeat_interval_seconds is not None
else _default_heartbeat_interval(active_job)
),
stop_timeout_seconds=self._heartbeat_stop_timeout_seconds,
)
heartbeat.start()
try:
result = adapter.execute(claim.job)
result = adapter.execute(active_job)
if not isinstance(result, ObservatoryWorkerExecutionResult):
raise TypeError("executor returned an unknown result contract")
except Exception as exc:
heartbeat.stop()
if heartbeat.failed:
return ObservatoryWorkerCycleReport(
state="lease-lost",
claim_request_id=claim_request_id,
job_id=claim.job.job_id,
failure_code="claim-heartbeat-lost",
)
failure_code = "executor-error"
acknowledgement = self._transport.fail(
claimant_id=WORKER_006_CONTOUR_ID,
@@ -468,6 +525,15 @@ class ObservatoryWorkerAgent:
failure_code=failure_code,
)
heartbeat.stop()
if heartbeat.failed:
return ObservatoryWorkerCycleReport(
state="lease-lost",
claim_request_id=claim_request_id,
job_id=claim.job.job_id,
failure_code="claim-heartbeat-lost",
)
acknowledgement = self._transport.succeed(
claimant_id=WORKER_006_CONTOUR_ID,
job_id=claim.job.job_id,
@@ -495,6 +561,92 @@ class ObservatoryWorkerAgent:
)
class _ClaimHeartbeat:
"""Renew one exact generation while an executor owns Worker resources."""
def __init__(
self,
*,
transport: ObservatoryWorkerTransport,
job: SealedObservatoryRecordedJob,
claim_token: str,
interval_seconds: float,
stop_timeout_seconds: float,
) -> None:
if interval_seconds <= 0:
raise ValueError("Worker heartbeat interval must be positive")
self._transport = transport
self._job = job
self._claim_token = claim_token
self._interval_seconds = interval_seconds
self._stop_timeout_seconds = stop_timeout_seconds
self._stop = threading.Event()
self._state_lock = threading.Lock()
self._failure: Exception | None = None
self._thread = threading.Thread(
target=self._run,
name=f"observatory-heartbeat-{job.job_id}",
daemon=True,
)
@property
def failed(self) -> bool:
with self._state_lock:
return self._failure is not None
def start(self) -> None:
self._thread.start()
def stop(self) -> None:
self._stop.set()
self._thread.join(timeout=self._stop_timeout_seconds)
if self._thread.is_alive():
self._record_failure(
ObservatoryWorkerClaimRejectedError(
"Worker claim heartbeat did not stop within its bound"
)
)
def _run(self) -> None:
sequence = self._job.claim_renewal_count + 1
# Renew immediately once start has been acknowledged. Waiting a full
# interval here would assume that claim/start transport latency consumed
# none of the original lease window.
while not self._stop.is_set():
try:
acknowledgement = self._transport.renew_claim(
claimant_id=WORKER_006_CONTOUR_ID,
job_id=self._job.job_id,
claim_token=self._claim_token,
claim_generation=self._job.claim_generation,
heartbeat_sequence=sequence,
)
renewed = _validate_transition_acknowledgement(
acknowledgement,
expected_job=self._job,
expected_state=("claimed", "running"),
)
if (
renewed.claim_lease is None
or renewed.claim_lease.renewal_count != sequence
):
raise ObservatoryWorkerClaimRejectedError(
"Worker heartbeat acknowledgement changed its sequence"
)
sequence += 1
except Exception as exc:
self._record_failure(exc)
self._stop.set()
return
if self._stop.wait(self._interval_seconds):
return
def _record_failure(self, exc: Exception) -> None:
with self._state_lock:
if self._failure is None:
self._failure = exc
def _validate_claim(
payload: Mapping[str, object],
*,
@@ -519,6 +671,10 @@ def _validate_claim(
raise ObservatoryWorkerClaimRejectedError(
"Worker claim job is not in a claimed generation"
)
if claim.job.claim_lease is None:
raise ObservatoryWorkerClaimRejectedError(
"Worker claim has no renewable lease"
)
if claim.job.result is not None or claim.job.terminal is not None:
raise ObservatoryWorkerClaimRejectedError(
"Worker claim already carries a terminal outcome"
@@ -556,6 +712,14 @@ def _seal_job(payload: _RecordedJobPayload) -> SealedObservatoryRecordedJob:
and payload.checkpoint_policy.allowed_checkpoints
):
raise ObservatoryWorkerClaimRejectedError("Worker claim checkpoint policy is inconsistent")
if payload.claim_lease is not None:
claimed_at = _parse_timestamp(payload.claim_lease.claimed_at_utc)
expires_at = _parse_timestamp(payload.claim_lease.expires_at_utc)
heartbeat_at = _parse_timestamp(payload.claim_lease.heartbeat_at_utc)
if not claimed_at <= heartbeat_at < expires_at:
raise ObservatoryWorkerClaimRejectedError(
"Worker claim lease chronology is inconsistent"
)
expected_request_sha256 = _sha256_document(
{
@@ -615,6 +779,7 @@ def _seal_job(payload: _RecordedJobPayload) -> SealedObservatoryRecordedJob:
job_id=payload.job_id,
request_sha256=payload.request_sha256,
identity_sha256=payload.identity_sha256,
submission_receipt_sha256=payload.submission_receipt_sha256,
source_session_id=payload.source.session_id,
source_catalog_sha256=payload.source.catalog_sha256,
source_bundle_sha256=payload.source.bundle_sha256,
@@ -638,6 +803,18 @@ def _seal_job(payload: _RecordedJobPayload) -> SealedObservatoryRecordedJob:
checkpoint_policy=payload.checkpoint_policy.mode,
allowed_checkpoints=tuple(payload.checkpoint_policy.allowed_checkpoints),
claim_generation=payload.claim_generation,
claim_claimed_at_utc=(
None if payload.claim_lease is None else payload.claim_lease.claimed_at_utc
),
claim_expires_at_utc=(
None if payload.claim_lease is None else payload.claim_lease.expires_at_utc
),
claim_heartbeat_at_utc=(
None if payload.claim_lease is None else payload.claim_lease.heartbeat_at_utc
),
claim_renewal_count=(
0 if payload.claim_lease is None else payload.claim_lease.renewal_count
),
restart_from_zero=payload.restart_from_zero,
)
@@ -662,6 +839,12 @@ def _validate_transition_acknowledgement(
raise ObservatoryWorkerClaimRejectedError(
"Worker transition acknowledgement has an unexpected state"
)
if acknowledgement.state in {"claimed", "running", "preemption-pending"} and (
acknowledgement.claim_lease is None
):
raise ObservatoryWorkerClaimRejectedError(
"Worker transition acknowledgement lost the active claim lease"
)
if (
sealed.job_id != expected_job.job_id
or sealed.identity_sha256 != expected_job.identity_sha256
@@ -677,6 +860,20 @@ def _default_claim_request_id() -> str:
return f"worker-006:{uuid4().hex}"
def _default_heartbeat_interval(job: SealedObservatoryRecordedJob) -> float:
if job.claim_heartbeat_at_utc is None or job.claim_expires_at_utc is None:
raise ObservatoryWorkerClaimRejectedError(
"Worker claim has no heartbeat lease bounds"
)
remaining = (
_parse_timestamp(job.claim_expires_at_utc)
- _parse_timestamp(job.claim_heartbeat_at_utc)
).total_seconds()
if remaining <= 0:
raise ObservatoryWorkerClaimRejectedError("Worker claim lease already expired")
return max(0.5, min(30.0, remaining / 3.0))
def _bounded_executor_failure(exc: Exception) -> str:
detail = " ".join(str(exc).split())
message = f"Executor adapter raised {type(exc).__name__}."
@@ -693,3 +890,7 @@ def _sha256_document(document: Mapping[str, object]) -> str:
separators=(",", ":"),
).encode()
return hashlib.sha256(payload).hexdigest()
def _parse_timestamp(value: str) -> datetime:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
File diff suppressed because it is too large Load Diff
+357
View File
@@ -0,0 +1,357 @@
"""Install-time composition and bounded polling for portable Worker 006.
The durable service owns transport cadence only. A reviewed Worker release
must inject an in-memory executor registry whose four-digest identities cover
every portable RunDefinition advertised as ready. Configuration cannot name
Python modules, commands, images, or executable paths, so neither Mission Core
nor an environment variable can turn an unsealed candidate into code.
"""
from __future__ import annotations
import os
import re
import stat
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from pathlib import Path
from threading import Event
from typing import Final
from urllib.parse import urlsplit
import httpx
from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry
from k1link.observatory.worker_agent import (
ObservatoryWorkerAgent,
ObservatoryWorkerCycleReport,
ObservatoryWorkerExecutorIdentity,
ObservatoryWorkerExecutorRegistry,
ObservatoryWorkerExecutorUnavailableError,
)
from k1link.observatory.worker_http_transport import (
ObservatoryWorkerHttpError,
ObservatoryWorkerHttpGateway,
)
OBSERVATORY_WORKER_BASE_URL_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_BASE_URL"
OBSERVATORY_WORKER_TOKEN_FILE_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE"
OBSERVATORY_WORKER_WORK_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT"
OBSERVATORY_WORKER_IDLE_POLL_SECONDS_ENV: Final = (
"MISSIONCORE_OBSERVATORY_WORKER_IDLE_POLL_SECONDS"
)
OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS_ENV: Final = (
"MISSIONCORE_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS"
)
OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES_ENV: Final = (
"MISSIONCORE_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES"
)
DEFAULT_OBSERVATORY_WORKER_BASE_URL: Final = "http://127.0.0.1:18080"
DEFAULT_OBSERVATORY_WORKER_IDLE_POLL_SECONDS: Final = 1.0
DEFAULT_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS: Final = 5.0
DEFAULT_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES: Final = 12
_TOKEN = re.compile(r"^[A-Za-z0-9._:-]{32,512}$")
class ObservatoryWorkerServiceError(RuntimeError):
"""Worker 006 cannot start without its exact local operational boundary."""
@dataclass(frozen=True, slots=True)
class ObservatoryWorkerServiceConfiguration:
"""Path-only service configuration; it never contains a plaintext secret."""
base_url: str
bearer_token_file: Path
work_root: Path
idle_poll_seconds: float = DEFAULT_OBSERVATORY_WORKER_IDLE_POLL_SECONDS
transport_backoff_seconds: float = (
DEFAULT_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS
)
max_consecutive_transport_failures: int = (
DEFAULT_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES
)
def __post_init__(self) -> None:
_validate_worker_base_url(self.base_url)
_absolute_path(self.bearer_token_file, "Worker bearer token file")
_absolute_path(self.work_root, "Worker work root")
if isinstance(self.idle_poll_seconds, bool) or not (
0.05 <= self.idle_poll_seconds <= 300.0
):
raise ValueError("Worker idle poll interval is invalid")
if isinstance(self.transport_backoff_seconds, bool) or not (
0.05 <= self.transport_backoff_seconds <= 300.0
):
raise ValueError("Worker transport backoff is invalid")
if (
isinstance(self.max_consecutive_transport_failures, bool)
or not isinstance(self.max_consecutive_transport_failures, int)
or not 1 <= self.max_consecutive_transport_failures <= 10_000
):
raise ValueError("Worker transport failure bound is invalid")
@classmethod
def from_environment(
cls,
environment: Mapping[str, str] | None = None,
) -> ObservatoryWorkerServiceConfiguration:
values = os.environ if environment is None else environment
token_file = _required_path(values, OBSERVATORY_WORKER_TOKEN_FILE_ENV)
work_root = _required_path(values, OBSERVATORY_WORKER_WORK_ROOT_ENV)
return cls(
base_url=values.get(
OBSERVATORY_WORKER_BASE_URL_ENV,
DEFAULT_OBSERVATORY_WORKER_BASE_URL,
),
bearer_token_file=token_file,
work_root=work_root,
idle_poll_seconds=_environment_float(
values,
OBSERVATORY_WORKER_IDLE_POLL_SECONDS_ENV,
DEFAULT_OBSERVATORY_WORKER_IDLE_POLL_SECONDS,
),
transport_backoff_seconds=_environment_float(
values,
OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS_ENV,
DEFAULT_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS,
),
max_consecutive_transport_failures=_environment_int(
values,
OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES_ENV,
DEFAULT_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES,
),
)
@dataclass(slots=True)
class InstalledObservatoryWorkerService:
"""A composed gateway and agent owned by one installed Worker release."""
configuration: ObservatoryWorkerServiceConfiguration
gateway: ObservatoryWorkerHttpGateway
agent: ObservatoryWorkerAgent
def close(self) -> None:
self.gateway.close()
def run(
self,
*,
stop: Event,
on_cycle: Callable[[ObservatoryWorkerCycleReport], None] | None = None,
) -> None:
"""Poll until stopped, with a bounded consecutive transport-failure gate."""
consecutive_transport_failures = 0
try:
while not stop.is_set():
try:
report = self.agent.run_once()
except ObservatoryWorkerHttpError as exc:
consecutive_transport_failures += 1
if (
consecutive_transport_failures
>= self.configuration.max_consecutive_transport_failures
):
raise ObservatoryWorkerServiceError(
"Worker transport exceeded its consecutive failure bound"
) from exc
stop.wait(self.configuration.transport_backoff_seconds)
continue
consecutive_transport_failures = 0
if on_cycle is not None:
on_cycle(report)
if report.state in {
"empty",
"deferred",
"rejected",
"lease-lost",
}:
stop.wait(self.configuration.idle_poll_seconds)
finally:
self.close()
def compose_installed_observatory_worker_service(
*,
configuration: ObservatoryWorkerServiceConfiguration,
definitions: PortableRunDefinitionRegistry,
executors: ObservatoryWorkerExecutorRegistry,
http_transport: httpx.BaseTransport | None = None,
) -> InstalledObservatoryWorkerService:
"""Bind transport to an install-time registry after exact coverage checks.
This is the fixed seam a reviewed Worker release calls. There is no
dynamic import/provider name in service configuration. Blocked catalog
candidates are ignored, while an empty ready set or any missing exact
local executor identity rejects service startup before the first claim.
"""
require_ready_executor_coverage(definitions=definitions, executors=executors)
bearer_token = load_observatory_worker_bearer_token(
configuration.bearer_token_file
)
try:
gateway = ObservatoryWorkerHttpGateway(
base_url=configuration.base_url,
bearer_token=bearer_token,
work_root=configuration.work_root,
transport=http_transport,
)
finally:
# The immutable string remains owned by the gateway headers for the
# service lifetime; this local binding must not outlive composition.
del bearer_token
return InstalledObservatoryWorkerService(
configuration=configuration,
gateway=gateway,
agent=ObservatoryWorkerAgent(transport=gateway, executors=executors),
)
def require_ready_executor_coverage(
*,
definitions: PortableRunDefinitionRegistry,
executors: ObservatoryWorkerExecutorRegistry,
) -> tuple[ObservatoryWorkerExecutorIdentity, ...]:
"""Prove every server-advertisable definition has one local identity."""
ready = definitions.ready_recorded_definitions()
if not ready:
raise ObservatoryWorkerServiceError(
"no portable RunDefinition has a sealed ready executor"
)
identities: list[ObservatoryWorkerExecutorIdentity] = []
for definition in ready:
identity = ObservatoryWorkerExecutorIdentity(
release_sha256=definition.executor_release_sha256,
image_sha256=definition.executor_image_sha256,
model_manifest_sha256=definition.model_manifest_sha256,
resource_profile_sha256=definition.resource_profile_sha256,
)
try:
executors.resolve(identity)
except ObservatoryWorkerExecutorUnavailableError as exc:
raise ObservatoryWorkerServiceError(
"a ready portable RunDefinition has no exact local executor identity"
) from exc
identities.append(identity)
return tuple(identities)
def load_observatory_worker_bearer_token(path: Path) -> str:
"""Read one private regular ASCII token without accepting links/newlines."""
candidate = path.expanduser().absolute()
descriptor: int | None = None
try:
descriptor = os.open(
candidate,
os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0),
)
metadata = os.fstat(descriptor)
if not stat.S_ISREG(metadata.st_mode):
raise ObservatoryWorkerServiceError(
"Worker bearer credential must be a regular file"
)
if os.name != "posix":
raise ObservatoryWorkerServiceError(
"native Worker credential ACL verification is not available; "
"use the admitted POSIX Worker service runtime"
)
if metadata.st_mode & 0o077:
raise ObservatoryWorkerServiceError(
"Worker bearer credential permissions are too broad"
)
if not 32 <= metadata.st_size <= 512:
raise ObservatoryWorkerServiceError(
"Worker bearer credential format is invalid"
)
with os.fdopen(descriptor, "rb") as stream:
descriptor = None
payload = stream.read(513)
except ObservatoryWorkerServiceError:
raise
except OSError as exc:
raise ObservatoryWorkerServiceError(
"Worker bearer credential is unavailable"
) from exc
finally:
if descriptor is not None:
os.close(descriptor)
try:
token = payload.decode("ascii")
except UnicodeDecodeError as exc:
raise ObservatoryWorkerServiceError(
"Worker bearer credential is not ASCII"
) from exc
if _TOKEN.fullmatch(token) is None:
raise ObservatoryWorkerServiceError(
"Worker bearer credential format is invalid"
)
return token
def _validate_worker_base_url(value: str) -> None:
parsed = urlsplit(value)
if (
value != value.strip()
or parsed.query
or parsed.fragment
or parsed.username is not None
or parsed.password is not None
or parsed.path not in {"", "/"}
or parsed.scheme not in {"http", "https"}
or not parsed.hostname
):
raise ValueError("Worker Mission Core base URL is invalid")
if parsed.scheme == "http" and parsed.hostname not in {
"127.0.0.1",
"localhost",
"::1",
}:
raise ValueError("unencrypted Worker transport requires a loopback tunnel")
def _absolute_path(path: Path, label: str) -> None:
if not path.is_absolute() or str(path) != str(path).strip():
raise ValueError(f"{label} must be an absolute path")
def _required_path(environment: Mapping[str, str], name: str) -> Path:
value = environment.get(name, "")
if not value or value != value.strip():
raise ObservatoryWorkerServiceError(f"{name} is required")
path = Path(value)
_absolute_path(path, name)
return path
def _environment_float(
environment: Mapping[str, str],
name: str,
default: float,
) -> float:
value = environment.get(name, "")
if not value:
return default
try:
return float(value)
except ValueError as exc:
raise ObservatoryWorkerServiceError(f"{name} is invalid") from exc
def _environment_int(
environment: Mapping[str, str],
name: str,
default: int,
) -> int:
value = environment.get(name, "")
if not value:
return default
if not value.isascii() or not value.isdecimal():
raise ObservatoryWorkerServiceError(f"{name} is invalid")
return int(value)
@@ -0,0 +1,142 @@
"""Pure launchd plan for the Mac-owned Worker 006 reverse SSH tunnel."""
from __future__ import annotations
import hashlib
import os
import plistlib
import stat
from dataclasses import dataclass
from pathlib import Path
from typing import Final
OBSERVATORY_WORKER_TUNNEL_LABEL: Final = (
"com.nodedc.observatory-worker-tunnel.local"
)
OBSERVATORY_WORKER_TUNNEL_SCHEMA: Final = (
"missioncore.observatory-worker-tunnel-plan/v1"
)
OBSERVATORY_WORKER_TUNNEL_PORT: Final = 18080
class ObservatoryWorkerTunnelPlanError(RuntimeError):
"""The reverse tunnel cannot be declared without weakening its boundary."""
@dataclass(frozen=True, slots=True)
class ObservatoryWorkerTunnelLaunchAgentPlan:
agent_path: Path
data_directory: Path
desired_sha256: str
desired_payload: bytes
def to_dict(self) -> dict[str, object]:
return {
"schema_version": OBSERVATORY_WORKER_TUNNEL_SCHEMA,
"label": OBSERVATORY_WORKER_TUNNEL_LABEL,
"agent_path": str(self.agent_path),
"data_directory": str(self.data_directory),
"desired_sha256": self.desired_sha256,
"transport": {
"owner": "mac-launchd",
"ssh_alias": "mission-gpu",
"worker_listener": "127.0.0.1:18080",
"mission_core_target": "127.0.0.1:8000",
"encrypted": True,
"worker_listener_loopback_only": True,
"bearer_credential_in_arguments": False,
},
"changes": {
"durable_mutation_performed": False,
"requires_hash-gated_install": True,
},
}
def plan_observatory_worker_tunnel_launch_agent(
*,
data_directory: Path,
agent_path: Path,
ssh_path: Path = Path("/usr/bin/ssh"),
) -> ObservatoryWorkerTunnelLaunchAgentPlan:
"""Build, but never install, the exact reverse-loopback declaration."""
data_root = _private_directory(data_directory)
ssh = _exact_executable(ssh_path)
target_path = agent_path.expanduser().absolute()
log_path = data_root / "observatory-worker-tunnel.log"
arguments = [
str(ssh),
"-o",
"BatchMode=yes",
"-o",
"ExitOnForwardFailure=yes",
"-o",
"ServerAliveInterval=15",
"-o",
"ServerAliveCountMax=3",
"-o",
"RequestTTY=no",
"-N",
"-T",
"-R",
"127.0.0.1:18080:127.0.0.1:8000",
"mission-gpu",
]
desired = {
"Label": OBSERVATORY_WORKER_TUNNEL_LABEL,
"ProgramArguments": arguments,
"KeepAlive": True,
"RunAtLoad": True,
"AbandonProcessGroup": False,
"ProcessType": "Background",
"ThrottleInterval": 5,
"ExitTimeOut": 10,
"StandardOutPath": str(log_path),
"StandardErrorPath": str(log_path),
}
payload = plistlib.dumps(desired, fmt=plistlib.FMT_XML, sort_keys=True)
return ObservatoryWorkerTunnelLaunchAgentPlan(
agent_path=target_path,
data_directory=data_root,
desired_sha256=hashlib.sha256(payload).hexdigest(),
desired_payload=payload,
)
def _private_directory(path: Path) -> Path:
candidate = path.expanduser().absolute()
try:
metadata = candidate.lstat()
resolved = candidate.resolve(strict=True)
except OSError as exc:
raise ObservatoryWorkerTunnelPlanError(
"Mission Core data directory is unavailable"
) from exc
if (
resolved != candidate
or not stat.S_ISDIR(metadata.st_mode)
or stat.S_IMODE(metadata.st_mode) != 0o700
or metadata.st_uid != os.getuid()
):
raise ObservatoryWorkerTunnelPlanError(
"Mission Core data directory is not private and canonical"
)
return resolved
def _exact_executable(path: Path) -> Path:
candidate = path.expanduser().absolute()
try:
metadata = candidate.lstat()
resolved = candidate.resolve(strict=True)
except OSError as exc:
raise ObservatoryWorkerTunnelPlanError("SSH executable is unavailable") from exc
if (
resolved != candidate
or stat.S_ISLNK(metadata.st_mode)
or not stat.S_ISREG(metadata.st_mode)
or not os.access(candidate, os.X_OK)
):
raise ObservatoryWorkerTunnelPlanError("SSH executable is not exact")
return resolved
+204 -31
View File
@@ -47,20 +47,38 @@ from k1link.observatory.m49_queue_binding import (
M49QueueBindingError,
M49RecordedQueueBindingService,
)
from k1link.observatory.portable_queue_binding import (
PortableQueueBindingError,
PortableRecordedQueueBindingService,
)
from k1link.observatory.portable_result_contract import (
PortableCalculationProfileRegistry,
PortableResultContractValidatorRegistry,
)
from k1link.observatory.portable_result_publisher import (
resolve_published_portable_calculation_profile,
)
from k1link.observatory.portable_run_definitions import (
PortableRunDefinitionRegistry,
PortableRunDefinitionRegistryError,
)
from k1link.observatory.portable_setup_projection import (
PORTABLE_LAB_V1_SETUP_ID,
PortableLabV1SetupProjector,
PortableSetupProjectionError,
PortableSetupProjector,
portable_calculation_profile_registry,
)
from k1link.observatory.portable_worker_integration import (
PortableObservatoryWorkerIntegration,
PortableWorkerIntegrationError,
PortableWorkerStorageRoots,
build_portable_observatory_worker_integration,
portable_result_validator_registry,
)
from k1link.observatory.recorded_jobs import (
ObservatoryRecordedJobQueue,
ObservatoryRecordedQueueError,
RecordedRunDefinitionRegistry,
)
from k1link.observatory.source_admission import RecordedK1SourceAdmissionService
from k1link.sessions import (
MaterializedRecording,
RecordedCameraFrameService,
@@ -73,6 +91,7 @@ from k1link.sessions import (
SessionRecordingPreparationManager,
SessionStore,
)
from k1link.sessions.models import SessionSummary
from k1link.simulation.projects import SimulationProjectService, SimulationProjectStore
from k1link.web.advanced_laboratory_api import build_advanced_laboratory_router
from k1link.web.artifact_health_api import build_artifact_health_router
@@ -261,6 +280,55 @@ plugin_catalog: DevicePluginCatalog = plugin_environment.catalog
plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher
session_store = SessionStore(REPOSITORY_ROOT)
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY: PortableRunDefinitionRegistry | None
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR: str | None
OBSERVATORY_PORTABLE_CALCULATION_PROFILES: PortableCalculationProfileRegistry | None
OBSERVATORY_PORTABLE_RESULT_VALIDATORS: PortableResultContractValidatorRegistry | None
try:
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY = PortableRunDefinitionRegistry.from_file(
REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
)
OBSERVATORY_PORTABLE_CALCULATION_PROFILES = portable_calculation_profile_registry(
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY
)
OBSERVATORY_PORTABLE_RESULT_VALIDATORS = portable_result_validator_registry(
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY
)
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR = None
except (
PortableRunDefinitionRegistryError,
PortableWorkerIntegrationError,
OSError,
ValueError,
) as exc:
# Portable definitions are an optional observation-only slice. Registry
# drift cannot affect K1, Simulation, legacy LAB, or the exact M49 binding.
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY = None
OBSERVATORY_PORTABLE_CALCULATION_PROFILES = None
OBSERVATORY_PORTABLE_RESULT_VALIDATORS = None
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR = str(exc)
def _resolve_observatory_calculation_profile(
summary: SessionSummary,
) -> dict[str, object] | None:
if OBSERVATORY_LABORATORY_SETUP_REGISTRY is not None:
legacy = OBSERVATORY_LABORATORY_SETUP_REGISTRY.observatory_calculation_profile(
summary
)
if legacy is not None:
return legacy
if (
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None
or OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None
):
return None
return resolve_published_portable_calculation_profile(
summary,
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
)
def _load_optional_observatory_worker_authentication(
recorded_job_queue: ObservatoryRecordedJobQueue | None,
@@ -299,9 +367,14 @@ try:
REPOSITORY_ROOT / "config" / "observatory-m49-recorded-queue-binding.json"
),
)
recorded_definitions = list(OBSERVATORY_RECORDED_BINDING_SERVICE.definitions.definitions)
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is not None:
recorded_definitions.extend(
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY.ready_recorded_definitions()
)
OBSERVATORY_RECORDED_JOB_QUEUE = ObservatoryRecordedJobQueue(
session_store.data_dir,
definitions=OBSERVATORY_RECORDED_BINDING_SERVICE.definitions,
definitions=RecordedRunDefinitionRegistry(tuple(recorded_definitions)),
)
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = None
except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError) as exc:
@@ -316,11 +389,14 @@ OBSERVATORY_WORKER_CLAIM_LEASE_READY = False
OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = False
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED = False
OBSERVATORY_WORKER_AUTHENTICATION: ObservatoryWorkerAuthentication | None
OBSERVATORY_WORKER_AUTHENTICATION_ERROR: str | None
OBSERVATORY_WORKER_API_ERROR: str | None
OBSERVATORY_WORKER_AUTHENTICATION = None
OBSERVATORY_WORKER_API_ERROR = (
"Worker pull API is hard-disabled until claim leases and a verified "
"Observatory result publisher are implemented and accepted"
(
OBSERVATORY_WORKER_AUTHENTICATION,
OBSERVATORY_WORKER_AUTHENTICATION_ERROR,
) = _load_optional_observatory_worker_authentication(
OBSERVATORY_RECORDED_JOB_QUEUE,
token_path=OBSERVATORY_WORKER_TOKEN_PATH,
)
simulation_project_store = SimulationProjectStore(session_store.data_dir)
simulation_project_service = SimulationProjectService(simulation_project_store)
@@ -336,37 +412,122 @@ session_recording_materializer = SessionRecordingMaterializer(
session_recorded_media_inspector = RecordedMediaInspector(
session_store.data_dir / "recorded-media-preparations"
)
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableLabV1SetupProjector | None
OBSERVATORY_PORTABLE_WORKER_INTEGRATION: PortableObservatoryWorkerIntegration | None
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR: str | None
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS: PortableWorkerStorageRoots | None = None
try:
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None:
raise PortableWorkerIntegrationError(
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR
or "portable definition registry is unavailable"
)
if OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None:
raise PortableWorkerIntegrationError(
"portable calculation profile registry is unavailable"
)
if OBSERVATORY_PORTABLE_RESULT_VALIDATORS is None:
raise PortableWorkerIntegrationError(
"portable result validator registry is unavailable"
)
if OBSERVATORY_RECORDED_JOB_QUEUE is None:
raise PortableWorkerIntegrationError(
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR
or "Observatory recorded-job queue is unavailable"
)
if session_artifact_gateway is None:
raise PortableWorkerIntegrationError(
"central artifact store is not configured"
)
if session_artifact_gateway.status().central_status != "ready":
raise PortableWorkerIntegrationError(
"central artifact store is unavailable"
)
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS = (
PortableWorkerStorageRoots.from_environment(
artifact_store_root=session_artifact_gateway.store.root,
)
)
OBSERVATORY_PORTABLE_WORKER_INTEGRATION = (
build_portable_observatory_worker_integration(
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
session_store=session_store,
media_inspector=session_recorded_media_inspector,
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
artifact_store=session_artifact_gateway.store,
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
validators=OBSERVATORY_PORTABLE_RESULT_VALIDATORS,
source_cas_root=(
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.source_cas_root
),
result_staging_root=(
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.result_staging_root
),
)
)
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = None
except (PortableWorkerIntegrationError, OSError, ValueError) as exc:
# Constructing this dormant foundation does not enable the Worker router.
# Failure remains isolated from K1, Simulation and legacy LAB.
OBSERVATORY_PORTABLE_WORKER_INTEGRATION = None
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = str(exc)
OBSERVATORY_WORKER_API_ERROR = (
"Worker pull API is hard-disabled pending sealed installed executors, "
"a configured Worker credential, explicit integration acceptance and the "
"production gate"
+ (
""
if OBSERVATORY_WORKER_AUTHENTICATION_ERROR is None
else (
"; authentication unavailable: "
f"{OBSERVATORY_WORKER_AUTHENTICATION_ERROR}"
)
)
+ (
""
if OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR is None
else f"; integration unavailable: {OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR}"
)
)
OBSERVATORY_WORKER_DISPATCH_READY = (
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED
and OBSERVATORY_WORKER_CLAIM_LEASE_READY
and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
and OBSERVATORY_WORKER_AUTHENTICATION is not None
and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
)
OBSERVATORY_PORTABLE_BINDING_SERVICE: PortableRecordedQueueBindingService | None
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableSetupProjector | None
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
try:
portable_definition_registry = PortableRunDefinitionRegistry.from_file(
REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
)
portable_lab_v1_definition = next(
definition
for definition in portable_definition_registry.definitions
if definition.setup_id == PORTABLE_LAB_V1_SETUP_ID
)
portable_source_capability_service = RecordedK1SourceAdmissionService(
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None:
raise PortableSetupProjectionError(
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR
or "portable definition registry is unavailable"
)
OBSERVATORY_PORTABLE_BINDING_SERVICE = PortableRecordedQueueBindingService(
data_dir=session_store.data_dir,
session_store=session_store,
media_inspector=session_recorded_media_inspector,
requirements=portable_lab_v1_definition.to_source_admission_requirements(),
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
)
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableLabV1SetupProjector(
registry=portable_definition_registry,
capability_probe=portable_source_capability_service,
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableSetupProjector(
registry=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
capability_probe=OBSERVATORY_PORTABLE_BINDING_SERVICE,
dispatch_available=OBSERVATORY_WORKER_DISPATCH_READY,
)
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None
except (
PortableQueueBindingError,
PortableRunDefinitionRegistryError,
PortableSetupProjectionError,
OSError,
StopIteration,
ValueError,
) as exc:
# Portable LAB V1 is an optional observation-only slice. A drifted
# Portable setup execution is an optional observation-only slice. A drifted
# registry cannot affect K1, Simulation, legacy LAB, or the exact M49 queue.
OBSERVATORY_PORTABLE_BINDING_SERVICE = None
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = None
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = str(exc)
_ffmpeg = _resolve_media_tool("ffmpeg")
@@ -829,6 +990,14 @@ app.include_router(
perception_overlay_provider=session_perception_overlay_store,
perception_media_provider=session_perception_epoch_store,
point_color_renderers=plugin_environment.point_color_renderers,
lab_calculation_profile_resolver=(
None
if (
OBSERVATORY_LABORATORY_SETUP_REGISTRY is None
and OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None
)
else _resolve_observatory_calculation_profile
),
)
)
app.include_router(
@@ -843,19 +1012,23 @@ app.include_router(
recorded_job_queue_error=OBSERVATORY_RECORDED_JOB_QUEUE_ERROR,
portable_setup_projector=OBSERVATORY_PORTABLE_SETUP_PROJECTOR,
portable_setup_projector_error=OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR,
portable_binding_service=OBSERVATORY_PORTABLE_BINDING_SERVICE,
)
)
if (
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED
and OBSERVATORY_WORKER_CLAIM_LEASE_READY
and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
and OBSERVATORY_WORKER_AUTHENTICATION is not None
):
if OBSERVATORY_WORKER_DISPATCH_READY:
assert OBSERVATORY_RECORDED_JOB_QUEUE is not None
assert OBSERVATORY_WORKER_AUTHENTICATION is not None
assert OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
app.include_router(
build_observatory_worker_router(
OBSERVATORY_RECORDED_JOB_QUEUE,
authentication=OBSERVATORY_WORKER_AUTHENTICATION,
artifact_transport=(
OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport
),
result_publisher=(
OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher
),
)
)
app.include_router(
+265 -8
View File
@@ -22,9 +22,19 @@ from k1link.observatory.m49_queue_binding import (
M49QueueBindingIntegrityError,
M49RecordedQueueBindingService,
)
from k1link.observatory.portable_queue_binding import (
PortableQueueBindingError,
PortableQueueBindingIntegrityError,
PortableQueueBindingStaleCheckError,
PortableRecordedQueueBindingService,
)
from k1link.observatory.portable_run_definitions import (
PortableRunDefinitionUnavailableError,
)
from k1link.observatory.portable_setup_projection import (
PortableLabV1SetupProjector,
PortableSetupProjectionError,
PortableSetupProjector,
)
from k1link.observatory.recorded_jobs import (
ObservatoryRecordedJobQueue,
@@ -33,6 +43,7 @@ from k1link.observatory.recorded_jobs import (
ObservatoryRecordedQueueError,
ObservatoryRecordedQueueNotFoundError,
)
from k1link.observatory.source_admission import PortableSourceAdmissionError
from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore
from k1link.sessions.models import SessionSummary
@@ -130,6 +141,8 @@ class ObservatoryRecordedRunSubmitRequest(_StrictApiModel):
max_length=96,
pattern=r"^[a-z][a-z0-9-]{2,95}$",
)
definition_sha256: str | None = Field(default=None, pattern=r"^[a-f0-9]{64}$")
check_sha256: str | None = Field(default=None, pattern=r"^[a-f0-9]{64}$")
def build_observatory_router(
@@ -142,8 +155,9 @@ def build_observatory_router(
recorded_binding_service: M49RecordedQueueBindingService | None = None,
recorded_job_queue: ObservatoryRecordedJobQueue | None = None,
recorded_job_queue_error: str | None = None,
portable_setup_projector: PortableLabV1SetupProjector | None = None,
portable_setup_projector: PortableSetupProjector | PortableLabV1SetupProjector | None = None,
portable_setup_projector_error: str | None = None,
portable_binding_service: PortableRecordedQueueBindingService | None = None,
) -> APIRouter:
"""Build bounded catalog-only mutations for typed Observatory projections."""
@@ -214,6 +228,126 @@ def build_observatory_router(
available.add(result_id)
return frozenset(available)
def portable_run_preflight(
source: SessionSummary,
request: ObservatoryRunPreflightRequest,
) -> dict[str, object] | None:
projector = portable_setup_projector
if projector is None or not projector.has_setup(request.setup_id):
return None
try:
projected = projector.project(source, setup_id=request.setup_id)
except PortableSetupProjectionError as exc:
raise HTTPException(
status_code=503,
detail="Portable-каталог сетапов нарушил контракт целостности.",
) from exc
definition = projected.get("run_definition")
compatibility = projected.get("source_compatibility")
executor = projected.get("executor")
if (
not isinstance(definition, dict)
or not isinstance(compatibility, dict)
or not isinstance(executor, dict)
):
raise HTTPException(
status_code=503,
detail="Portable-каталог сетапов нарушил контракт целостности.",
)
expected_digest = definition.get("definition_sha256")
if request.definition_sha256 != expected_digest:
raise HTTPException(
status_code=409,
detail="Идентичность RunDefinition изменилась; обновите каталог.",
)
compatible = compatibility.get("compatible") is True
executor_ready = executor.get("state") == "ready" and executor.get("ready") is True
checked = None
check_reason: str | None = None
if (
compatible
and executor_ready
and portable_binding_service is not None
and recorded_job_queue is not None
and isinstance(expected_digest, str)
):
try:
checked = portable_binding_service.check(
source_session_id=request.source_session_id,
setup_id=request.setup_id,
definition_sha256=expected_digest,
)
except PortableRunDefinitionUnavailableError as exc:
check_reason = str(exc)
except (PortableQueueBindingError, PortableSourceAdmissionError, ValueError):
check_reason = (
"Источник или исполняемый portable-релиз не прошёл проверку целостности."
)
elif compatible and executor_ready:
check_reason = "Portable dispatch-контур или durable-очередь недоступны."
check_sha256 = None if checked is None else checked.check_sha256
queueable = check_sha256 is not None
checks: list[dict[str, Any]] = [
{
"check_id": "source-compatibility",
"outcome": "pass" if compatible else "fail",
"reason_code": "source-compatible" if compatible else "source-incompatible",
"message": str(compatibility.get("reason")),
},
{
"check_id": "executor",
"outcome": "pass" if executor_ready else "fail",
"reason_code": (
"executor-release-sealed"
if executor_ready
else str(executor.get("reason_code"))
),
"message": (
"Исполняемый portable-релиз и image запечатаны."
if executor_ready
else str(executor.get("reason"))
),
},
{
"check_id": "definition-check",
"outcome": "pass" if checked is not None else "fail",
"reason_code": (
"portable-check-sealed" if checked is not None else "portable-check-unavailable"
),
"message": (
"Источник и RunDefinition связаны одноразовым check SHA."
if checked is not None
else check_reason
or "Portable-проверка недоступна до установки executor-релиза."
),
},
{
"check_id": "durable-queue",
"outcome": "pass" if queueable else "fail",
"reason_code": (
"durable-queue-ready" if queueable else "durable-queue-unavailable"
),
"message": (
"Durable-очередь готова принять расчёт по check SHA."
if queueable
else "Расчёт нельзя поставить в очередь."
),
},
]
return {
"schema_version": OBSERVATORY_RUN_PREFLIGHT_SCHEMA,
"source_session_id": request.source_session_id,
"setup_id": request.setup_id,
"definition_sha256": expected_digest,
"check_sha256": check_sha256,
"outcome": "queueable" if queueable else "blocked",
"submission_allowed": queueable,
"checks": checks,
"existing_result_ids": [],
"executor": executor,
"authority": projected.get("authority", dict(_OBSERVATION_ONLY_AUTHORITY)),
}
if portable_setup_projector is not None:
@router.get("/api/v1/observatory/portable-laboratory-setups")
@@ -230,7 +364,7 @@ def build_observatory_router(
except PortableSetupProjectionError as exc:
raise HTTPException(
status_code=503,
detail="Portable-каталог LAB V1 нарушил контракт целостности.",
detail="Portable-каталог сетапов нарушил контракт целостности.",
) from exc
elif portable_setup_projector_error is not None:
@@ -246,10 +380,10 @@ def build_observatory_router(
del source_session_id
raise HTTPException(
status_code=503,
detail="Portable-каталог LAB V1 недоступен.",
detail="Portable-каталог сетапов недоступен.",
)
if setup_registry is not None:
if setup_registry is not None or portable_setup_projector is not None:
@router.get("/api/v1/observatory/laboratory-setups")
def list_observatory_laboratory_setups(
@@ -259,6 +393,11 @@ def build_observatory_router(
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
),
) -> dict[str, object]:
if setup_registry is None:
raise HTTPException(
status_code=503,
detail="Каталог legacy-сетапов Обсерватории недоступен.",
)
source = source_summary(source_session_id)
return setup_registry.catalog(
source,
@@ -270,6 +409,11 @@ def build_observatory_router(
request: ObservatoryRunPreflightRequest,
) -> dict[str, object]:
source = source_summary(request.source_session_id)
portable = portable_run_preflight(source, request)
if portable is not None:
return portable
if setup_registry is None:
raise HTTPException(status_code=404, detail="Сетап лаборатории не найден.")
try:
setup_registry.setup(request.setup_id)
except KeyError as exc:
@@ -639,16 +783,18 @@ def build_observatory_router(
detail="Подготовка расчётов Обсерватории недоступна.",
)
if (
setup_registry is not None
and recorded_binding_service is not None
and recorded_job_queue is not None
if recorded_job_queue is not None and (
recorded_binding_service is not None or portable_binding_service is not None
):
@router.post("/api/v1/observatory/runs", status_code=202)
def submit_observatory_recorded_run(
request: ObservatoryRecordedRunSubmitRequest,
) -> dict[str, object]:
portable_request = (
portable_setup_projector is not None
and portable_setup_projector.has_setup(request.setup_id)
)
try:
existing_job = recorded_job_queue.get_by_idempotency_key(request.idempotency_key)
except ObservatoryRecordedQueueNotFoundError:
@@ -662,6 +808,10 @@ def build_observatory_router(
if (
existing_job.source_session_id != request.source_session_id
or existing_job.setup_id != request.setup_id
or (
portable_request
and existing_job.definition_sha256 != request.definition_sha256
)
):
raise HTTPException(
status_code=409,
@@ -670,6 +820,113 @@ def build_observatory_router(
return existing_job.as_dict()
source = source_summary(request.source_session_id)
if portable_request:
if portable_binding_service is None:
raise HTTPException(
status_code=503,
detail="Portable dispatch-контур недоступен.",
)
if request.definition_sha256 is None or request.check_sha256 is None:
raise HTTPException(
status_code=409,
detail=(
"Для portable-расчёта требуются актуальные definition SHA "
"и check SHA из preflight."
),
)
assert portable_setup_projector is not None
try:
portable_projection = portable_setup_projector.project(
source,
setup_id=request.setup_id,
)
except PortableSetupProjectionError as exc:
raise HTTPException(
status_code=503,
detail="Portable-каталог сетапов нарушил контракт целостности.",
) from exc
projected_definition = portable_projection.get("run_definition")
projected_executor = portable_projection.get("executor")
if not isinstance(projected_definition, dict) or not isinstance(
projected_executor, dict
):
raise HTTPException(
status_code=503,
detail="Portable-каталог сетапов нарушил контракт целостности.",
)
if projected_definition.get("definition_sha256") != request.definition_sha256:
raise HTTPException(
status_code=409,
detail="Идентичность RunDefinition изменилась; повторите preflight.",
)
if (
projected_executor.get("state") != "ready"
or projected_executor.get("ready") is not True
):
raise HTTPException(
status_code=409,
detail="Portable executor-релиз не установлен.",
)
try:
job, _created = portable_binding_service.submit(
source_session_id=request.source_session_id,
setup_id=request.setup_id,
definition_sha256=request.definition_sha256,
expected_check_sha256=request.check_sha256,
idempotency_key=request.idempotency_key,
)
return job.as_dict()
except PortableQueueBindingStaleCheckError as exc:
raise HTTPException(
status_code=409,
detail="Источник или RunDefinition изменились; повторите preflight.",
) from exc
except PortableRunDefinitionUnavailableError as exc:
raise HTTPException(
status_code=409,
detail="Portable executor-релиз не установлен.",
) from exc
except (
PortableQueueBindingIntegrityError,
PortableSourceAdmissionError,
) as exc:
raise HTTPException(
status_code=409,
detail="Portable-привязка источника не прошла проверку целостности.",
) from exc
except ObservatoryRecordedQueueConflictError as exc:
try:
raced = recorded_job_queue.get_by_idempotency_key(request.idempotency_key)
except ObservatoryRecordedQueueNotFoundError:
raced = None
if (
raced is not None
and raced.source_session_id == request.source_session_id
and raced.setup_id == request.setup_id
and raced.definition_sha256 == request.definition_sha256
):
return raced.as_dict()
raise HTTPException(
status_code=409,
detail="Ключ идемпотентности уже связан с другим расчётом.",
) from exc
except ObservatoryRecordedQueueCapacityError as exc:
raise HTTPException(
status_code=503,
detail="Квота durable-очереди расчётов исчерпана.",
) from exc
except (
PortableQueueBindingError,
ObservatoryRecordedQueueError,
ValueError,
) as exc:
raise HTTPException(
status_code=503,
detail="Portable dispatch-контур недоступен.",
) from exc
if setup_registry is None or recorded_binding_service is None:
raise HTTPException(status_code=404, detail="Сетап лаборатории не найден.")
try:
setup_registry.setup(request.setup_id)
except KeyError as exc:
+292 -3
View File
@@ -18,11 +18,23 @@ from dataclasses import dataclass
from pathlib import Path
from typing import Annotated, Final, Literal
from fastapi import APIRouter, Depends, Header, HTTPException, Response
from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response
from fastapi import Path as ApiPath
from fastapi.responses import FileResponse
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from pydantic import BaseModel, ConfigDict, Field
from k1link.observatory.portable_artifact_transport import (
MAX_RESULT_MANIFEST_BYTES,
PortableArtifactTransportError,
PortableArtifactTransportIntegrityError,
PortableArtifactTransportUnavailableError,
PortableObservatoryArtifactTransport,
)
from k1link.observatory.portable_result_contract import PortableResultPublisherError
from k1link.observatory.portable_result_publisher import (
PortableObservatoryResultPublisher,
)
from k1link.observatory.recorded_jobs import (
ObservatoryRecordedCheckpointError,
ObservatoryRecordedJobQueue,
@@ -38,6 +50,7 @@ from k1link.observatory.recorded_jobs import (
OBSERVATORY_WORKER_CLAIM_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-claim-request/v1"
OBSERVATORY_WORKER_START_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-start-request/v1"
OBSERVATORY_WORKER_RENEW_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-renew-request/v1"
OBSERVATORY_WORKER_CHECKPOINT_REQUEST_SCHEMA: Final = (
"missioncore.observatory-worker-checkpoint-request/v1"
)
@@ -46,6 +59,10 @@ OBSERVATORY_WORKER_SUCCEED_REQUEST_SCHEMA: Final = (
)
OBSERVATORY_WORKER_FAIL_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-fail-request/v1"
OBSERVATORY_WORKER_CONTOUR_HEADER: Final = "X-Mission-Core-Contour-Id"
OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER: Final = "X-Mission-Core-Claim-Token"
OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER: Final = (
"X-Mission-Core-Claim-Generation"
)
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
@@ -138,6 +155,13 @@ class ObservatoryWorkerStartRequest(_StrictWorkerRequest):
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
class ObservatoryWorkerRenewRequest(_StrictWorkerRequest):
schema_version: Literal["missioncore.observatory-worker-renew-request/v1"]
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
claim_generation: int = Field(ge=1)
heartbeat_sequence: int = Field(ge=1)
class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest):
schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"]
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
@@ -174,6 +198,8 @@ def build_observatory_worker_router(
queue: ObservatoryRecordedJobQueue,
*,
authentication: ObservatoryWorkerAuthentication,
artifact_transport: PortableObservatoryArtifactTransport | None = None,
result_publisher: PortableObservatoryResultPublisher | None = None,
) -> APIRouter:
"""Build the bounded Worker pull/state-transition router.
@@ -182,6 +208,9 @@ def build_observatory_worker_router(
hashed, and is compared to the configured digest in constant time.
"""
if result_publisher is not None and artifact_transport is None:
raise ValueError("portable result publisher requires artifact transport")
def require_configured_worker(
credentials: Annotated[
HTTPAuthorizationCredentials | None,
@@ -245,6 +274,20 @@ def build_observatory_worker_router(
) -> dict[str, object]:
return _queue_call(lambda: queue.start(job_id, claim_token=request.claim_token)).as_dict()
@router.post("/recorded-jobs/{job_id}/lease/renew")
def renew_job_claim(
request: ObservatoryWorkerRenewRequest,
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
) -> dict[str, object]:
return _queue_call(
lambda: queue.renew_claim(
job_id,
claim_token=request.claim_token,
claim_generation=request.claim_generation,
heartbeat_sequence=request.heartbeat_sequence,
)
).as_dict()
@router.post("/recorded-jobs/{job_id}/checkpoint")
def checkpoint_job(
request: ObservatoryWorkerCheckpointRequest,
@@ -263,14 +306,39 @@ def build_observatory_worker_router(
request: ObservatoryWorkerSucceedRequest,
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
) -> dict[str, object]:
return _queue_call(
if artifact_transport is not None:
_artifact_call(
lambda: artifact_transport.require_completed_for_success(
job_id=job_id,
result_id=request.result_id,
result_sha256=request.result_sha256,
claim_token=request.claim_token,
claimant_id=authentication.contour_id,
)
)
succeeded = _queue_call(
lambda: queue.succeed(
job_id,
claim_token=request.claim_token,
result_id=request.result_id,
result_sha256=request.result_sha256,
)
).as_dict()
)
if artifact_transport is not None and result_publisher is not None:
package_root = _artifact_call(
lambda: artifact_transport.package_root_for_terminal(succeeded)
)
try:
result_publisher.publish(job=succeeded, package_root=package_root)
except PortableResultPublisherError as exc:
raise HTTPException(
status_code=503,
detail=(
"Recorded result is sealed but its verified publication "
"requires reconciliation."
),
) from exc
return succeeded.as_dict()
@router.post("/recorded-jobs/{job_id}/fail")
def fail_job(
@@ -286,6 +354,164 @@ def build_observatory_worker_router(
)
).as_dict()
if artifact_transport is not None:
@router.get("/recorded-jobs/{job_id}/source-materialization")
def source_materialization(
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
claim_token: Annotated[
str,
Header(
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
pattern=_CLAIM_TOKEN_PATTERN,
),
],
claim_generation: Annotated[
int,
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
],
) -> dict[str, object]:
return _artifact_call(
lambda: artifact_transport.source_manifest(
job_id=job_id,
claim_token=claim_token,
claim_generation=claim_generation,
claimant_id=authentication.contour_id,
)
.as_dict()
)
@router.get("/recorded-jobs/{job_id}/source-members/{member_id}")
def source_member(
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
member_id: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
claim_token: Annotated[
str,
Header(
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
pattern=_CLAIM_TOKEN_PATTERN,
),
],
claim_generation: Annotated[
int,
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
],
) -> FileResponse:
member, path = _artifact_call(
lambda: artifact_transport.materialize_source_member(
job_id=job_id,
member_id=member_id,
claim_token=claim_token,
claim_generation=claim_generation,
claimant_id=authentication.contour_id,
)
)
return FileResponse(
path,
media_type=member.media_type,
headers={
"ETag": f'"{member.sha256}"',
"Cache-Control": "private, no-store",
"X-Content-Type-Options": "nosniff",
"X-Mission-Core-Content-Sha256": member.sha256,
},
)
@router.put(
"/recorded-jobs/{job_id}/result-packages/{result_sha256}/manifest"
)
async def stage_result_manifest(
request: Request,
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
result_sha256: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
claim_token: Annotated[
str,
Header(
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
pattern=_CLAIM_TOKEN_PATTERN,
),
],
claim_generation: Annotated[
int,
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
],
) -> dict[str, object]:
payload = await _read_bounded_body(request, MAX_RESULT_MANIFEST_BYTES)
return _artifact_call(
lambda: artifact_transport.stage_result_manifest(
job_id=job_id,
result_sha256=result_sha256,
manifest_payload=payload,
claim_token=claim_token,
claim_generation=claim_generation,
claimant_id=authentication.contour_id,
)
.as_dict()
)
@router.put(
"/recorded-jobs/{job_id}/result-packages/{result_sha256}/members/{member_id}"
)
async def upload_result_member(
request: Request,
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
result_sha256: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
member_id: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
claim_token: Annotated[
str,
Header(
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
pattern=_CLAIM_TOKEN_PATTERN,
),
],
claim_generation: Annotated[
int,
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
],
) -> dict[str, object]:
try:
plan = await artifact_transport.upload_result_member(
job_id=job_id,
result_sha256=result_sha256,
member_id=member_id,
chunks=request.stream(),
claim_token=claim_token,
claim_generation=claim_generation,
claimant_id=authentication.contour_id,
)
except Exception as exc:
_raise_artifact_or_queue_error(exc)
return plan.as_dict()
@router.post(
"/recorded-jobs/{job_id}/result-packages/{result_sha256}/complete"
)
def complete_result_package(
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
result_sha256: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
claim_token: Annotated[
str,
Header(
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
pattern=_CLAIM_TOKEN_PATTERN,
),
],
claim_generation: Annotated[
int,
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
],
) -> dict[str, object]:
return _artifact_call(
lambda: artifact_transport.complete_result_upload(
job_id=job_id,
result_sha256=result_sha256,
claim_token=claim_token,
claim_generation=claim_generation,
claimant_id=authentication.contour_id,
)
.as_dict()
)
return router
@@ -341,3 +567,66 @@ def _queue_call[T](operation: Callable[[], T]) -> T:
status_code=503,
detail="Recorded-job queue is unavailable.",
) from exc
def _artifact_call[T](operation: Callable[[], T]) -> T:
try:
return _queue_call(operation)
except PortableArtifactTransportIntegrityError as exc:
raise HTTPException(
status_code=409,
detail="Worker artifact identity was rejected.",
) from exc
except PortableArtifactTransportUnavailableError as exc:
raise HTTPException(
status_code=409,
detail="Worker artifact member is unavailable for this claim.",
) from exc
except PortableArtifactTransportError as exc:
raise HTTPException(
status_code=503,
detail="Worker artifact transport is unavailable.",
) from exc
def _raise_artifact_or_queue_error(exc: Exception) -> None:
if isinstance(exc, PortableArtifactTransportIntegrityError):
raise HTTPException(
status_code=409,
detail="Worker artifact identity was rejected.",
) from exc
if isinstance(exc, PortableArtifactTransportUnavailableError):
raise HTTPException(
status_code=409,
detail="Worker artifact member is unavailable for this claim.",
) from exc
if isinstance(exc, PortableArtifactTransportError):
raise HTTPException(
status_code=503,
detail="Worker artifact transport is unavailable.",
) from exc
_queue_call(lambda: _raise(exc))
raise AssertionError("unreachable")
def _raise(exc: Exception) -> None:
raise exc
async def _read_bounded_body(request: Request, maximum_bytes: int) -> bytes:
content_length = request.headers.get("content-length")
if content_length is not None:
try:
declared = int(content_length)
except ValueError as exc:
raise HTTPException(status_code=400, detail="Content-Length is invalid.") from exc
if declared < 1 or declared > maximum_bytes:
raise HTTPException(status_code=413, detail="Request body is outside bounds.")
payload = bytearray()
async for chunk in request.stream():
payload.extend(chunk)
if len(payload) > maximum_bytes:
raise HTTPException(status_code=413, detail="Request body is outside bounds.")
if not payload:
raise HTTPException(status_code=400, detail="Request body is empty.")
return bytes(payload)
+24 -5
View File
@@ -41,6 +41,7 @@ from k1link.sessions.canonical_lab_spatial import (
CANONICAL_LAB_SPATIAL_PROFILE,
canonical_lab_spatial_frame,
)
from k1link.sessions.models import SessionSummary
from k1link.sessions.plugin_contract import RecordedPointColorRenderer
from k1link.viewer.recorded import (
APPLICATION_ID as RECORDED_APPLICATION_ID,
@@ -328,6 +329,9 @@ def build_session_router(
perception_overlay_provider: RecordedPerceptionOverlayProvider | None = None,
perception_media_provider: RecordedPerceptionMediaProvider | None = None,
point_color_renderers: Mapping[str, RecordedPointColorRenderer] | None = None,
lab_calculation_profile_resolver: (
Callable[[SessionSummary], Mapping[str, object] | None] | None
) = None,
allow_synchronous_recording_fallback: bool = False,
replay_action_id: str = DEFAULT_REPLAY_ACTION_ID,
) -> APIRouter:
@@ -336,12 +340,29 @@ def build_session_router(
router = APIRouter(tags=["observation-sessions"])
recorded_media_inspector = media_inspector or RecordedMediaInspector()
def lab_catalog_document(
summary: SessionSummary,
contract: Literal["v1", "v2", "v3"],
) -> dict[str, Any]:
lab = summary.lab
if lab is None:
raise ValueError("LAB catalog document requires a LAB summary")
document = lab.as_dict(include_replay_capability=contract in ("v2", "v3"))
if contract == "v3":
profile = (
None
if lab_calculation_profile_resolver is None
else lab_calculation_profile_resolver(summary)
)
document["calculation_profile"] = None if profile is None else dict(profile)
return document
@router.get("/api/v1/observation-sessions")
def list_observation_sessions(
limit: int = Query(default=20, ge=1, le=100),
cursor: str | None = Query(default=None, max_length=128),
scope: Literal["all", "source", "laboratory"] = "all",
lab_contract: Literal["v1", "v2"] = "v1",
lab_contract: Literal["v1", "v2", "v3"] = "v1",
) -> dict[str, Any]:
try:
_refresh_catalog(catalog_refresher)
@@ -349,7 +370,7 @@ def build_session_router(
limit=limit,
cursor=cursor,
scope=scope,
include_capability_projections=lab_contract == "v2",
include_capability_projections=lab_contract in ("v2", "v3"),
)
return {
"items": [
@@ -364,9 +385,7 @@ def build_session_router(
"replayable": item.replayable,
**(
{
"lab": item.lab.as_dict(
include_replay_capability=lab_contract == "v2"
)
"lab": lab_catalog_document(item, lab_contract)
}
if item.lab is not None
else {}